diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index d5f40051f2e..7c4dcf51911 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,3 +1,23 @@ # Contributing to SQLAlchemy -Please see out current Developer Guide at [Develop](https://www.sqlalchemy.org/develop.html) +For general developer guidelines, please see out current Developer Guide at +[Develop](https://www.sqlalchemy.org/develop.html). + +## Note on use of AI, agents and bots ## + +Some of us here use large language models (LLM) to help us with our work, and +some of us are even employer mandated to do so. Getting help whereever you +need is fine. + +However we must ask that **AI/LLM generated content is not spammed onto SQLAlchemy +discussions, issues, or PRs**, whether this is cut and pasted, fully automated, +or even just lightly edited. **Please use your own words and don't come +off like you're a bot**, because that only makes you seem like you're trying +to gamify our organization for unearned gain. + +In particular, **users who post content that appears to be trolling for karma / +upvotes / vanity commits / positive responses, whether or not this content is +machine generated, will be banned**. We are not a casino and we're not here +to be part of gamification of any kind. + + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 865a58c6688..eb0f6a2c38d 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,13 +2,10 @@ blank_issues_enabled: false contact_links: - name: Usage Questions (GitHub Discussions) url: https://github.com/sqlalchemy/sqlalchemy/discussions/new?category=Usage-Questions - about: Questions and Answers for SQLAlchemy Users + about: Questions and Answers for SQLAlchemy Users - name: Live Chat on Gitter url: https://gitter.im/sqlalchemy/community about: Searchable Web-Based Chat - - name: SQLAlchemy Mailing List - url: https://groups.google.com/forum/#!forum/sqlalchemy - about: Over a decade of questions and answers are here - name: Ideas / Feature Proposal (GitHub Discussions) url: https://github.com/sqlalchemy/sqlalchemy/discussions/new?category=Ideas about: Use this for initial discussion for new features and suggestions diff --git a/.github/workflows/create-wheels.yaml b/.github/workflows/create-wheels.yaml index d087afe3c02..15a3c14272e 100644 --- a/.github/workflows/create-wheels.yaml +++ b/.github/workflows/create-wheels.yaml @@ -20,43 +20,62 @@ jobs: matrix: # emulated wheels on linux take too much time, split wheels into multiple runs python: - - "cp39-*" - "cp310-* cp311-*" - - "cp312-* cp313-*" + - "cp312-* cp313-* cp314-*" + - "cp313t-* cp314t-*" wheel_mode: - compiled os: - "windows-2022" - # TODO: macos-14 uses arm macs (only python 3.10+) - make arm wheel on it - - "macos-13" + - "windows-11-arm" + - "macos-15" - "ubuntu-22.04" - "ubuntu-22.04-arm" linux_archs: # this is only meaningful on linux. windows and macos ignore exclude all but one arch - "aarch64" - "x86_64" + - "riscv64" include: # create pure python build - os: ubuntu-22.04 wheel_mode: pure-python - python: "cp-312*" + python: "cp-314*" exclude: - os: "windows-2022" linux_archs: "aarch64" - - os: "macos-13" + # ignored on windows, just avoid to run it multiple times + - os: "windows-11-arm" linux_archs: "aarch64" + - os: "macos-15" + linux_archs: "x86_64" - os: "ubuntu-22.04" linux_archs: "aarch64" - os: "ubuntu-22.04-arm" linux_archs: "x86_64" + - os: "windows-2022" + linux_archs: "riscv64" + - os: "windows-11-arm" + linux_archs: "riscv64" + - os: "macos-15" + linux_archs: "riscv64" + - os: "ubuntu-22.04-arm" + linux_archs: "riscv64" fail-fast: false steps: - uses: actions/checkout@v4 + # See details at https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation + - name: Set up QEMU + if: matrix.linux_archs == 'riscv64' + uses: docker/setup-qemu-action@v3 + with: + platforms: riscv64 + - name: Remove tag-build from pyproject.toml # sqlalchemy has `tag-build` set to `dev` in pyproject.toml. It needs to be removed before creating the wheel # otherwise it gets tagged with `dev0` @@ -69,20 +88,13 @@ jobs: run: | (get-content pyproject.toml) | %{$_ -replace 'tag-build.?=.?"dev"',""} | set-content pyproject.toml - # See details at https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation - # no longer needed since arm runners are now available - # - name: Set up QEMU on linux - # if: ${{ runner.os == 'Linux' }} - # uses: docker/setup-qemu-action@v3 - # with: - # platforms: all - - name: Build compiled wheels if: ${{ matrix.wheel_mode == 'compiled' }} - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v3.3.0 env: CIBW_ARCHS_LINUX: ${{ matrix.linux_archs }} CIBW_BUILD: ${{ matrix.python }} + CIBW_ENABLE: ${{ matrix.python == 'cp313t-* cp314t-*' && 'cpython-freethreading' || '' }} # setting it here does not work on linux # PYTHONNOUSERSITE: "1" @@ -90,7 +102,7 @@ jobs: - name: Set up Python for twine and pure-python wheel uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.14" - name: Build pure-python wheel if: ${{ matrix.wheel_mode == 'pure-python' && runner.os == 'Linux' }} @@ -121,5 +133,5 @@ jobs: # TWINE_PASSWORD: ${{ secrets.test_pypi_token }} TWINE_PASSWORD: ${{ secrets.pypi_token }} run: | - pip install -U twine + python -m pip install -U twine twine upload --skip-existing ./wheelhouse/* diff --git a/.github/workflows/run-on-pr.yaml b/.github/workflows/run-on-pr.yaml index 0d1313bf39c..8cdd78421de 100644 --- a/.github/workflows/run-on-pr.yaml +++ b/.github/workflows/run-on-pr.yaml @@ -25,8 +25,10 @@ jobs: os: - "ubuntu-22.04" python-version: - - "3.12" + - "3.13" + - "3.14" build-type: + - "cext-greenlet" - "cext" - "nocext" architecture: @@ -48,25 +50,25 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade tox setuptools + pip install --upgrade nox setuptools pip list - name: Run tests - run: tox -e github-${{ matrix.build-type }} -- -q --nomemory --notimingintensive ${{ matrix.pytest-args }} + run: nox -v -s github-${{ matrix.build-type }} -- ${{ matrix.pytest-args }} - run-tox: - name: ${{ matrix.tox-env }}-${{ matrix.python-version }} + run-nox: + name: ${{ matrix.nox-env }}-${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: matrix: os: - "ubuntu-22.04" python-version: - - "3.12" - tox-env: + - "3.14" + nox-env: - mypy - - lint - pep484 + - pep8 fail-fast: false @@ -83,8 +85,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade tox setuptools + pip install --upgrade nox setuptools pip list - - name: Run tox - run: tox -e ${{ matrix.tox-env }} ${{ matrix.pytest-args }} + - name: Run nox + run: nox -v -s ${{ matrix.nox-env }} -- ${{ matrix.pytest-args }} diff --git a/.github/workflows/run-test.yaml b/.github/workflows/run-test.yaml index 38e96b250b8..76cab1e04a7 100644 --- a/.github/workflows/run-test.yaml +++ b/.github/workflows/run-test.yaml @@ -29,16 +29,22 @@ jobs: - "ubuntu-22.04" - "ubuntu-22.04-arm" - "windows-latest" + - "windows-11-arm" - "macos-latest" - - "macos-13" python-version: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - - "pypy-3.10" + - "3.14" + - "3.14t" + - "pypy-3.11" build-type: + # builds greenlet, runs asyncio tests. includes aiosqlite driver + - "cext-greenlet" + + # these do not install greenlet at all and skip asyncio tests. + # does not include aiosqlite driver - "cext" - "nocext" architecture: @@ -48,15 +54,21 @@ jobs: include: # autocommit tests fail on the ci for some reason - - python-version: "pypy-3.10" + - python-version: "pypy-3.11" pytest-args: "-k 'not test_autocommit_on and not test_turn_autocommit_off_via_default_iso_level and not test_autocommit_isolation_level'" - - os: "ubuntu-22.04" - pytest-args: "--dbdriver pysqlite --dbdriver aiosqlite" - - os: "ubuntu-22.04-arm" - pytest-args: "--dbdriver pysqlite --dbdriver aiosqlite" - exclude: + + # the threaded pythons are not stable under greenlet. Even + # though we can run individual tests, when you run the whole suite + # with xdist and the greenlet wrapper, the workers keep crashing + # and getting replaced + - build-type: "cext-greenlet" + python-version: "3.13t" + + - build-type: "cext-greenlet" + python-version: "3.14t" + # linux do not have x86 / arm64 python - os: "ubuntu-22.04" architecture: x86 @@ -70,26 +82,31 @@ jobs: # windows des not have arm64 python - os: "windows-latest" architecture: arm64 - # macos: latests uses arm macs. only 3.10+; no x86/x64 + # macos: latests uses arm macs. no x86/x64 - os: "macos-latest" architecture: x86 - os: "macos-latest" architecture: x64 - - os: "macos-latest" - python-version: "3.9" - # macos 13: uses intel macs. no arm64, x86 - - os: "macos-13" - architecture: arm64 - - os: "macos-13" - architecture: x86 # pypy does not have cext or x86 or arm on linux - - python-version: "pypy-3.10" + - python-version: "pypy-3.11" build-type: "cext" - os: "ubuntu-22.04-arm" - python-version: "pypy-3.10" + python-version: "pypy-3.11" - os: "windows-latest" - python-version: "pypy-3.10" + python-version: "pypy-3.11" + architecture: x86 + # Setup-python does not support any versions before 3.11 for arm64 windows + - os: "windows-11-arm" + python-version: "pypy-3.11" + - os: "windows-11-arm" + python-version: "3.10" + - os: "windows-11-arm" architecture: x86 + - os: "windows-11-arm" + architecture: x64 + # 3.14t is not yet supported on windows-11-arm + - os: "windows-11-arm" + python-version: "3.14t" fail-fast: false @@ -104,24 +121,18 @@ jobs: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.architecture }} - - name: Remove greenlet - if: ${{ matrix.no-greenlet == 'true' }} - shell: pwsh - run: | - (cat setup.cfg) | %{$_ -replace "^\s*greenlet.+",""} | set-content setup.cfg - - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade tox setuptools + pip install --upgrade nox setuptools pip list - name: Run tests - run: tox -e github-${{ matrix.build-type }} -- -q --nomemory --notimingintensive ${{ matrix.pytest-args }} - continue-on-error: ${{ matrix.python-version == 'pypy-3.10' }} + run: nox -v -s github-${{ matrix.build-type }} -- ${{ matrix.pytest-args }} + continue-on-error: ${{ matrix.python-version == 'pypy-3.11' }} - run-tox: - name: ${{ matrix.tox-env }}-${{ matrix.python-version }} + run-nox: + name: ${{ matrix.nox-env }}-${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: # run this job using this matrix, excluding some combinations below. @@ -129,24 +140,11 @@ jobs: os: - "ubuntu-22.04" python-version: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - tox-env: + - "3.14" + nox-env: - mypy - pep484 - - include: - # run lint only on 3.12 - - tox-env: lint - python-version: "3.12" - os: "ubuntu-22.04" - exclude: - # run pep484 only on 3.10+ - - tox-env: pep484 - python-version: "3.9" + - pep8 fail-fast: false @@ -164,8 +162,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade tox setuptools + pip install --upgrade nox setuptools pip list - - name: Run tox - run: tox -e ${{ matrix.tox-env }} ${{ matrix.pytest-args }} + - name: Run nox + run: nox -v -e ${{ matrix.nox-env }} ${{ matrix.pytest-args }} diff --git a/.github/workflows/scripts/can_install.py b/.github/workflows/scripts/can_install.py deleted file mode 100644 index ecb24b5623f..00000000000 --- a/.github/workflows/scripts/can_install.py +++ /dev/null @@ -1,25 +0,0 @@ -import sys - -from packaging import tags - -to_check = "--" -found = False -if len(sys.argv) > 1: - to_check = sys.argv[1] - for t in tags.sys_tags(): - start = "-".join(str(t).split("-")[:2]) - if to_check.lower() == start: - print( - "Wheel tag {0} matches installed version {1}.".format( - to_check, t - ) - ) - found = True - break -if not found: - print( - "Wheel tag {0} not found in installed version tags {1}.".format( - to_check, [str(t) for t in tags.sys_tags()] - ) - ) - exit(1) diff --git a/.gitignore b/.gitignore index 2fdd7eb9519..9101cb30d36 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ *.orig *,cover /.tox +/.nox /venv/ .venv *.egg-info diff --git a/.gitreview b/.gitreview index 01d8b1770f7..1be256fc4f0 100644 --- a/.gitreview +++ b/.gitreview @@ -1,4 +1,7 @@ [gerrit] -host=gerrit.sqlalchemy.org +host=ssh.gerrit.sqlalchemy.org project=sqlalchemy/sqlalchemy defaultbranch=main + +# non-standard config, used by publishthing +httphost=gerrit.sqlalchemy.org diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d58505b79f..07d00fbfb08 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,26 @@ -# See https://pre-commit.com for more information -# See https://pre-commit.com/hooks.html for more hooks + +default_language_version: + python: python3.14 + repos: - repo: https://github.com/python/black - rev: 24.10.0 + rev: 25.11.0 hooks: - id: black - repo: https://github.com/sqlalchemyorg/zimports - rev: v0.6.0 + rev: v0.7.0 hooks: - id: zimports - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 + rev: 7.3.0 hooks: - id: flake8 + entry: python -m flake8p additional_dependencies: - - flake8-import-order + - flake8-pyproject + - flake8-import-order>=0.19.2 - flake8-import-single==0.1.5 - flake8-builtins - flake8-future-annotations>=0.0.5 @@ -33,6 +37,8 @@ repos: - id: black-docs name: Format docs code block with black entry: python tools/format_docs_code.py -f - language: system + language: python types: [rst] exclude: README.* + additional_dependencies: + - black==25.9.0 diff --git a/LICENSE b/LICENSE index dfe1a4d815b..903d67495fc 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2005-2025 SQLAlchemy authors and contributors . +Copyright 2005-2026 SQLAlchemy authors and contributors . Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/MANIFEST.in b/MANIFEST.in index 22a39e89c77..a26ab88764b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,5 +15,5 @@ exclude lib/sqlalchemy/**/*.so # don't come in if --with-cextensions isn't specified. recursive-include lib *.pxd *.txt *.typed -include README* AUTHORS LICENSE CHANGES* tox.ini +include README* AUTHORS LICENSE CHANGES* tox.ini noxfile.py prune doc/build/output diff --git a/README.unittests.rst b/README.unittests.rst index 07b93503781..ab7b23ecd01 100644 --- a/README.unittests.rst +++ b/README.unittests.rst @@ -5,30 +5,51 @@ SQLALCHEMY UNIT TESTS Basic Test Running ================== -Tox is used to run the test suite fully. For basic test runs against +Nox is used to run the test suite fully. For basic test runs against a single Python interpreter:: + nox + +The previous runner, tox, still retains functionality in the near term however +will eventually be removed:: + + # still works but deprecated tox -Advanced Tox Options +The newer nox version retains most of the same kinds of functionality as the +tox version, including a custom tagging utility that allows the nox runner +to accept similar "tag" style arguments as were used by the tox runner. + +Advanced Nox Options ==================== -For more elaborate CI-style test running, the tox script provided will +For more elaborate CI-style test running, the nox script provided will run against various Python / database targets. For a basic run against -Python 3.11 using an in-memory SQLite database:: +Python 3.13 using an in-memory SQLite database:: - tox -e py311-sqlite + nox -t py313-sqlite -The tox runner contains a series of target combinations that can run -against various combinations of databases. The test suite can be -run against SQLite with "backend" tests also running against a PostgreSQL -database:: +The nox runner contains a series of target combinations that can run +against each database backend. Unlike the previous tox runner, targets +that refer to multiple database backends at once are no longer +supported at the nox level, in favor of running against multiple tags +instead. So for example to run tests for sqlite and postgresql, while +reducing how many tests run for postgresql to just those that are sensitive +to the database backend:: - tox -e py311-sqlite-postgresql + nox -t py313-sqlite py313-postgresql-backendonly -Or to run just "backend" tests against a MySQL database:: +Where above, the full suite will run against SQLite under Python 3.13, then +the "backend only" version of the suite will for the PostgreSQL database. - tox -e py311-mysql-backendonly +The nox runner, like the tox runner before it, has options for running the +tests with or without the Cython extensions built, with or without greenlet +installed, as well as tags that select or deselect various memory/threading/ +performance intensive tests; the rules for how these environments are selected +should be much more straightforward to understand with nox's imperative +configuration style. For advanced use of nox it's worth it +to poke around ``noxfile.py`` to get a general sense of what varieties +of tests it can run. Running against backends other than SQLite requires that a database of that vendor be available at a specific URL. See "Setting Up Databases" below @@ -37,7 +58,7 @@ for details. The pytest Engine ================= -The tox runner is using pytest to invoke the test suite. Within the realm of +The nox runner uses pytest to invoke the test suite. Within the realm of pytest, SQLAlchemy itself is adding a large series of option and customizations to the pytest runner using plugin points, to allow for SQLAlchemy's multiple database support, database setup/teardown and @@ -127,13 +148,13 @@ Above, we can now run the tests with ``my_postgresql``:: pytest --db my_postgresql We can also override the existing names in our ``test.cfg`` file, so that we can run -with the tox runner also:: +with the nox/tox runners also:: # test.cfg file [db] postgresql=postgresql+psycopg2://username:pass@hostname/dbname -Now when we run ``tox -e py311-postgresql``, it will use our custom URL instead +Now when we run ``nox -t py313-postgresql``, it will use our custom URL instead of the fixed one in setup.cfg. Database Configuration diff --git a/doc/build/Makefile b/doc/build/Makefile index e9684a20738..325da5046e6 100644 --- a/doc/build/Makefile +++ b/doc/build/Makefile @@ -14,6 +14,7 @@ PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +AUTOBUILDSPHINXOPTS = -T . .PHONY: help clean html autobuild dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest dist-html site-mako gettext @@ -48,7 +49,7 @@ html: @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." autobuild: - $(AUTOBUILD) $(ALLSPHINXOPTS) $(BUILDDIR)/html + $(AUTOBUILD) $(AUTOBUILDSPHINXOPTS) $(BUILDDIR)/html gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale diff --git a/doc/build/changelog/changelog_05.rst b/doc/build/changelog/changelog_05.rst index c0125f7dee4..9ae9470c62f 100644 --- a/doc/build/changelog/changelog_05.rst +++ b/doc/build/changelog/changelog_05.rst @@ -1254,7 +1254,7 @@ :tags: postgresql :tickets: 1327 - Refection of unknown PG types won't crash when those + Reflection of unknown PG types won't crash when those types are specified within a domain. .. change:: @@ -2348,7 +2348,7 @@ :tags: general :tickets: - global "propigate"->"propagate" change. + global "propagate"->"propagate" change. .. change:: :tags: orm @@ -3666,7 +3666,7 @@ :tags: general :tickets: - global "propigate"->"propagate" change. + global "propagate"->"propagate" change. .. change:: :tags: orm diff --git a/doc/build/changelog/changelog_07.rst b/doc/build/changelog/changelog_07.rst index 300985f0215..02e0147076a 100644 --- a/doc/build/changelog/changelog_07.rst +++ b/doc/build/changelog/changelog_07.rst @@ -1862,7 +1862,7 @@ There's probably no real-world performance hit here; select() objects are almost always made ad-hoc, and systems that - wish to optimize the re-use of a select() + wish to optimize the reuse of a select() would be using the "compiled_cache" feature. A hit which would occur when calling select.bind has been reduced, but the vast majority diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index d00e043326e..992188a69d7 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -76,7 +76,7 @@ Fixed bug where when using extended attribute instrumentation system, the correct exception would not be raised when :func:`.class_mapper` were called with an invalid input that also happened to not - be weak referencable, such as an integer. + be weak referenceable, such as an integer. .. change:: :tags: bug, tests, pypy diff --git a/doc/build/changelog/changelog_12.rst b/doc/build/changelog/changelog_12.rst index a0187bc8571..4453260d56d 100644 --- a/doc/build/changelog/changelog_12.rst +++ b/doc/build/changelog/changelog_12.rst @@ -1395,7 +1395,7 @@ .. change:: :tags: feature, orm - Added new argument :paramref:`.attributes.set_attribute.inititator` + Added new argument :paramref:`.attributes.set_attribute.initiator` to the :func:`.attributes.set_attribute` function, allowing an event token received from a listener function to be propagated to subsequent set events. diff --git a/doc/build/changelog/changelog_14.rst b/doc/build/changelog/changelog_14.rst index e2d2f4d6c92..0f18ece0b6b 100644 --- a/doc/build/changelog/changelog_14.rst +++ b/doc/build/changelog/changelog_14.rst @@ -5045,7 +5045,7 @@ This document details individual issue-level changes made throughout Fixed issue where using a :class:`_sql.Select` as a subquery in an ORM context would modify the :class:`_sql.Select` in place to disable eagerloads on that object, which would then cause that same - :class:`_sql.Select` to not eagerload if it were then re-used in a + :class:`_sql.Select` to not eagerload if it were then reused in a top-level execution context. @@ -5380,7 +5380,7 @@ This document details individual issue-level changes made throughout :tags: usecase, orm :tickets: 6267 - Established support for :func:`_orm.synoynm` in conjunction with + Established support for :func:`_orm.synonym` in conjunction with hybrid property, assocaitionproxy is set up completely, including that synonyms can be established linking to these constructs which work fully. This is a behavior that was semi-explicitly disallowed previously, @@ -6663,9 +6663,10 @@ This document details individual issue-level changes made throughout :meth:`_sql.GenerativeSelect.apply_labels` with explicit getters and setters :meth:`_sql.GenerativeSelect.get_label_style` and :meth:`_sql.GenerativeSelect.set_label_style` to accommodate the three - supported label styles: :data:`_sql.LABEL_STYLE_DISAMBIGUATE_ONLY`, - :data:`_sql.LABEL_STYLE_TABLENAME_PLUS_COL`, and - :data:`_sql.LABEL_STYLE_NONE`. + supported label styles: + :attr:`_sql.SelectLabelStyle.LABEL_STYLE_DISAMBIGUATE_ONLY`, + :attr:`_sql.SelectLabelStyle.LABEL_STYLE_TABLENAME_PLUS_COL`, and + :attr:`_sql.SelectLabelStyle.LABEL_STYLE_NONE`. In addition, for Core and "future style" ORM queries, ``LABEL_STYLE_DISAMBIGUATE_ONLY`` is now the default label style. This @@ -6690,7 +6691,7 @@ This document details individual issue-level changes made throughout :tickets: 5735 Improved the unit of work topological sorting system such that the - toplogical sort is now deterministic based on the sorting of the input set, + topological sort is now deterministic based on the sorting of the input set, which itself is now sorted at the level of mappers, so that the same inputs of affected mappers should produce the same output every time, among mappers / tables that don't have any dependency on each other. This further @@ -7945,7 +7946,7 @@ This document details individual issue-level changes made throughout The bulk update and delete methods :meth:`.Query.update` and :meth:`.Query.delete`, as well as their 2.0-style counterparts, now make use of RETURNING when the "fetch" strategy is used in order to fetch the - list of affected primary key identites, rather than emitting a separate + list of affected primary key identities, rather than emitting a separate SELECT, when the backend in use supports RETURNING. Additionally, the "fetch" strategy will in ordinary cases not expire the attributes that have been updated, and will instead apply the updated values directly in the diff --git a/doc/build/changelog/changelog_20.rst b/doc/build/changelog/changelog_20.rst index 38ed6399c9a..3345d1185de 100644 --- a/doc/build/changelog/changelog_20.rst +++ b/doc/build/changelog/changelog_20.rst @@ -9,9 +9,1171 @@ .. changelog:: - :version: 2.0.40 + :version: 2.0.50 :include_notes_from: unreleased_20 +.. changelog:: + :version: 2.0.49 + :released: April 3, 2026 + + .. change:: + :tags: postgresql, bug + :tickets: 10902 + + Fixed regular expression used when reflecting foreign keys in PostgreSQL to + support escaped quotes in table names. + Pull request courtesy of Austin Graham + + .. change:: + :tags: bug, oracle + :tickets: 13150 + + Fixed issue in Oracle dialect where the :class:`_oracle.RAW` datatype would + not reflect the length parameter. Pull request courtesy Daniel Sullivan. + + + .. change:: + :tags: usecase, mssql + :tickets: 13152 + + Enhanced the ``aioodbc`` dialect to expose the ``fast_executemany`` + attribute of the pyodbc cursor. This allows the ``fast_executemany`` + parameter to work with the ``mssql+aioodbc`` dialect. Pull request + courtesy Georg Sieber. + + .. change:: + :tags: bug, typing + :tickets: 13167 + + Fixed a typing issue where the typed members of :data:`.func` would return + the appropriate class of the same name, however this creates an issue for + typecheckers such as Zuban and pyrefly that assume :pep:`749` style + typechecking even if the file states that it's a :pep:`563` file; they see + the returned name as indicating the method object and not the class object. + These typecheckers are actually following along with an upcoming test + harness that insists on :pep:`749` style name resolution for this case + unconditionally. Since :pep:`749` is the way of the future regardless, + differently-named type aliases have been added for these return types. + + + .. change:: + :tags: bug, orm + :tickets: 13176 + + Fixed issue where :meth:`_orm.Session.get` would bypass the identity map + and emit unnecessary SQL when ``with_for_update=False`` was passed, + rather than treating it equivalently to the default of ``None``. + Pull request courtesy of Joshua Swanson. + + .. change:: + :tags: bug, mssql, reflection + :tickets: 13181, 13182 + + Fixed regression from version 2.0.42 caused by :ticket:`12654` where the + updated column reflection query would receive SQL Server "type alias" names + for special types such as ``sysname``, whereas previously the base name + would be received (e.g. ``nvarchar`` for ``sysname``), leading to warnings + that such types could not be reflected and resulting in :class:`.NullType`, + rather than the expected :class:`.NVARCHAR` for a type like ``sysname``. + The column reflection query now joins ``sys.types`` a second time to look + up the base type when the user type name is not present in + :attr:`.MSDialect.ischema_names`, and both names are checked in + :attr:`.MSDialect.ischema_names` for a match. Pull request courtesy Carlos + Serrano. + + .. change:: + :tags: mssql, usecase + :tickets: 13185 + + Remove warning for SQL Server dialect when a new version is detected. + The warning was originally added more than 15 years ago due to an unexpected + value returned when using an old version of FreeTDS. + The assumption is that since then the issue has been resolved, so make the + SQL Server dialect behave like the other ones that don't have an upper bound + check on the version number. + + .. change:: + :tags: bug, orm + :tickets: 13193 + + Fixed issue where chained :func:`_orm.joinedload` options would not be + applied correctly when the final relationship in the chain is declared on a + base mapper and accessed through a subclass mapper in a + :func:`_orm.with_polymorphic` query. The path registry now correctly + computes the natural path when a property declared on a base class is + accessed through a path containing a subclass mapper, ensuring the loader + option can be located during query compilation. + + .. change:: + :tags: bug, orm, inheritance + :tickets: 13202 + + Fixed issue where using :meth:`_orm.Load.options` to apply a chained loader + option such as :func:`_orm.joinedload` or :func:`_orm.selectinload` with + :meth:`_orm.PropComparator.of_type` for a polymorphic relationship would + not generate the necessary clauses for the polymorphic subclasses. The + polymorphic loading strategy is now correctly propagated when using a call + such as ``joinedload(A.b).options(joinedload(B.c.of_type(poly)))`` to match + the behavior of direct chaining e.g. + ``joinedload(A.b).joinedload(B.c.of_type(poly))``. + + .. change:: + :tags: bug, orm, inheritance + :tickets: 13209 + + Fixed issue where using chained loader options such as + :func:`_orm.selectinload` after :func:`_orm.joinedload` with + :meth:`_orm.PropComparator.of_type` for a polymorphic relationship would + not properly apply the chained loader option. The loader option is now + correctly applied when using a call such as + ``joinedload(A.b.of_type(poly)).selectinload(poly.SubClass.c)`` to eagerly + load related objects. + +.. changelog:: + :version: 2.0.48 + :released: March 2, 2026 + + .. change:: + :tags: bug, engine + :tickets: 13144 + + Fixed a critical issue in :class:`.Engine` where connections created in + conjunction with the :meth:`.DialectEvents.do_connect` event listeners + would receive shared, mutable collections for the connection arguments, + leading to a variety of potential issues including unlimited growth of the + argument list as well as elements within the parameter dictionary being + shared among concurrent connection calls. In particular this could impact + do_connect routines making use of complex mutable authentication + structures. + +.. changelog:: + :version: 2.0.47 + :released: February 24, 2026 + + .. change:: + :tags: bug, orm + :tickets: 13104 + + Fixed issue when using ORM mappings with Python 3.14's :pep:`649` feature + that no longer requires "future annotations", where the ORM's introspection + of the ``__init__`` method of mapped classes would fail if non-present + identifiers in annotations were present. The vendored ``getfullargspec()`` + method has been amended to use ``Format.FORWARDREF`` under Python 3.14 to + prevent resolution of names that aren't present. + + + .. change:: + :tags: bug, postgresql + :tickets: 13105 + + Fixed an issue in the PostgreSQL dialect where foreign key constraint + reflection would incorrectly swap or fail to capture ``onupdate`` and + ``ondelete`` values when these clauses appeared in a different order than + expected in the constraint definition. This issue primarily affected + PostgreSQL-compatible databases such as CockroachDB, which may return ``ON + DELETE`` before ``ON UPDATE`` in the constraint definition string. The + reflection logic now correctly parses both clauses regardless of their + ordering. + + .. change:: + :tags: bug, postgresql + :tickets: 13107 + + Fixed issue in the :ref:`engine_insertmanyvalues` feature where using + PostgreSQL's ``ON CONFLICT`` clause with + :paramref:`_dml.Insert.returning.sort_by_parameter_order` enabled would + generate invalid SQL when the insert used an implicit sentinel (server-side + autoincrement primary key). The generated SQL would incorrectly declare a + sentinel counter column in the ``imp_sen`` table alias without providing + corresponding values in the ``VALUES`` clause, leading to a + ``ProgrammingError`` indicating column count mismatch. The fix allows batch + execution mode when ``embed_values_counter`` is active, as the embedded + counter provides the ordering capability needed even with upsert behaviors, + rather than unnecessarily downgrading to row-at-a-time execution. + + .. change:: + :tags: bug, postgresql + :tickets: 13110 + + Fixed issue where :meth:`_postgresql.Insert.on_conflict_do_update` + parameters were not respecting compilation options such as + ``literal_binds=True``. Pull request courtesy Loïc Simon. + + + .. change:: + :tags: bug, sqlite + :tickets: 13110 + + Fixed issue where :meth:`_sqlite.Insert.on_conflict_do_update` + parameters were not respecting compilation options such as + ``literal_binds=True``. Pull request courtesy Loïc Simon. + + .. change:: + :tags: usecase, engine + :tickets: 13116 + + The connection object returned by :meth:`_engine.Engine.raw_connection` + now supports the context manager protocol, automatically returning the + connection to the pool when exiting the context. + + .. change:: + :tags: bug, postgresql + :tickets: 13130 + + Fixed issue where :meth:`_postgresql.Insert.on_conflict_do_update` + using parametrized bound parameters in the ``set_`` clause would fail + when used with executemany batching. For dialects that use the + ``use_insertmanyvalues_wo_returning`` optimization (psycopg2), + insertmanyvalues is now disabled when there is an ON CONFLICT clause. + For cases with RETURNING, row-at-a-time mode is used when the SET + clause contains parametrized bindparams (bindparams that receive + values from the parameters dict), ensuring each row's parameters are + correctly applied. ON CONFLICT statements using expressions like + ``excluded.`` continue to batch normally. + + + .. change:: + :tags: bug, sqlite + :tickets: 13130 + + Fixed issue where :meth:`_sqlite.Insert.on_conflict_do_update` + using parametrized bound parameters in the ``set_`` clause would fail + when used with executemany batching. Row-at-a-time mode is now used + for ON CONFLICT statements with RETURNING that contain parametrized + bindparams, ensuring each row's parameters are correctly applied. ON + CONFLICT statements using expressions like ``excluded.`` + continue to batch normally. + + .. change:: + :tags: bug, mysql + :tickets: 13134 + + Fixed issue where DDL compilation options were registered to the hard-coded + dialect name ``mysql``. This made it awkward for MySQL-derived dialects + like MariaDB, StarRocks, etc. to work with such options when different sets + of options exist for different platforms. Options are now registered under + the actual dialect name, and a fallback was added to help avoid errors when + an option does not exist for that dialect. + + To maintain backwards compatibility, when using the MariaDB dialect with + the options ``mysql_with_parser`` or ``mysql_using`` without also specifying + the corresponding ``mariadb_`` prefixed options, a deprecation warning will + be emitted. The ``mysql_`` prefixed options will continue to work during + the deprecation period. Users should update their code to additionally + specify ``mariadb_with_parser`` and ``mariadb_using`` when using the + ``mariadb://`` dialect, or specify both options to support both dialects. + + Pull request courtesy Tiansu Yu. + +.. changelog:: + :version: 2.0.46 + :released: January 21, 2026 + + .. change:: + :tags: bug, sqlite + :tickets: 13039 + + Fixed issue in the aiosqlite driver where SQLAlchemy's setting of + aiosqlite's worker thread to "daemon" stopped working because the aiosqlite + architecture moved the location of the worker thread in version 0.22.0. + This "daemon" flag is necessary so that a program is able to exit if the + SQLite connection itself was not explicitly closed, which is particularly + likely with SQLAlchemy as it maintains SQLite connections in a connection + pool. While it's perfectly fine to call :meth:`.AsyncEngine.dispose` + before program exit, this is not historically or technically necessary for + any driver of any known backend, since a primary feature of relational + databases is durability. The change also implements support for + "terminate" with aiosqlite when using version version 0.22.1 or greater, + which implements a sync ``.stop()`` method. + + .. change:: + :tags: usecase, mssql + :tickets: 13045 + + Added support for the ``IF EXISTS`` clause when dropping indexes on SQL + Server 2016 (13.x) and later versions. The :paramref:`.DropIndex.if_exists` + parameter is now honored by the SQL Server dialect, allowing conditional + index drops that will not raise an error if the index does not exist. + Pull request courtesy Edgar Ramírez Mondragón. + + .. change:: + :tags: bug, postgresql + :tickets: 13059 + + Fixed issue where PostgreSQL JSONB operators + :meth:`_postgresql.JSONB.Comparator.path_match` and + :meth:`_postgresql.JSONB.Comparator.path_exists` were applying incorrect + ``VARCHAR`` casts to the right-hand side operand when used with newer + PostgreSQL drivers such as psycopg. The operators now indicate the + right-hand type as ``JSONPATH``, which currently results in no casting + taking place, but is also compatible with explicit casts if the + implementation were require it at a later point. + + + + .. change:: + :tags: bug, postgresql + :tickets: 13067 + + Fixed regression in PostgreSQL dialect where JSONB subscription syntax + would generate incorrect SQL for :func:`.cast` expressions returning JSONB, + causing syntax errors. The dialect now properly wraps cast expressions in + parentheses when using the ``[]`` subscription syntax, generating + ``(CAST(...))[index]`` instead of ``CAST(...)[index]`` to comply with + PostgreSQL syntax requirements. This extends the fix from :ticket:`12778` + which addressed the same issue for function calls. + + .. change:: + :tags: bug, mariadb + :tickets: 13070 + + Fixed the SQL compilation for the mariadb sequence "NOCYCLE" keyword that + is to be emitted when the :paramref:`.Sequence.cycle` parameter is set to + False on a :class:`.Sequence`. Pull request courtesy Diego Dupin. + + .. change:: + :tags: bug, typing + :tickets: 13075 + + Fixed typing issues where ORM mapped classes and aliased entities could not + be used as keys in result row mappings or as join targets in select + statements. Patterns such as ``row._mapping[User]``, + ``row._mapping[aliased(User)]``, ``row._mapping[with_polymorphic(...)]`` + (rejected by both mypy and Pylance), and ``.join(aliased(User))`` + (rejected by Pylance) are documented and fully supported at runtime but + were previously rejected by type checkers. The type definitions for + :class:`._KeyType` and :class:`._FromClauseArgument` have been updated to + accept these ORM entity types. + + .. change:: + :tags: bug, postgresql + + Improved the foreign key reflection regular expression pattern used by the + PostgreSQL dialect to be more permissive in matching identifier characters, + allowing it to correctly handle unicode characters in table and column + names. This change improves compatibility with PostgreSQL variants such as + CockroachDB that may use different quoting patterns in combination with + unicode characters in their identifiers. Pull request courtesy Gord + Thompson. + +.. changelog:: + :version: 2.0.45 + :released: December 9, 2025 + + .. change:: + :tags: bug, typing + :tickets: 12730 + + Fixed typing issue where :meth:`.Select.with_for_update` would not support + lists of ORM entities or other FROM clauses in the + :paramref:`.Select.with_for_update.of` parameter. Pull request courtesy + Shamil. + + .. change:: + :tags: bug, orm + :tickets: 12858 + + Fixed issue where calling :meth:`.Mapper.add_property` within mapper event + hooks such as :meth:`.MapperEvents.instrument_class`, + :meth:`.MapperEvents.after_mapper_constructed`, or + :meth:`.MapperEvents.before_mapper_configured` would raise an + ``AttributeError`` because the mapper's internal property collections were + not yet initialized. The :meth:`.Mapper.add_property` method now handles + early-stage property additions correctly, allowing properties including + column properties, deferred columns, and relationships to be added during + mapper initialization events. Pull request courtesy G Allajmi. + + .. change:: + :tags: bug, postgresql + :tickets: 12867 + + Fixed issue where PostgreSQL dialect options such as ``postgresql_include`` + on :class:`.PrimaryKeyConstraint` and :class:`.UniqueConstraint` were + rendered in the wrong position when combined with constraint deferrability + options like ``deferrable=True``. Pull request courtesy G Allajmi. + + .. change:: + :tags: bug, sql + :tickets: 12915 + + Some improvements to the :meth:`_sql.ClauseElement.params` method to + replace bound parameters in a query were made, however the ultimate issue + in :ticket:`12915` involving ORM :func:`_orm.aliased` cannot be fixed fully + until 2.1, where the method is being rewritten to work without relying on + Core cloned traversal. + + .. change:: + :tags: bug, sqlite, reflection + :tickets: 12924 + + A series of improvements have been made for reflection of CHECK constraints + on SQLite. The reflection logic now correctly handles table names + containing the strings "CHECK" or "CONSTRAINT", properly supports all four + SQLite identifier quoting styles (double quotes, single quotes, brackets, + and backticks) for constraint names, and accurately parses CHECK constraint + expressions containing parentheses within string literals using balanced + parenthesis matching with string context tracking. Big thanks to + GruzdevAV for new test cases and implementation ideas. + + .. change:: + :tags: bug, orm + :tickets: 12952 + + Fixed issue in Python 3.14 where dataclass transformation would fail when + a mapped class using :class:`.MappedAsDataclass` included a + :func:`.relationship` referencing a class that was not available at + runtime (e.g., within a ``TYPE_CHECKING`` block). This occurred when using + Python 3.14's :pep:`649` deferred annotations feature, which is the + default behavior without a ``from __future__ import annotations`` + directive. + + .. change:: + :tags: bug, sqlite + :tickets: 12954 + + Fixed issue where SQLite dialect would fail to reflect constraint names + that contained uppercase letters or other characters requiring quoting. The + regular expressions used to parse primary key, foreign key, and unique + constraint names from the ``CREATE TABLE`` statement have been updated to + properly handle both quoted and unquoted constraint names. + + .. change:: + :tags: bug, typing + + Fixed typing issue where :class:`.coalesce` would not return the correct + return type when a nullable form of that argument were passed, even though + this function is meant to select the non-null entry among possibly null + arguments. Pull request courtesy Yannick PÉROUX. + + + .. change:: + :tags: usecase, mysql + :tickets: 12964 + + Added support for MySQL 8.0.1 + ``FOR SHARE`` to be emitted for the + :meth:`.Select.with_for_update` method, which offers compatibility with + ``NOWAIT`` and ``SKIP LOCKED``. The new syntax is used only for MySQL when + version 8.0.1 or higher is detected. Pull request courtesy JetDrag. + + .. change:: + :tags: bug, sql + :tickets: 12987 + + Fixed issue where using the :meth:`.ColumnOperators.in_` operator with a + nested :class:`.CompoundSelect` statement (e.g. an ``INTERSECT`` of + ``UNION`` queries) would raise a :class:`NotImplementedError` when the + nested compound select was the first argument to the outer compound select. + The ``_scalar_type()`` internal method now properly handles nested compound + selects. + + .. change:: + :tags: bug, postgresql + :tickets: 13015 + + Fixed the structure of the SQL string used for the + :ref:`engine_insertmanyvalues` feature when an explicit sequence with + ``nextval()`` is used. The SQL function invocation for the sequence has + been moved from being rendered inline within each tuple inside of VALUES to + being rendered once in the SELECT that reads from VALUES. This change + ensures the function is invoked in the correct order as rows are processed, + rather than assuming PostgreSQL will execute inline function calls within + VALUES in a particular order. While current PostgreSQL versions appear to + handle the previous approach correctly, the database does not guarantee + this behavior for future versions. + + .. change:: + :tags: usecase, postgresql + :tickets: 6511 + + Added support for reflection of collation in types for PostgreSQL. + The ``collation`` will be set only if different from the default + one for the type. + Pull request courtesy Denis Laxalde. + + .. change:: + :tags: bug, examples + + Fixed the "short_selects" performance example where the cache was being + used in all the examples, making it impossible to compare performance with + and without the cache. Less important comparisons like "lambdas" and + "baked queries" have been removed. + + + .. change:: + :tags: change, tests + + A noxfile.py has been added to allow testing with nox. This is a direct + port of 2.1's move to nox, however leaves the tox.ini file in place and + retains all test documentation in terms of tox. Version 2.1 will move to + nox fully, including deprecation warnings for tox and new testing + documentation. + +.. changelog:: + :version: 2.0.44 + :released: October 10, 2025 + + .. change:: + :tags: bug, sql + :tickets: 12271 + + Improved the implementation of :meth:`.UpdateBase.returning` to use more + robust logic in setting up the ``.c`` collection of a derived statement + such as a CTE. This fixes issues related to RETURNING clauses that feature + expressions based on returned columns with or without qualifying labels. + + .. change:: + :tags: usecase, asyncio + :tickets: 12273 + + Generalize the terminate logic employed by the asyncpg dialect to reuse + it in the aiomysql and asyncmy dialect implementation. + + .. change:: + :tags: bug, mssql + :tickets: 12798 + + Improved the base implementation of the asyncio cursor such that it + includes the option for the underlying driver's cursor to be actively + closed in those cases where it requires ``await`` in order to complete the + close sequence, rather than relying on garbage collection to "close" it, + when a plain :class:`.Result` is returned that does not use ``await`` for + any of its methods. The previous approach of relying on gc was fine for + MySQL and SQLite dialects but has caused problems with the aioodbc + implementation on top of SQL Server. The new option is enabled + for those dialects which have an "awaitable" ``cursor.close()``, which + includes the aioodbc, aiomysql, and asyncmy dialects (aiosqlite is also + modified for 2.1 only). + + .. change:: + :tags: bug, ext + :tickets: 12802 + + Fixed issue caused by an unwanted functional change while typing + the :class:`.MutableList` class. + This change also reverts all other functional changes done in + the same change. + + .. change:: + :tags: bug, typing + :tickets: 12813 + + Fixed typing bug where the :meth:`.Session.execute` method advertised that + it would return a :class:`.CursorResult` if given an insert/update/delete + statement. This is not the general case as several flavors of ORM + insert/update do not actually yield a :class:`.CursorResult` which cannot + be differentiated at the typing overload level, so the method now yields + :class:`.Result` in all cases. For those cases where + :class:`.CursorResult` is known to be returned and the ``.rowcount`` + attribute is required, please use ``typing.cast()``. + + .. change:: + :tags: usecase, orm + :tickets: 12829 + + The way ORM Annotated Declarative interprets Python :pep:`695` type aliases + in ``Mapped[]`` annotations has been refined to expand the lookup scheme. A + :pep:`695` type can now be resolved based on either its direct presence in + :paramref:`_orm.registry.type_annotation_map` or its immediate resolved + value, as long as a recursive lookup across multiple :pep:`695` types is + not required for it to resolve. This change reverses part of the + restrictions introduced in 2.0.37 as part of :ticket:`11955`, which + deprecated (and disallowed in 2.1) the ability to resolve any :pep:`695` + type that was not explicitly present in + :paramref:`_orm.registry.type_annotation_map`. Recursive lookups of + :pep:`695` types remains deprecated in 2.0 and disallowed in version 2.1, + as do implicit lookups of ``NewType`` types without an entry in + :paramref:`_orm.registry.type_annotation_map`. + + Additionally, new support has been added for generic :pep:`695` aliases that + refer to :pep:`593` ``Annotated`` constructs containing + :func:`_orm.mapped_column` configurations. See the sections below for + examples. + + .. seealso:: + + :ref:`orm_declarative_type_map_pep695_types` + + :ref:`orm_declarative_mapped_column_generic_pep593` + + .. change:: + :tags: bug, postgresql + :tickets: 12847 + + Fixed issue where selecting an enum array column containing NULL values + would fail to parse properly in the PostgreSQL dialect. The + :func:`._split_enum_values` function now correctly handles NULL entries by + converting them to Python ``None`` values. + + .. change:: + :tags: bug, typing + :tickets: 12855 + + Added new decorator :func:`_orm.mapped_as_dataclass`, which is a function + based form of :meth:`_orm.registry.mapped_as_dataclass`; the method form + :meth:`_orm.registry.mapped_as_dataclass` does not seem to be correctly + recognized within the scope of :pep:`681` in recent mypy versions. + + .. change:: + :tags: bug, sqlite + :tickets: 12864 + + Fixed issue where SQLite table reflection would fail for tables using + ``WITHOUT ROWID`` and/or ``STRICT`` table options when the table contained + generated columns. The regular expression used to parse ``CREATE TABLE`` + statements for generated column detection has been updated to properly + handle these SQLite table options that appear after the column definitions. + Pull request courtesy Tip ten Brink. + + .. change:: + :tags: bug, postgresql + :tickets: 12874 + + Fixed issue where the :func:`_sql.any_` and :func:`_sql.all_` aggregation + operators would not correctly coerce the datatype of the compared value, in + those cases where the compared value were not a simple int/str etc., such + as a Python ``Enum`` or other custom value. This would lead to execution + time errors for these values. This issue is essentially the same as + :ticket:`6515` which was for the now-legacy :meth:`.ARRAY.any` and + :meth:`.ARRAY.all` methods. + + .. change:: + :tags: bug, engine + :tickets: 12881 + + Implemented initial support for free-threaded Python by adding new tests + and reworking the test harness to include Python 3.13t and Python 3.14t in + test runs. Two concurrency issues have been identified and fixed: the first + involves initialization of the ``.c`` collection on a ``FromClause``, a + continuation of :ticket:`12302`, where an optional mutex under + free-threading is added; the second involves synchronization of the pool + "first_connect" event, which first received thread synchronization in + :ticket:`2964`, however under free-threading the creation of the mutex + itself runs under the same free-threading mutex. Support for free-threaded + wheels on Pypi is implemented as well within the 2.1 series only. Initial + pull request and test suite courtesy Lysandros Nikolaou. + + .. change:: + :tags: bug, schema + :tickets: 12884 + + Fixed issue where :meth:`_schema.MetaData.reflect` did not forward + dialect-specific keyword arguments to the :class:`_engine.Inspector` + methods, causing options like ``oracle_resolve_synonyms`` to be ignored + during reflection. The method now ensures that all extra kwargs passed to + :meth:`_schema.MetaData.reflect` are forwarded to + :meth:`_engine.Inspector.get_table_names` and related reflection methods. + Pull request courtesy Lukáš Kožušník. + + .. change:: + :tags: bug, mssql + :tickets: 12894 + + Fixed issue where the index reflection for SQL Server would + not correctly return the order of the column inside an index + when the order of the columns in the index did not match the + order of the columns in the table. + Pull request courtesy of Allen Chen. + + .. change:: + :tags: bug, orm + :tickets: 12905 + + Fixed a caching issue where :func:`_orm.with_loader_criteria` would + incorrectly reuse cached bound parameter values when used with + :class:`_sql.CompoundSelect` constructs such as :func:`_sql.union`. The + issue was caused by the cache key for compound selects not including the + execution options that are part of the :class:`_sql.Executable` base class, + which :func:`_orm.with_loader_criteria` uses to apply its criteria + dynamically. The fix ensures that compound selects and other executable + constructs properly include execution options in their cache key traversal. + + .. change:: + :tags: bug, mssql, reflection + :tickets: 12907 + + Fixed issue in the MSSQL dialect's foreign key reflection query where + duplicate rows could be returned when a foreign key column and its + referenced primary key column have the same name, and both the referencing + and referenced tables have indexes with the same name. This resulted in an + "ForeignKeyConstraint with duplicate source column references are not + supported" error when attempting to reflect such tables. The query has been + corrected to exclude indexes on the child table when looking for unique + indexes referenced by foreign keys. + + .. change:: + :tags: bug, platform + + Unblocked automatic greenlet installation for Python 3.14 now that + there are greenlet wheels on pypi for python 3.14. + +.. changelog:: + :version: 2.0.43 + :released: August 11, 2025 + + .. change:: + :tags: usecase, oracle + :tickets: 12711 + + Extended :class:`_oracle.VECTOR` to support sparse vectors. This update + introduces :class:`_oracle.VectorStorageType` to specify sparse or dense + storage and added :class:`_oracle.SparseVector`. Pull request courtesy + Suraj Shaw. + + .. change:: + :tags: bug, orm + :tickets: 12748 + + Fixed issue where using the ``post_update`` feature would apply incorrect + "pre-fetched" values to the ORM objects after a multi-row UPDATE process + completed. These "pre-fetched" values would come from any column that had + an :paramref:`.Column.onupdate` callable or a version id generator used by + :paramref:`.orm.Mapper.version_id_generator`; for a version id generator + that delivered random identifiers like timestamps or UUIDs, this incorrect + data would lead to a DELETE statement against those same rows to fail in + the next step. + + + .. change:: + :tags: bug, postgresql + :tickets: 12778 + + Fixed regression in PostgreSQL dialect where JSONB subscription syntax + would generate incorrect SQL for JSONB-returning functions, causing syntax + errors. The dialect now properly wraps function calls and expressions in + parentheses when using the ``[]`` subscription syntax, generating + ``(function_call)[index]`` instead of ``function_call[index]`` to comply + with PostgreSQL syntax requirements. + + .. change:: + :tags: usecase, engine + :tickets: 12784 + + Added new parameter :paramref:`.create_engine.skip_autocommit_rollback` + which provides for a per-dialect feature of preventing the DBAPI + ``.rollback()`` from being called under any circumstances, if the + connection is detected as being in "autocommit" mode. This improves upon + a critical performance issue identified in MySQL dialects where the network + overhead of the ``.rollback()`` call remains prohibitive even if autocommit + mode is set. + + .. seealso:: + + :ref:`dbapi_autocommit_skip_rollback` + + .. change:: + :tags: bug, orm + :tickets: 12787 + + Fixed issue where :paramref:`_orm.mapped_column.use_existing_column` + parameter in :func:`_orm.mapped_column` would not work when the + :func:`_orm.mapped_column` is used inside of an ``Annotated`` type alias in + polymorphic inheritance scenarios. The parameter is now properly recognized + and processed during declarative mapping configuration. + + .. change:: + :tags: bug, orm + :tickets: 12790 + + Improved the implementation of the :func:`_orm.selectin_polymorphic` + inheritance loader strategy to properly render the IN expressions using + chunks of 500 records each, in the same manner as that of the + :func:`_orm.selectinload` relationship loader strategy. Previously, the IN + expression would be arbitrarily large, leading to failures on databases + that have limits on the size of IN expressions including Oracle Database. + +.. changelog:: + :version: 2.0.42 + :released: July 29, 2025 + + .. change:: + :tags: usecase, orm + :tickets: 10674 + + Added ``dataclass_metadata`` argument to all ORM attribute constructors + that accept dataclasses parameters, e.g. :paramref:`.mapped_column.dataclass_metadata`, + :paramref:`.relationship.dataclass_metadata`, etc. + It's passed to the underlying dataclass ``metadata`` attribute + of the dataclass field. Pull request courtesy Sigmund Lahn. + + .. change:: + :tags: usecase, postgresql + :tickets: 10927 + + Added support for PostgreSQL 14+ :class:`.JSONB` subscripting syntax. + When connected to PostgreSQL 14 or later, JSONB columns now + automatically use the native subscript notation ``jsonb_col['key']`` + instead of the arrow operator ``jsonb_col -> 'key'`` for both read and + write operations. This provides better compatibility with PostgreSQL's + native JSONB subscripting feature while maintaining backward + compatibility with older PostgreSQL versions. JSON columns continue to + use the traditional arrow syntax regardless of PostgreSQL version. + + .. warning:: + + **For applications that have indexes against JSONB subscript + expressions** + + This change caused an unintended side effect for indexes that were + created against expressions that use subscript notation, e.g. + ``Index("ix_entity_json_ab_text", data["a"]["b"].astext)``. If these + indexes were generated with the older syntax e.g. ``((entity.data -> + 'a') ->> 'b')``, they will not be used by the PostgreSQL query + planner when a query is made using SQLAlchemy 2.0.42 or higher on + PostgreSQL versions 14 or higher. This occurs because the new text + will resemble ``(entity.data['a'] ->> 'b')`` which will fail to + produce the exact textual syntax match required by the PostgreSQL + query planner. Therefore, for users upgrading to SQLAlchemy 2.0.42 + or higher, existing indexes that were created against :class:`.JSONB` + expressions that use subscripting would need to be dropped and + re-created in order for them to work with the new query syntax, e.g. + an expression like ``((entity.data -> 'a') ->> 'b')`` would become + ``(entity.data['a'] ->> 'b')``. + + .. seealso:: + + :ticket:`12868` - discussion of this issue + + .. change:: + :tags: bug, orm + :tickets: 12593 + + Implemented the :func:`_orm.defer`, :func:`_orm.undefer` and + :func:`_orm.load_only` loader options to work for composite attributes, a + use case that had never been supported previously. + + .. change:: + :tags: bug, postgresql, reflection + :tickets: 12600 + + Fixed regression caused by :ticket:`10665` where the newly modified + constraint reflection query would fail on older versions of PostgreSQL + such as version 9.6. Pull request courtesy Denis Laxalde. + + .. change:: + :tags: bug, mysql + :tickets: 12648 + + Fixed yet another regression caused by by the DEFAULT rendering changes in + 2.0.40 :ticket:`12425`, similar to :ticket:`12488`, this time where using a + CURRENT_TIMESTAMP function with a fractional seconds portion inside a + textual default value would also fail to be recognized as a + non-parenthesized server default. + + + + .. change:: + :tags: bug, mssql + :tickets: 12654 + + Reworked SQL Server column reflection to be based on the ``sys.columns`` + table rather than ``information_schema.columns`` view. By correctly using + the SQL Server ``object_id()`` function as a lead and joining to related + tables on object_id rather than names, this repairs a variety of issues in + SQL Server reflection, including: + + * Issue where reflected column comments would not correctly line up + with the columns themselves in the case that the table had been ALTERed + * Correctly targets tables with awkward names such as names with brackets, + when reflecting not just the basic table / columns but also extended + information including IDENTITY, computed columns, comments which + did not work previously + * Correctly targets IDENTITY, computed status from temporary tables + which did not work previously + + .. change:: + :tags: bug, sql + :tickets: 12681 + + Fixed issue where :func:`.select` of a free-standing scalar expression that + has a unary operator applied, such as negation, would not apply result + processors to the selected column even though the correct type remains in + place for the unary expression. + + + .. change:: + :tags: bug, sql + :tickets: 12692 + + Hardening of the compiler's actions for UPDATE statements that access + multiple tables to report more specifically when tables or aliases are + referenced in the SET clause; on cases where the backend does not support + secondary tables in the SET clause, an explicit error is raised, and on the + MySQL or similar backends that support such a SET clause, more specific + checking for not-properly-included tables is performed. Overall the change + is preventing these erroneous forms of UPDATE statements from being + compiled, whereas previously it was relied on the database to raise an + error, which was not always guaranteed to happen, or to be non-ambiguous, + due to cases where the parent table included the same column name as the + secondary table column being updated. + + + .. change:: + :tags: bug, orm + :tickets: 12692 + + Fixed bug where the ORM would pull in the wrong column into an UPDATE when + a key name inside of the :meth:`.ValuesBase.values` method could be located + from an ORM entity mentioned in the statement, but where that ORM entity + was not the actual table that the statement was inserting or updating. An + extra check for this edge case is added to avoid this problem. + + .. change:: + :tags: bug, postgresql + :tickets: 12728 + + Re-raise caught ``CancelledError`` in the terminate method of the + asyncpg dialect to avoid possible hangs of the code execution. + + + .. change:: + :tags: usecase, sql + :tickets: 12734 + + The :func:`_sql.values` construct gains a new method :meth:`_sql.Values.cte`, + which allows creation of a named, explicit-columns :class:`.CTE` against an + unnamed ``VALUES`` expression, producing a syntax that allows column-oriented + selection from a ``VALUES`` construct on modern versions of PostgreSQL, SQLite, + and MariaDB. + + .. change:: + :tags: bug, reflection, postgresql + :tickets: 12744 + + Fixes bug that would mistakenly interpret a domain or enum type + with name starting in ``interval`` as an ``INTERVAL`` type while + reflecting a table. + + .. change:: + :tags: usecase, postgresql + :tickets: 8664 + + Added ``postgresql_ops`` key to the ``dialect_options`` entry in reflected + dictionary. This maps names of columns used in the index to respective + operator class, if distinct from the default one for column's data type. + Pull request courtesy Denis Laxalde. + + .. seealso:: + + :ref:`postgresql_operator_classes` + + .. change:: + :tags: engine + + Improved validation of execution parameters passed to the + :meth:`_engine.Connection.execute` and similar methods to + provided a better error when tuples are passed in. + Previously the execution would fail with a difficult to + understand error message. + +.. changelog:: + :version: 2.0.41 + :released: May 14, 2025 + + .. change:: + :tags: usecase, postgresql + :tickets: 10665 + + Added support for ``postgresql_include`` keyword argument to + :class:`_schema.UniqueConstraint` and :class:`_schema.PrimaryKeyConstraint`. + Pull request courtesy Denis Laxalde. + + .. seealso:: + + :ref:`postgresql_constraint_options` + + .. change:: + :tags: usecase, oracle + :tickets: 12317, 12341 + + Added new datatype :class:`_oracle.VECTOR` and accompanying DDL and DQL + support to fully support this type for Oracle Database. This change + includes the base :class:`_oracle.VECTOR` type that adds new type-specific + methods ``l2_distance``, ``cosine_distance``, ``inner_product`` as well as + new parameters ``oracle_vector`` for the :class:`.Index` construct, + allowing vector indexes to be configured, and ``oracle_fetch_approximate`` + for the :meth:`.Select.fetch` clause. Pull request courtesy Suraj Shaw. + + .. seealso:: + + :ref:`oracle_vector_datatype` + + + .. change:: + :tags: bug, platform + :tickets: 12405 + + Adjusted the test suite as well as the ORM's method of scanning classes for + annotations to work under current beta releases of Python 3.14 (currently + 3.14.0b1) as part of an ongoing effort to support the production release of + this Python release. Further changes to Python's means of working with + annotations is expected in subsequent beta releases for which SQLAlchemy's + test suite will need further adjustments. + + + + .. change:: + :tags: bug, mysql + :tickets: 12488 + + Fixed regression caused by the DEFAULT rendering changes in version 2.0.40 + via :ticket:`12425` where using lowercase ``on update`` in a MySQL server + default would incorrectly apply parenthesis, leading to errors when MySQL + interpreted the rendered DDL. Pull request courtesy Alexander Ruehe. + + .. change:: + :tags: bug, sqlite + :tickets: 12566 + + Fixed and added test support for some SQLite SQL functions hardcoded into + the compiler, most notably the ``localtimestamp`` function which rendered + with incorrect internal quoting. + + .. change:: + :tags: bug, engine + :tickets: 12579 + + The error message that is emitted when a URL cannot be parsed no longer + includes the URL itself within the error message. + + + .. change:: + :tags: bug, typing + :tickets: 12588 + + Removed ``__getattr__()`` rule from ``sqlalchemy/__init__.py`` that + appeared to be trying to correct for a previous typographical error in the + imports. This rule interferes with type checking and is removed. + + + .. change:: + :tags: bug, installation + + Removed the "license classifier" from setup.cfg for SQLAlchemy 2.0, which + eliminates loud deprecation warnings when building the package. SQLAlchemy + 2.1 will use a full :pep:`639` configuration in pyproject.toml while + SQLAlchemy 2.0 remains using ``setup.cfg`` for setup. + + + +.. changelog:: + :version: 2.0.40 + :released: March 27, 2025 + + .. change:: + :tags: usecase, postgresql + :tickets: 11595 + + Added support for specifying a list of columns for ``SET NULL`` and ``SET + DEFAULT`` actions of ``ON DELETE`` clause of foreign key definition on + PostgreSQL. Pull request courtesy Denis Laxalde. + + .. seealso:: + + :ref:`postgresql_constraint_options` + + .. change:: + :tags: bug, orm + :tickets: 12329 + + Fixed regression which occurred as of 2.0.37 where the checked + :class:`.ArgumentError` that's raised when an inappropriate type or object + is used inside of a :class:`.Mapped` annotation would raise ``TypeError`` + with "boolean value of this clause is not defined" if the object resolved + into a SQL expression in a boolean context, for programs where future + annotations mode was not enabled. This case is now handled explicitly and + a new error message has also been tailored for this case. In addition, as + there are at least half a dozen distinct error scenarios for interpretation + of the :class:`.Mapped` construct, these scenarios have all been unified + under a new subclass of :class:`.ArgumentError` called + :class:`.MappedAnnotationError`, to provide some continuity between these + different scenarios, even though specific messaging remains distinct. + + .. change:: + :tags: bug, mysql + :tickets: 12332 + + Support has been re-added for the MySQL-Connector/Python DBAPI using the + ``mysql+mysqlconnector://`` URL scheme. The DBAPI now works against + modern MySQL versions as well as MariaDB versions (in the latter case it's + required to pass charset/collation explicitly). Note however that + server side cursor support is disabled due to unresolved issues with this + driver. + + .. change:: + :tags: bug, sql + :tickets: 12363 + + Fixed issue in :class:`.CTE` constructs involving multiple DDL + :class:`_sql.Insert` statements with multiple VALUES parameter sets where the + bound parameter names generated for these parameter sets would conflict, + generating a compile time error. + + + .. change:: + :tags: bug, sqlite + :tickets: 12425 + + Expanded the rules for when to apply parenthesis to a server default in DDL + to suit the general case of a default string that contains non-word + characters such as spaces or operators and is not a string literal. + + .. change:: + :tags: bug, mysql + :tickets: 12425 + + Fixed issue in MySQL server default reflection where a default that has + spaces would not be correctly reflected. Additionally, expanded the rules + for when to apply parenthesis to a server default in DDL to suit the + general case of a default string that contains non-word characters such as + spaces or operators and is not a string literal. + + + .. change:: + :tags: usecase, postgresql + :tickets: 12432 + + When building a PostgreSQL ``ARRAY`` literal using + :class:`_postgresql.array` with an empty ``clauses`` argument, the + :paramref:`_postgresql.array.type_` parameter is now significant in that it + will be used to render the resulting ``ARRAY[]`` SQL expression with a + cast, such as ``ARRAY[]::INTEGER``. Pull request courtesy Denis Laxalde. + + .. change:: + :tags: sql, usecase + :tickets: 12450 + + Implemented support for the GROUPS frame specification in window functions + by adding :paramref:`_sql.over.groups` option to :func:`_sql.over` + and :meth:`.FunctionElement.over`. Pull request courtesy Kaan Dikmen. + + .. change:: + :tags: bug, sql + :tickets: 12451 + + Fixed regression caused by :ticket:`7471` leading to a SQL compilation + issue where name disambiguation for two same-named FROM clauses with table + aliasing in use at the same time would produce invalid SQL in the FROM + clause with two "AS" clauses for the aliased table, due to double aliasing. + + .. change:: + :tags: bug, asyncio + :tickets: 12471 + + Fixed issue where :meth:`.AsyncSession.get_transaction` and + :meth:`.AsyncSession.get_nested_transaction` would fail with + ``NotImplementedError`` if the "proxy transaction" used by + :class:`.AsyncSession` were garbage collected and needed regeneration. + + .. change:: + :tags: bug, orm + :tickets: 12473 + + Fixed regression in ORM Annotated Declarative class interpretation caused + by ``typing_extension==4.13.0`` that introduced a different implementation + for ``TypeAliasType`` while SQLAlchemy assumed that it would be equivalent + to the ``typing`` version, leading to pep-695 type annotations not + resolving to SQL types as expected. + .. changelog:: :version: 2.0.39 :released: March 11, 2025 @@ -160,7 +1322,7 @@ ``connection.transaction()`` call sent to asyncpg sends ``None`` for ``isolation_level`` if not otherwise set in the SQLAlchemy dialect/wrapper, thereby allowing asyncpg to make use of the server level setting for - ``isolation_level`` in the absense of a client-level setting. Previously, + ``isolation_level`` in the absence of a client-level setting. Previously, this behavior of asyncpg was blocked by a hardcoded ``read_committed``. .. change:: @@ -274,6 +1436,9 @@ :tags: bug, orm :tickets: 11955 + .. note:: this change has been revised in version 2.0.44. Simple matches + of ``TypeAliasType`` without a type map entry are no longer deprecated. + Consistently handle ``TypeAliasType`` (defined in PEP 695) obtained with the ``type X = int`` syntax introduced in python 3.12. Now in all cases one such alias must be explicitly added to the type map for it to be usable @@ -759,10 +1924,10 @@ Fixed issue in history_meta example where the "version" column in the versioned table needs to default to the most recent version number in the history table on INSERT, to suit the use case of a table where rows are - deleted, and can then be replaced by new rows that re-use the same primary - key identity. This fix adds an additonal SELECT query per INSERT in the + deleted, and can then be replaced by new rows that reuse the same primary + key identity. This fix adds an additional SELECT query per INSERT in the main table, which may be inefficient; for cases where primary keys are not - re-used, the default function may be omitted. Patch courtesy Philipp H. + reused, the default function may be omitted. Patch courtesy Philipp H. v. Loewenfeld. .. change:: @@ -940,7 +2105,7 @@ Fixed internal typing issues to establish compatibility with mypy 1.11.0. Note that this does not include issues which have arisen with the - deprecated mypy plugin used by SQLAlchemy 1.4-style code; see the addiional + deprecated mypy plugin used by SQLAlchemy 1.4-style code; see the additional change note for this plugin indicating revised compatibility. .. changelog:: @@ -1122,8 +2287,8 @@ :tickets: 11306 Fixed issue in cursor handling which affected handling of duplicate - :class:`_sql.Column` or similar objcts in the columns clause of - :func:`_sql.select`, both in combination with arbitary :func:`_sql.text()` + :class:`_sql.Column` or similar objects in the columns clause of + :func:`_sql.select`, both in combination with arbitrary :func:`_sql.text()` clauses in the SELECT list, as well as when attempting to retrieve :meth:`_engine.Result.mappings` for the object, which would lead to an internal error. @@ -1234,7 +2399,7 @@ Fixed issue in :ref:`engine_insertmanyvalues` feature where using a primary key column with an "inline execute" default generator such as an explicit - :class:`.Sequence` with an explcit schema name, while at the same time + :class:`.Sequence` with an explicit schema name, while at the same time using the :paramref:`_engine.Connection.execution_options.schema_translate_map` feature would fail to render the sequence or the parameters properly, @@ -1257,7 +2422,7 @@ "Can't match sentinel values in result set to parameter sets". Rather than attempt to further explain and document this implementation detail of the "insertmanyvalues" feature including a public version of the new - method, the approach is intead revised to no longer need this extra + method, the approach is instead revised to no longer need this extra conversion step, and the logic that does the comparison now works on the pre-converted bound parameter value compared to the post-result-processed value, which should always be of a matching datatype. In the unusual case @@ -1273,7 +2438,7 @@ Fixed regression from version 2.0.28 caused by the fix for :ticket:`11085` where the newer method of adjusting post-cache bound parameter values would - interefere with the implementation for the :func:`_orm.subqueryload` loader + interfere with the implementation for the :func:`_orm.subqueryload` loader option, which has some more legacy patterns in use internally, when the additional loader criteria feature were used with this loader option. @@ -1853,7 +3018,7 @@ parameter for new style bulk ORM inserts, allowing ``render_nulls=True`` as an execution option. This allows for bulk ORM inserts with a mixture of ``None`` values in the parameter dictionaries to use a single batch of rows - for a given set of dicationary keys, rather than breaking up into batches + for a given set of dictionary keys, rather than breaking up into batches that omit the NULL columns from each INSERT. .. seealso:: @@ -2036,7 +3201,7 @@ However, mariadb-connector does not support invoking ``cursor.rowcount`` after the cursor itself is closed, raising an error instead. Generic test support has been added to ensure all backends support the allowing - :attr:`.Result.rowcount` to succceed (that is, returning an integer + :attr:`.Result.rowcount` to succeed (that is, returning an integer value with -1 for "not available") after the result is closed. @@ -2185,7 +3350,7 @@ :class:`.Update` and :class:`.Delete` to not interfere with the target "from" object passed to the statement, such as when passing an ORM-mapped :class:`_orm.aliased` construct that should be maintained within a phrase - like "UPDATE FROM". Cases like ORM session synchonize using "SELECT" + like "UPDATE FROM". Cases like ORM session synchronize using "SELECT" statements such as with MySQL/ MariaDB will still have issues with UPDATE/DELETE of this form so it's best to disable synchonize_session when using DML statements of this type. @@ -2539,7 +3704,7 @@ Fixed regression caused by improvements to PostgreSQL URL parsing in :ticket:`10004` where "host" query string arguments that had colons in them, to support various third party proxy servers and/or dialects, would - not parse correctly as these were evaluted as ``host:port`` combinations. + not parse correctly as these were evaluated as ``host:port`` combinations. Parsing has been updated to consider a colon as indicating a ``host:port`` value only if the hostname contains only alphanumeric characters with dots or dashes only (e.g. no slashes), followed by exactly one colon followed by @@ -2914,13 +4079,13 @@ Fixed issue where the :paramref:`.ColumnOperators.like.escape` and similar parameters did not allow an empty string as an argument that would be passed through as the "escape" character; this is a supported syntax by - PostgreSQL. Pull requset courtesy Martin Caslavsky. + PostgreSQL. Pull request courtesy Martin Caslavsky. .. change:: :tags: bug, orm :tickets: 9869 - Improved the argument chacking on the + Improved the argument checking on the :paramref:`_orm.registry.map_imperatively.local_table` parameter of the :meth:`_orm.registry.map_imperatively` method, ensuring only a :class:`.Table` or other :class:`.FromClause` is passed, and not an @@ -4388,7 +5553,7 @@ :paramref:`_orm.Mapper.primary_key` parameter to be specified within ``__mapper_args__`` when using :func:`_orm.mapped_column`. Despite this usage being directly in the 2.0 documentation, the :class:`_orm.Mapper` was - not accepting the :func:`_orm.mapped_column` construct in this context. Ths + not accepting the :func:`_orm.mapped_column` construct in this context. This feature was already working for the :paramref:`_orm.Mapper.version_id_col` and :paramref:`_orm.Mapper.polymorphic_on` parameters. @@ -5618,7 +6783,7 @@ Improved the typing for :class:`.sessionmaker` and :class:`.async_sessionmaker`, so that the default type of their return value will be :class:`.Session` or :class:`.AsyncSession`, without the need to - type this explicitly. Previously, Mypy would not automaticaly infer these + type this explicitly. Previously, Mypy would not automatically infer these return types from its generic base. As part of this change, arguments for :class:`.Session`, @@ -5769,7 +6934,7 @@ :tickets: 8718 Fixed issue in new dataclass mapping feature where a column declared on the - decalrative base / abstract base / mixin would leak into the constructor + declarative base / abstract base / mixin would leak into the constructor for an inheriting subclass under some circumstances. .. change:: @@ -5925,7 +7090,7 @@ being mentioned in other parts of the query. If other elements of the :class:`_sql.Select` also generate FROM clauses, such as the columns clause or WHERE clause, these will render after the clauses delivered by - :meth:`_sql.Select.select_from` assuming they were not explictly passed to + :meth:`_sql.Select.select_from` assuming they were not explicitly passed to :meth:`_sql.Select.select_from` also. This improvement is useful in those cases where a particular database generates a desirable query plan based on a particular ordering of FROM clauses and allows full control over the @@ -6412,7 +7577,7 @@ visible in messaging as well as typing, have been changed to more succinct names which also match the name of their constructing function (with different casing), in all cases maintaining aliases to the old names for - the forseeable future: + the foreseeable future: * :class:`_orm.RelationshipProperty` becomes an alias for the primary name :class:`_orm.Relationship`, which is constructed as always from the diff --git a/doc/build/changelog/changelog_21.rst b/doc/build/changelog/changelog_21.rst index 2ecbbaaea62..ad096cf55e4 100644 --- a/doc/build/changelog/changelog_21.rst +++ b/doc/build/changelog/changelog_21.rst @@ -9,5 +9,1484 @@ .. changelog:: - :version: 2.1.0b1 + :version: 2.1.0b3 :include_notes_from: unreleased_21 + +.. changelog:: + :version: 2.1.0b2 + :released: April 16, 2026 + + .. change:: + :tags: feature, oracle + :tickets: 10375 + + Added support for the :class:`_sqltypes.JSON` datatype when using the + Oracle database with the oracledb dialect. JSON values are serialized and + deserialized using configurable strategies that accommodate Oracle's native + JSON type available as of Oracle 21c. Pull request courtesy Abdallah + Alhadad. + + .. seealso:: + + :class:`_oracle.JSON` - Oracle-specific JSON class that includes + implementation and platform notes. + + :ref:`oracledb_json` + + .. change:: + :tags: bug, schema + :tickets: 10604 + + Amended the ``repr()`` output for :class:`.Enum` so that the + :class:`.MetaData` is not shown in the output, as this interferes with + Alembic-autogenerated forms of this type which should be inheriting the + :class:`.MetaData` of the parent table in the migration script. + + + .. change:: + :tags: bug, sql + :tickets: 11526 + + A warning is emitted when using the standalone :func:`_sql.distinct` + function in a :func:`_sql.select` columns list outside of an aggregate + function; this function is not intended as a replacement for the use of + :meth:`.Select.distinct`. Pull request courtesy bekapono. + + .. change:: + :tags: usecase, sql + :tickets: 11671 + + Added new parameter :paramref:`_sql.over.exclude` to :func:`_sql.over` and + related methods, enabling SQL standard frame exclusion clauses ``EXCLUDE + CURRENT ROW``, ``EXCLUDE GROUP``, ``EXCLUDE TIES``, ``EXCLUDE NO OTHERS`` + in window functions. Pull request courtesy of Varun Chawla. + + .. change:: + :tags: feature, mssql + :tickets: 12869 + + Added support for the ``mssql-python`` driver, Microsoft's official Python + driver for SQL Server. + + .. seealso:: + + :ref:`mssql_python` - Documentation for the mssql-python dialect + + + .. change:: + :tags: schema, usecase + :tickets: 13085 + + Most :class:`_sql.FromClause` subclasses are now generic on + :class:`_schema.TypedColumns` subclasses, that can be used to type their + :attr:`_sql.FromClause.c` collection. + This applied to :class:`_schema.Table`, :class:`_sql.Join`, + :class:`_sql.Subquery`, :class:`_sql.CTE` and more. + + .. seealso:: + + :ref:`change_13085` + + .. change:: + :tags: bug, typing + :tickets: 13091 + + Fixed issue in new :pep:`646` support for result sets where an issue in the + mypy type checker prevented "scalar" methods including + :meth:`.Connection.scalar`, :meth:`.Result.scalar`, + :meth:`_orm.Session.scalar`, as well as async versions of these methods + from applying the correct type to the scalar result value, when the columns + in the originating :func:`_sql.select` were typed as ``Any``. Pull request + courtesy Yurii Karabas. + + + .. change:: + :tags: bug, typing + :tickets: 13131 + + Improved typing of :class:`_sqltypes.JSON` as well as dialect specific + variants like :class:`_postgresql.JSON` to include generic capabilities, so + that the types may be parameterized to indicate any specific type of + contents expected, e.g. ``JSONB[list[str]]()``. + + + .. change:: + :tags: bug, sql + :tickets: 13140 + + Improved the ability for :class:`.TypeDecorator` to produce a correct + ``repr()`` for "schema" types such as :class:`.Enum` and :class:`.Boolean`. + This is mostly to support the Alembic autogenerate use case so that custom + types render with relevant arguments present. Improved the architecture + used by :class:`.TypeEngine` to produce ``repr()`` strings to be more + modular for compound types like :class:`.TypeDecorator`. + + .. change:: + :tags: usecase, orm + :tickets: 13198 + + The ``metadata``, ``type_annotation_map``, or ``registry`` can now be + set up in a declarative base also via a mixin class, not only by + directly setting them on the subclass like before. + The declarative class setup now uses ``getattr()`` to look for these + attributes, instead of relying only on the class ``__dict__``. + + .. change:: + :tags: usecase, sql + + The :class:`.ColumnCollection` class hierarchy has been refactored to allow + column names such as ``add``, ``remove``, ``update``, ``extend``, and + ``clear`` to be used without conflicts. :class:`.ColumnCollection` is now + an abstract base class, with mutation operations moved to + :class:`.WriteableColumnCollection` and :class:`.DedupeColumnCollection` + subclasses. The :class:`.ReadOnlyColumnCollection` exposed as attributes + such as :attr:`.Table.c` no longer includes mutation methods that raised + :class:`.NotImplementedError`, allowing these common column names to be + accessed naturally, e.g. ``table.c.add``, ``table.c.remove``, + ``table.c.update``, etc. + +.. changelog:: + :version: 2.1.0b1 + :released: January 21, 2026 + + .. change:: + :tags: feature, orm + :tickets: 10050 + + The :paramref:`_orm.relationship.back_populates` argument to + :func:`_orm.relationship` may now be passed as a Python callable, which + resolves to either the direct linked ORM attribute, or a string value as + before. ORM attributes are also accepted directly by + :paramref:`_orm.relationship.back_populates`. This change allows type + checkers and IDEs to confirm the argument for + :paramref:`_orm.relationship.back_populates` is valid. Thanks to Priyanshu + Parikh for the help on suggesting and helping to implement this feature. + + .. seealso:: + + :ref:`change_10050` + + + .. change:: + :tags: change, platform + :tickets: 10197 + + The ``greenlet`` dependency used for asyncio support no longer installs + by default. This dependency does not publish wheel files for every architecture + and is not needed for applications that aren't using asyncio features. + Use the ``sqlalchemy[asyncio]`` install target to include this dependency. + + .. seealso:: + + :ref:`change_10197` + + + + .. change:: + :tags: change, sql + :tickets: 10236 + + The ``.c`` and ``.columns`` attributes on the :class:`.Select` and + :class:`.TextualSelect` constructs, which are not instances of + :class:`.FromClause`, have been removed completely, in addition to the + ``.select()`` method as well as other codepaths which would implicitly + generate a subquery from a :class:`.Select` without the need to explicitly + call the :meth:`.Select.subquery` method. + + In the case of ``.c`` and ``.columns``, these attributes were never useful + in practice and have caused a great deal of confusion, hence were + deprecated back in version 1.4, and have emitted warnings since that + version. Accessing the columns that are specific to a :class:`.Select` + construct is done via the :attr:`.Select.selected_columns` attribute, which + was added in version 1.4 to suit the use case that users often expected + ``.c`` to accomplish. In the larger sense, implicit production of + subqueries works against SQLAlchemy's modern practice of making SQL + structure as explicit as possible. + + Note that this is **not related** to the usual :attr:`.FromClause.c` and + :attr:`.FromClause.columns` attributes, common to objects such as + :class:`.Table` and :class:`.Subquery`, which are unaffected by this + change. + + .. seealso:: + + :ref:`change_4617` - original notes from SQLAlchemy 1.4 + + + .. change:: + :tags: schema + :tickets: 10247 + + Deprecate Oracle only parameters :paramref:`_schema.Sequence.order`, + :paramref:`_schema.Identity.order` and :paramref:`_schema.Identity.on_null`. + They should be configured using the dialect kwargs ``oracle_order`` and + ``oracle_on_null``. + + .. change:: + :tags: change, asyncio + :tickets: 10296 + + Added an initialize step to the import of + ``sqlalchemy.ext.asyncio`` so that ``greenlet`` will + be imported only when the asyncio extension is first imported. + Alternatively, the ``greenlet`` library is still imported lazily on + first use to support use case that don't make direct use of the + SQLAlchemy asyncio extension. + + .. change:: + :tags: bug, sql + :tickets: 10300 + + The :class:`.Double` type is now used when a Python float value is detected + as a literal value to be sent as a bound parameter, rather than the + :class:`.Float` type. :class:`.Double` has the same implementation as + :class:`.Float`, but when rendered in a CAST, produces ``DOUBLE`` or + ``DOUBLE PRECISION`` rather than ``FLOAT``. The former better matches + Python's ``float`` datatype which uses 8-byte double-precision storage. + Third party dialects which don't support the :class:`.Double` type directly + may need adjustment so that they render an appropriate keyword (e.g. + ``FLOAT``) when the :class:`.Double` datatype is encountered. + + .. seealso:: + + :ref:`change_10300` + + .. change:: + :tags: usecase, mariadb + :tickets: 10339 + + Modified the MariaDB dialect so that when using the :class:`_sqltypes.Uuid` + datatype with MariaDB >= 10.7, leaving the + :paramref:`_sqltypes.Uuid.native_uuid` parameter at its default of True, + the native ``UUID`` datatype will be rendered in DDL and used for database + communication, rather than ``CHAR(32)`` (the non-native UUID type) as was + the case previously. This is a behavioral change since 2.0, where the + generic :class:`_sqltypes.Uuid` datatype delivered ``CHAR(32)`` for all + MySQL and MariaDB variants. Support for all major DBAPIs is implemented + including support for less common "insertmanyvalues" scenarios where UUID + values are generated in different ways for primary keys. Thanks much to + Volodymyr Kochetkov for delivering the PR. + + + .. change:: + :tags: change, asyncio + :tickets: 10415 + + Adapted all asyncio dialects, including aiosqlite, aiomysql, asyncmy, + psycopg, asyncpg to use the generic asyncio connection adapter first added + in :ticket:`6521` for the aioodbc DBAPI, allowing these dialects to take + advantage of a common framework. + + .. change:: + :tags: change, orm + :tickets: 10497 + + A sweep through class and function names in the ORM renames many classes + and functions that have no intent of public visibility to be underscored. + This is to reduce ambiguity as to which APIs are intended to be targeted by + third party applications and extensions. Third parties are encouraged to + propose new public APIs in Discussions to the extent they are needed to + replace those that have been clarified as private. + + .. change:: + :tags: change, orm + :tickets: 10500 + + The ``first_init`` ORM event has been removed. This event was + non-functional throughout the 1.4 and 2.0 series and could not be invoked + without raising an internal error, so it is not expected that there is any + real-world use of this event hook. + + .. change:: + :tags: feature, postgresql + :tickets: 10556 + + Adds a new ``str`` subclass :class:`_postgresql.BitString` representing + PostgreSQL bitstrings in python, that includes + functionality for converting to and from ``int`` and ``bytes``, in + addition to implementing utility methods and operators for dealing with bits. + + This new class is returned automatically by the :class:`postgresql.BIT` type. + + .. seealso:: + + :ref:`change_10556` + + .. change:: + :tags: bug, orm + :tickets: 10564 + + The :paramref:`_orm.relationship.secondary` parameter no longer uses Python + ``eval()`` to evaluate the given string. This parameter when passed a + string should resolve to a table name that's present in the local + :class:`.MetaData` collection only, and never needs to be any kind of + Python expression otherwise. To use a real deferred callable based on a + name that may not be locally present yet, use a lambda instead. + + .. change:: + :tags: usecase, postgresql + :tickets: 10604 + + Added new parameter :paramref:`.Enum.create_type` to the Core + :class:`.Enum` class. This parameter is automatically passed to the + corresponding :class:`_postgresql.ENUM` native type during DDL operations, + allowing control over whether the PostgreSQL ENUM type is implicitly + created or dropped within DDL operations that are otherwise targeting + tables only. This provides control over the + :paramref:`_postgresql.ENUM.create_type` behavior without requiring + explicit creation of a :class:`_postgresql.ENUM` object. + + .. change:: + :tags: typing, feature + :tickets: 10635 + + The :class:`.Row` object now no longer makes use of an intermediary + ``Tuple`` in order to represent its individual element types; instead, + the individual element types are present directly, via new :pep:`646` + integration, now available in more recent versions of Mypy. Mypy + 1.7 or greater is now required for statements, results and rows + to be correctly typed. Pull request courtesy Yurii Karabas. + + .. seealso:: + + :ref:`change_10635` + + .. change:: + :tags: typing + :tickets: 10646 + + The default implementation of :attr:`_types.TypeEngine.python_type` now + returns ``object`` instead of ``NotImplementedError``, since that's the + base for all types in Python3. + The ``python_type`` of :class:`_types.JSON` no longer returns ``dict``, + but instead fallbacks to the generic implementation. + + .. change:: + :tags: change, orm + :tickets: 10721 + + Removed legacy signatures dating back to 0.9 release from the + :meth:`_orm.SessionEvents.after_bulk_update` and + :meth:`_orm.SessionEvents.after_bulk_delete`. + + .. change:: + :tags: bug, sql + :tickets: 10788 + + Fixed issue in name normalization (e.g. "uppercase" backends like Oracle) + where using a :class:`.TextualSelect` would not properly maintain as + uppercase column names that were quoted as uppercase, even though + the :class:`.TextualSelect` includes a :class:`.Column` that explicitly + holds this uppercase name. + + .. change:: + :tags: usecase, engine + :tickets: 10789 + + Added new execution option + :paramref:`_engine.Connection.execution_options.driver_column_names`. This + option disables the "name normalize" step that takes place against the + DBAPI ``cursor.description`` for uppercase-default backends like Oracle, + and will cause the keys of a result set (e.g. named tuple names, dictionary + keys in :attr:`.Row._mapping`, etc.) to be exactly what was delivered in + cursor.description. This is mostly useful for plain textual statements + using :func:`_sql.text` or :meth:`_engine.Connection.exec_driver_sql`. + + .. change:: + :tags: bug, engine + :tickets: 10802 + + Fixed issue in "insertmanyvalues" feature where an INSERT..RETURNING + that also made use of a sentinel column to track results would fail to + filter out the additional column when :meth:`.Result.unique` were used + to uniquify the result set. + + .. change:: + :tags: usecase, orm + :tickets: 10816 + + The :paramref:`_orm.Session.flush.objects` parameter is now + deprecated. + + .. change:: + :tags: change, postgresql + :tickets: 10821 + + The :meth:`_types.ARRAY.Comparator.any` and + :meth:`_types.ARRAY.Comparator.all` methods for the :class:`_types.ARRAY` + type are now deprecated for removal; these two methods along with + :func:`_postgresql.Any` and :func:`_postgresql.All` have been legacy for + some time as they are superseded by the :func:`_sql.any_` and + :func:`_sql.all_` functions, which feature more intuitive use. + + + .. change:: + :tags: postgresql, feature + :tickets: 10909 + + Support for storage parameters in ``CREATE TABLE`` using the ``WITH`` + clause has been added. The ``postgresql_with`` dialect option of + :class:`_schema.Table` accepts a mapping of key/value options. + + .. seealso:: + + :ref:`postgresql_table_options_with` - in the PostgreSQL dialect + documentation + + .. change:: + :tags: postgresql, usecase + :tickets: 10909 + + The PostgreSQL dialect now support reflection of table options, including + the storage parameters, table access method and table spaces. These options + are automatically reflected when autoloading a table, and are also + available via the :meth:`_engine.Inspector.get_table_options` and + :meth:`_engine.Inspector.get_multi_table_optionsmethod` methods. + + .. change:: + :tags: orm + :tickets: 11045 + + The :func:`_orm.noload` relationship loader option and related + ``lazy='noload'`` setting is deprecated and will be removed in a future + release. This option was originally intended for custom loader patterns + that are no longer applicable in modern SQLAlchemy. + + .. change:: + :tags: bug, sqlite + :tickets: 11074 + + Improved the behavior of JSON accessors :meth:`.JSON.Comparator.as_string`, + :meth:`.JSON.Comparator.as_boolean`, :meth:`.JSON.Comparator.as_float`, + :meth:`.JSON.Comparator.as_integer` to use CAST in a similar way that + the PostgreSQL, MySQL and SQL Server dialects do to help enforce the + expected Python type is returned. + + + + .. change:: + :tags: bug, mssql + :tickets: 11074 + + The :meth:`.JSON.Comparator.as_boolean` method when used on a JSON value on + SQL Server will now force a cast to occur for values that are not simple + `true`/`false` JSON literals, forcing SQL Server to attempt to interpret + the given value as a 1/0 BIT, or raise an error if not possible. Previously + the expression would return NULL. + + + + .. change:: + :tags: orm + :tickets: 11163 + + Ignore :paramref:`_orm.Session.join_transaction_mode` in all cases when + the bind provided to the :class:`_orm.Session` is an + :class:`_engine.Engine`. + Previously if an event that executed before the session logic, + like :meth:`_engine.ConnectionEvents.engine_connect`, + left the connection with an active transaction, the + :paramref:`_orm.Session.join_transaction_mode` behavior took + place, leading to a surprising behavior. + + .. change:: + :tags: bug, orm + :tickets: 11226 + + Fixed issue where joined eager loading would fail to use the "nested" form + of the query when GROUP BY or DISTINCT were present if the eager joins + being added were many-to-ones, leading to additional columns in the columns + clause which would then cause errors. The check for "nested" is tuned to + be enabled for these queries even for many-to-one joined eager loaders, and + the "only do nested if it's one to many" aspect is now localized to when + the query only has LIMIT or OFFSET added. + + .. change:: + :tags: bug, engine + :tickets: 11234 + + Adjusted URL parsing and stringification to apply url quoting to the + "database" portion of the URL. This allows a URL where the "database" + portion includes special characters such as question marks to be + accommodated. + + .. seealso:: + + :ref:`change_11234` + + .. change:: + :tags: bug, mssql + :tickets: 11250 + + Fix mssql+pyodbc issue where valid plus signs in an already-unquoted + ``odbc_connect=`` (raw DBAPI) connection string are replaced with spaces. + + The pyodbc connector would unconditionally pass the odbc_connect value + to unquote_plus(), even if it was not required. So, if the (unquoted) + odbc_connect value contained ``PWD=pass+word`` that would get changed to + ``PWD=pass word``, and the login would fail. One workaround was to quote + just the plus sign — ``PWD=pass%2Bword`` — which would then get unquoted + to ``PWD=pass+word``. + + .. change:: + :tags: bug, orm + :tickets: 11349 + + Revised the set "binary" operators for the association proxy ``set()`` + interface to correctly raise ``TypeError`` for invalid use of the ``|``, + ``&``, ``^``, and ``-`` operators, as well as the in-place mutation + versions of these methods, to match the behavior of standard Python + ``set()`` as well as SQLAlchemy ORM's "instrumented" set implementation. + + + + .. change:: + :tags: bug, sql + :tickets: 11515 + + Enhanced the caching structure of the :paramref:`_expression.over.rows` + and :paramref:`_expression.over.range` so that different numerical + values for the rows / + range fields are cached on the same cache key, to the extent that the + underlying SQL does not actually change (i.e. "unbounded", "current row", + negative/positive status will still change the cache key). This prevents + the use of many different numerical range/rows value for a query that is + otherwise identical from filling up the SQL cache. + + Note that the semi-private compiler method ``_format_frame_clause()`` + is removed by this fix, replaced with a new method + ``visit_frame_clause()``. Third party dialects which may have referred + to this method will need to change the name and revise the approach to + rendering the correct SQL for that dialect. + + + .. change:: + :tags: feature, oracle + :tickets: 11633 + + Added support for native BOOLEAN support in Oracle Database 23c and above. + The Oracle dialect now renders ``BOOLEAN`` automatically when + :class:`.Boolean` is used in DDL, and also now supports direct use of the + :class:`.BOOLEAN` datatype, when 23c and above is in use. For Oracle + versions prior to 23c, boolean values continue to be emulated using + SMALLINT as before. Special case handling is also present to ensure a + SMALLINT that's interpreted with the :class:`.Boolean` datatype on Oracle + Database 23c and above continues to return bool values. Pull request + courtesy Yeongbae Jeon. + + .. seealso:: + + :ref:`oracle_boolean_support` + + .. change:: + :tags: orm, usecase + :tickets: 11776 + + Added the utility method :meth:`_orm.Session.merge_all` and + :meth:`_orm.Session.delete_all` that operate on a collection + of instances. + + .. change:: + :tags: bug, schema + :tickets: 11811 + + The :class:`.Float` and :class:`.Numeric` types are no longer automatically + considered as auto-incrementing columns when the + :paramref:`_schema.Column.autoincrement` parameter is left at its default + of ``"auto"`` on a :class:`_schema.Column` that is part of the primary key. + When the parameter is set to ``True``, a :class:`.Numeric` type will be + accepted as an auto-incrementing datatype for primary key columns, but only + if its scale is explicitly given as zero; otherwise, an error is raised. + This is a change from 2.0 where all numeric types including floats were + automatically considered as "autoincrement" for primary key columns. + + .. change:: + :tags: bug, asyncio + :tickets: 11956 + + Refactored all asyncio dialects so that exceptions which occur on failed + connection attempts are appropriately wrapped with SQLAlchemy exception + objects, allowing for consistent error handling. + + .. change:: + :tags: bug, orm + :tickets: 12168 + + A significant behavioral change has been made to the behavior of the + :paramref:`_orm.mapped_column.default` and + :paramref:`_orm.relationship.default` parameters, as well as the + :paramref:`_orm.relationship.default_factory` parameter with + collection-based relationships, when used with SQLAlchemy's + :ref:`orm_declarative_native_dataclasses` feature introduced in 2.0, where + the given value (assumed to be an immutable scalar value for + :paramref:`_orm.mapped_column.default` and a simple collection class for + :paramref:`_orm.relationship.default_factory`) is no longer passed to the + ``@dataclass`` API as a real default, instead a token that leaves the value + un-set in the object's ``__dict__`` is used, in conjunction with a + descriptor-level default. This prevents an un-set default value from + overriding a default that was actually set elsewhere, such as in + relationship / foreign key assignment patterns as well as in + :meth:`_orm.Session.merge` scenarios. See the full writeup in the + :ref:`migration_21_toplevel` document which includes guidance on how to + re-enable the 2.0 version of the behavior if needed. + + .. seealso:: + + :ref:`change_12168` + + .. change:: + :tags: feature, sql + :tickets: 12195 + + Added the ability to create custom SQL constructs that can define new + clauses within SELECT, INSERT, UPDATE, and DELETE statements without + needing to modify the construction or compilation code of of + :class:`.Select`, :class:`_sql.Insert`, :class:`.Update`, or :class:`.Delete` + directly. Support for testing these constructs, including caching support, + is present along with an example test suite. The use case for these + constructs is expected to be third party dialects for analytical SQL + (so-called NewSQL) or other novel styles of database that introduce new + clauses to these statements. A new example suite is included which + illustrates the ``QUALIFY`` SQL construct used by several NewSQL databases + which includes a cacheable implementation as well as a test suite. + + .. seealso:: + + :ref:`examples_syntax_extensions` + + :ref:`change_new_syntax_ext` + + + .. change:: + :tags: sql + :tickets: 12218 + + Removed the automatic coercion of executable objects, such as + :class:`_orm.Query`, when passed into :meth:`_orm.Session.execute`. + This usage raised a deprecation warning since the 1.4 series. + + .. change:: + :tags: reflection, mysql, mariadb + :tickets: 12240 + + Updated the reflection logic for indexes in the MariaDB and MySQL + dialect to avoid setting the undocumented ``type`` key in the + :class:`_engine.ReflectedIndex` dicts returned by + :class:`_engine.Inspector.get_indexes` method. + + .. change:: + :tags: typing, orm + :tickets: 12293 + + Removed the deprecated mypy plugin. + The plugin was non-functional with newer version of mypy and it's no + longer needed with modern SQLAlchemy declarative style. + + .. change:: + :tags: feature, postgresql + :tickets: 12342 + + Added syntax extension :func:`_postgresql.distinct_on` to build ``DISTINCT + ON`` clauses. The old api, that passed columns to + :meth:`_sql.Select.distinct`, is now deprecated. + + .. change:: + :tags: typing, orm + :tickets: 12346 + + Deprecated the ``declarative_mixin`` decorator since it was used only + by the now removed mypy plugin. + + .. change:: + :tags: bug, orm + :tickets: 12395 + + The behavior of :func:`_orm.with_polymorphic` when used with a single + inheritance mapping has been changed such that its behavior should match as + closely as possible to that of an equivalent joined inheritance mapping. + Specifically this means that the base class specified in the + :func:`_orm.with_polymorphic` construct will be the basemost class that is + loaded, as well as all descendant classes of that basemost class. + The change includes that the descendant classes named will no longer be + exclusively indicated in "WHERE polymorphic_col IN" criteria; instead, the + whole hierarchy starting with the given basemost class will be loaded. If + the query indicates that rows should only be instances of a specific + subclass within the polymorphic hierarchy, an error is raised if an + incompatible superclass is loaded in the result since it cannot be made to + match the requested class; this behavior is the same as what joined + inheritance has done for many years. The change also allows a single result + set to include column-level results from multiple sibling classes at once + which was not previously possible with single table inheritance. + + .. change:: + :tags: orm, changed + :tickets: 12437 + + The "non primary" mapper feature, long deprecated in SQLAlchemy since + version 1.3, has been removed. The sole use case for "non primary" + mappers was that of using :func:`_orm.relationship` to link to a mapped + class against an alternative selectable; this use case is now suited by the + :ref:`relationship_aliased_class` feature. + + + + .. change:: + :tags: misc, changed + :tickets: 12441 + + Removed multiple api that were deprecated in the 1.3 series and earlier. + The list of removed features includes: + + * The ``force`` parameter of ``IdentifierPreparer.quote`` and + ``IdentifierPreparer.quote_schema``; + * The ``threaded`` parameter of the cx-Oracle dialect; + * The ``_json_serializer`` and ``_json_deserializer`` parameters of the + SQLite dialect; + * The ``collection.converter`` decorator; + * The ``Mapper.mapped_table`` property; + * The ``Session.close_all`` method; + * Support for multiple arguments in :func:`_orm.defer` and + :func:`_orm.undefer`. + + .. change:: + :tags: core, feature, sql + :tickets: 12479 + + The Core operator system now includes the ``matmul`` operator, i.e. the + ``@`` operator in Python as an optional operator. + In addition to the ``__matmul__`` and ``__rmatmul__`` operator support + this change also adds the missing ``__rrshift__`` and ``__rlshift__``. + Pull request courtesy Aramís Segovia. + + .. change:: + :tags: feature, sql + :tickets: 12496 + + Added new Core feature :func:`_sql.from_dml_column` that may be used in + expressions inside of :meth:`.UpdateBase.values` for INSERT or UPDATE; this + construct will copy whatever SQL expression is used for the given target + column in the statement to be used with additional columns. The construct + is mostly intended to be a helper with ORM :class:`.hybrid_property` within + DML hooks. + + .. change:: + :tags: feature, orm + :tickets: 12496 + + Added new hybrid method :meth:`.hybrid_property.bulk_dml` which + works in a similar way as :meth:`.hybrid_property.update_expression` for + bulk ORM operations. A user-defined class method can now populate a bulk + insert mapping dictionary using the desired hybrid mechanics. New + documentation is added showing how both of these methods can be used + including in combination with the new :func:`_sql.from_dml_column` + construct. + + .. seealso:: + + :ref:`change_12496` + + .. change:: + :tags: feature, sql + :tickets: 12548 + + Added support for Python 3.14+ template strings (t-strings) via the new + :func:`_sql.tstring` construct. This feature makes use of Python 3.14 + template strings as defined in :pep:`750`, allowing for ergonomic SQL + statement construction by automatically interpolating Python values and + SQLAlchemy expressions within template strings. + + .. seealso:: + + :ref:`change_12548` - in :ref:`migration_21_toplevel` + + .. change:: + :tags: feature, orm + :tickets: 12570 + + Added new parameter :paramref:`_orm.composite.return_none_on` to + :func:`_orm.composite`, which allows control over if and when this + composite attribute should resolve to ``None`` when queried or retrieved + from the object directly. By default, a composite object is always present + on the attribute, including for a pending object which is a behavioral + change since 2.0. When :paramref:`_orm.composite.return_none_on` is + specified, a callable is passed that returns True or False to indicate if + the given arguments indicate the composite should be returned as None. This + parameter may also be set automatically when ORM Annotated Declarative is + used; if the annotation is given as ``Mapped[SomeClass|None]``, a + :paramref:`_orm.composite.return_none_on` rule is applied that will return + ``None`` if all contained columns are themselves ``None``. + + .. seealso:: + + :ref:`change_12570` + + .. change:: + :tags: bug, sql + :tickets: 12596 + + Updated the :func:`_sql.over` clause to allow non integer values in + :paramref:`_sql.over.range_` clause. Previously, only integer values + were allowed and any other values would lead to a failure. + To specify a non-integer value, use the new :class:`_sql.FrameClause` + construct along with the new :class:`_sql.FrameClauseType` enum to specify + the frame boundaries. For example:: + + from sqlalchemy import FrameClause, FrameClauseType + + select( + func.sum(table.c.value).over( + range_=FrameClause( + 3.14, + 2.71, + FrameClauseType.PRECEDING, + FrameClauseType.FOLLOWING, + ) + ) + ) + + .. seealso:: + + :ref:`change_12596` - in the :ref:`migration guide + ` + + .. change:: + :tags: usecase, orm + :tickets: 12631 + + Added support for using :func:`_orm.with_expression` to populate a + :func:`_orm.query_expression` attribute that is also configured as the + ``polymorphic_on`` discriminator column. The ORM now detects when a query + expression column is serving as the polymorphic discriminator and updates + it to use the column provided via :func:`_orm.with_expression`, enabling + polymorphic loading to work correctly in this scenario. This allows for + patterns such as where the discriminator value is computed from a related + table. + + .. change:: + :tags: feature, orm + :tickets: 12659 + + Added support for per-session execution options that are merged into all + queries executed within that session. The :class:`_orm.Session`, + :class:`_orm.sessionmaker`, :class:`_orm.scoped_session`, + :class:`_asyncio.AsyncSession`, and + :class:`_asyncio.async_sessionmaker` constructors now accept an + :paramref:`_orm.Session.execution_options` parameter that will be applied + to all explicit query executions (e.g. using :meth:`_orm.Session.execute`, + :meth:`_orm.Session.get`, :meth:`_orm.Session.scalars`) for that session + instance. + + .. change:: + :tags: change, postgresql + :tickets: 10594, 12690 + + Named types such as :class:`_postgresql.ENUM` and + :class:`_postgresql.DOMAIN` (as well as the dialect-agnostic + :class:`_types.Enum` version) are now more strongly associated with the + :class:`_schema.MetaData` at the top of the table hierarchy and are + de-associated with any particular :class:`_schema.Table` they may be a part + of. This better represents how PostgreSQL named types exist independently + of any particular table, and that they may be used across many tables + simultaneously. The change impacts the behavior of the "default schema" + for a named type, as well as the CREATE/DROP behavior in relationship to + the :class:`.MetaData` and :class:`.Table` construct. The change also + includes a new :class:`.CheckFirst` enumeration which allows fine grained + control over "check" queries during DDL operations, as well as that the + :paramref:`_types.SchemaType.inherit_schema` parameter is deprecated and + will emit a deprecation warning when used. See the migration notes for + full details. + + .. seealso:: + + :ref:`change_10594_postgresql` - Complete details on PostgreSQL named type changes + + .. change:: + :tags: bug, sql + :tickets: 12736 + + Added a new concept of "operator classes" to the SQL operators supported by + SQLAlchemy, represented within the enum :class:`.OperatorClass`. The + purpose of this structure is to provide an extra layer of validation when a + particular kind of SQL operation is used with a particular datatype, to + catch early the use of an operator that does not have any relevance to the + datatype in use; a simple example is an integer or numeric column used with + a "string match" operator. + + .. seealso:: + + :ref:`change_12736` + + + + .. change:: + :tags: bug, postgresql + :tickets: 12761 + + A :class:`.CompileError` is raised if attempting to create a PostgreSQL + :class:`_postgresql.ENUM` or :class:`_postgresql.DOMAIN` datatype using a + name that matches a known pg_catalog datatype name, and a default schema is + not specified. These types must be explicit within a schema in order to + be differentiated from the built-in pg_catalog type. The "public" or + otherwise default schema is not chosen by default here since the type can + only be reflected back using the explicit schema name as well (it is + otherwise not visible due to the pg_catalog name). Pull request courtesy + Kapil Dagur. + + + + .. change:: + :tags: bug, orm + :tickets: 12769 + + Improved the behavior of standalone "operators" like :func:`_sql.desc`, + :func:`_sql.asc`, :func:`_sql.all_`, etc. so that they consult the given + expression object for an overriding method for that operator, even if the + object is not itself a ``ClauseElement``, such as if it's an ORM attribute. + This allows custom comparators for things like :func:`_orm.composite` to + provide custom implementations of methods like ``desc()``, ``asc()``, etc. + + + .. change:: + :tags: usecase, orm + :tickets: 12769 + + Added default implementations of :meth:`.ColumnOperators.desc`, + :meth:`.ColumnOperators.asc`, :meth:`.ColumnOperators.nulls_first`, + :meth:`.ColumnOperators.nulls_last` to :func:`_orm.composite` attributes, + by default applying the modifier to all contained columns. Can be + overridden using a custom comparator. + + .. change:: + :tags: usecase, orm + :tickets: 12838 + + The :func:`_orm.aliased` object now emits warnings when an attribute is + accessed on an aliased class that cannot be located in the target + selectable, for those cases where the :func:`_orm.aliased` is against a + different FROM clause than the regular mapped table (such as a subquery). + This helps users identify cases where column names don't match between the + aliased class and the underlying selectable. When + :paramref:`_orm.aliased.adapt_on_names` is ``True``, the warning suggests + checking the column name; when ``False``, it suggests using the + ``adapt_on_names`` parameter for name-based matching. + + .. change:: + :tags: bug, orm + :tickets: 12843 + + ORM entities can now be involved within the SQL expressions used within + :paramref:`_orm.relationship.primaryjoin` and + :paramref:`_orm.relationship.secondaryjoin` parameters without the ORM + entity information being implicitly sanitized, allowing ORM-specific + features such as single-inheritance criteria in subqueries to continue + working even when used in this context. This is made possible by overall + ORM simplifications that occurred as of the 2.0 series. The changes here + also provide a performance boost (up to 20%) for certain query compilation + scenarios. + + .. change:: + :tags: usecase, sql + :tickets: 12853 + + Added new generalized aggregate function ordering to functions via the + :func:`_functions.FunctionElement.aggregate_order_by` method, which + receives an expression and generates the appropriate embedded "ORDER BY" or + "WITHIN GROUP (ORDER BY)" phrase depending on backend database. This new + function supersedes the use of the PostgreSQL + :func:`_postgresql.aggregate_order_by` function, which remains present for + backward compatibility. To complement the new parameter, the + :paramref:`_functions.aggregate_strings.order_by` which adds ORDER BY + capability to the :class:`_functions.aggregate_strings` dialect-agnostic + function which works for all included backends. Thanks much to Reuven + Starodubski with help on this patch. + + + + .. change:: + :tags: usecase, orm + :tickets: 12854 + + Improvements to the use case of using :ref:`Declarative Dataclass Mapping + ` with intermediary classes that are + unmapped. As was the existing behavior, classes can subclass + :class:`_orm.MappedAsDataclass` alone without a declarative base to act as + mixins, or along with a declarative base as well as ``__abstract__ = True`` + to define an abstract base. However, the improved behavior scans ORM + attributes like :func:`_orm.mapped_column` in this case to create correct + ``dataclasses.field()`` constructs based on their arguments, allowing for + more natural ordering of fields without dataclass errors being thrown. + Additionally, added a new :func:`_orm.unmapped_dataclass` decorator + function, which may be used to create unmapped mixins in a mapped hierarchy + that is using the :func:`_orm.mapped_dataclass` decorator to create mapped + dataclasses. + + .. seealso:: + + :ref:`orm_declarative_dc_mixins` + + .. change:: + :tags: feature, postgresql + :tickets: 12866 + + Support for ``VIRTUAL`` computed columns on PostgreSQL 18 and later has + been added. The default behavior when :paramref:`.Computed.persisted` is + not specified has been changed to align with PostgreSQL 18's default of + ``VIRTUAL``. When :paramref:`.Computed.persisted` is not specified, no + keyword is rendered on PostgreSQL 18 and later; on older versions a + warning is emitted and ``STORED`` is used as the default. To explicitly + request ``STORED`` behavior on all PostgreSQL versions, specify + ``persisted=True``. + + .. change:: + :tags: feature, platform + :tickets: 12881 + + Free-threaded Python versions are now supported in wheels released on Pypi. + This integrates with overall free-threaded support added as part of + :ticket:`12881` for the 2.0 and 2.1 series, which includes new test suites + as well as a few improvements to race conditions observed under + freethreading. + + + .. change:: + :tags: bug, orm + :tickets: 12921 + + The :meth:`_events.SessionEvents.do_orm_execute` event now allows direct + mutation or replacement of the :attr:`.ORMExecuteState.parameters` + dictionary or list, which will take effect when the the statement is + executed. Previously, changes to this collection were not accommodated by + the event hook. Pull request courtesy Shamil. + + + .. change:: + :tags: bug, sql + :tickets: 12931 + + Fixed an issue in :meth:`_sql.Select.join_from` where the join condition + between the left and right tables specified in the method call could be + incorrectly determined based on an intermediate table already present in + the FROM clause, rather than matching the foreign keys between the + immediate left and right arguments. The join condition is now determined by + matching primary keys between the two tables explicitly passed to + :meth:`_sql.Select.join_from`, ensuring consistent and predictable join + behavior regardless of the order of join operations or other tables present + in the query. The fix is applied to both the Core and ORM implementations + of :meth:`_sql.Select.join_from`. + + .. change:: + :tags: usecase, sql + :tickets: 12932 + + Changed the query style for ORM queries emitted by :meth:`.Session.get` as + well as many-to-one lazy load queries to use the default labeling style, + :attr:`_sql.SelectLabelStyle.LABEL_STYLE_DISAMBIGUATE_ONLY`, which normally + does not apply labels to columns in a SELECT statement. Previously, the + older style :attr:`_sql.SelectLabelStyle.LABEL_STYLE_TABLENAME_PLUS_COL` + that labels columns as `_` was used for + :meth:`.Session.get` to maintain compatibility with :class:`_orm.Query`. + The change allows the string representation of ORM queries to be less + verbose in all cases outside of legacy :class:`_orm.Query` use. Pull + request courtesy Inada Naoki. + + .. change:: + :tags: usecase, postgresql + :tickets: 12948 + + Added support for PostgreSQL 14+ HSTORE subscripting syntax. When connected + to PostgreSQL 14 or later, HSTORE columns now automatically use the native + subscript notation ``hstore_col['key']`` instead of the arrow operator + ``hstore_col -> 'key'`` for both read and write operations. This provides + better compatibility with PostgreSQL's native HSTORE subscripting feature + while maintaining backward compatibility with older PostgreSQL versions. + + .. warning:: Indexes in existing PostgreSQL databases which were indexed + on an HSTORE subscript expression would need to be updated in order to + match the new SQL syntax. + + .. seealso:: + + :ref:`change_12948` - in the :ref:`migration guide + ` + + + + .. change:: + :tags: orm, usecase + :tickets: 12960 + + Added :class:`_orm.DictBundle` as a subclass of :class:`_orm.Bundle` + that returns ``dict`` objects. + + .. change:: + :tags: bug, sql + :tickets: 12990 + + Fixed issue where anonymous label generation for :class:`.CTE` constructs + could produce name collisions when Python's garbage collector reused memory + addresses during complex query compilation. The anonymous name generation + for :class:`.CTE` and other aliased constructs like :class:`.Alias`, + :class:`.Subquery` and others now use :func:`os.urandom` to generate unique + identifiers instead of relying on object ``id()``, ensuring uniqueness even + in cases of aggressive garbage collection and memory reuse. + + .. change:: + :tags: schema, usecase + :tickets: 13006 + + The the parameter :paramref:`_schema.DropConstraint.isolate_from_table` + was deprecated since it has no effect on the drop table behavior. + Its default values was also changed to ``False``. + + .. change:: + :tags: usecase, oracle + :tickets: 13010 + + The default DBAPI driver for the Oracle Database dialect has been changed + to ``oracledb`` instead of ``cx_oracle``. The ``cx_oracle`` driver remains + fully supported and can be explicitly specified in the connection URL + using ``oracle+cx_oracle://``. + + The ``oracledb`` driver is a modernized version of ``cx_oracle`` with + better performance characteristics and ongoing active development from + Oracle. + + .. seealso:: + + :ref:`change_13010_oracle` + + + .. change:: + :tags: usecase, postgresql + :tickets: 13010 + + The default DBAPI driver for the PostgreSQL dialect has been changed to + ``psycopg`` (psycopg version 3) instead of ``psycopg2``. The ``psycopg2`` + driver remains fully supported and can be explicitly specified in the + connection URL using ``postgresql+psycopg2://``. + + The ``psycopg`` (version 3) driver includes improvements over ``psycopg2`` + including better performance when using C extensions and native support + for async operations. + + .. seealso:: + + :ref:`change_13010_postgresql` + + .. change:: + :tags: feature, postgresql, sql + :tickets: 13014 + + Added support for monotonic server-side functions such as PostgreSQL 18's + ``uuidv7()`` to work with the :ref:`engine_insertmanyvalues` feature. + By passing ``monotonic=True`` to any :class:`.Function`, the function can + be used as a sentinel for tracking row order in batched INSERT operations + with RETURNING, allowing the ORM and Core to efficiently batch INSERT + statements while maintaining deterministic row ordering. + + .. seealso:: + + :ref:`change_13014_postgresql` + + :ref:`engine_insertmanyvalues_monotonic_functions` + + :ref:`postgresql_monotonic_functions` + + .. change:: + :tags: bug, engine + :tickets: 13018 + + Fixed issue in the :meth:`.ConnectionEvents.after_cursor_execute` method + where the SQL statement and parameter list for an "insertmanyvalues" + operation sent to the event would not be the actual SQL / parameters just + emitted on the cursor, instead being the non-batched form of the statement + that's used as a template to generate the batched statements. + + .. change:: + :tags: bug, orm + :tickets: 13021 + + A change in the mechanics of how Python dataclasses are applied to classes + that use :class:`.MappedAsDataclass` or + :meth:`.registry.mapped_as_dataclass` to apply ``__annotations__`` that are + as identical as is possible to the original ``__annotations__`` given, + while also adding attributes that SQLAlchemy considers to be part of + dataclass ``__annotations__``, then restoring the previous annotations in + exactly the same format as they were, using patterns that work with + :pep:`649` as closely as possible. + + .. change:: + :tags: bug, orm + :tickets: 13060 + + Removed the ``ORDER BY`` clause from queries generated by + :func:`_orm.selectin_polymorphic` and the + :paramref:`_orm.Mapper.polymorphic_load` parameter set to ``"selectin"``. + The ``ORDER BY`` clause appears to have been an unnecessary implementation + artifact. + + .. change:: + :tags: bug, orm + :tickets: 13070 + + A significant change to the ORM mechanics involved with both + :func:`.orm.with_loader_criteria` as well as single table inheritance, to + more aggressively locate WHERE criteria which should be augmented by either + the custom criteria or single-table inheritance criteria; SELECT statements + that do not include the entity within the columns clause or as an explicit + FROM, but still reference the entity within the WHERE clause, are now + covered, in particular this will allow subqueries using ``EXISTS (SELECT + 1)`` such as those rendered by :meth:`.RelationshipProperty.Comparator.any` + and :meth:`.RelationshipProperty.Comparator.has`. + + .. change:: + :tags: bug, mariadb + :tickets: 13076 + + Fixes to the MySQL/MariaDB dialect so that mariadb-specific features such + as the :class:`.mariadb.INET4` and :class:`.mariadb.INET6` datatype may be + used with an :class:`.Engine` that uses a ``mysql://`` URL, if the backend + database is actually a mariadb database. Previously, support for MariaDB + features when ``mysql://`` URLs were used instead of ``mariadb://`` URLs + was ad-hoc; with this issue resolution, the full set of schema / compiler / + type features are now available regardless of how the URL was presented. + + .. change:: + :tags: feature, schema + :tickets: 181 + + Added support for the SQL ``CREATE VIEW`` statement via the new + :class:`.CreateView` DDL class. The new class allows creating database + views from SELECT statements, with support for options such as + ``TEMPORARY``, ``IF NOT EXISTS``, and ``MATERIALIZED`` where supported by + the target database. Views defined with :class:`.CreateView` integrate with + :class:`.MetaData` for automated DDL generation and provide a + :class:`.Table` object for querying. + + .. seealso:: + + :ref:`change_4950` + + + + .. change:: + :tags: feature, schema + :tickets: 4950 + + Added support for the SQL ``CREATE TABLE ... AS SELECT`` construct via the + new :class:`_schema.CreateTableAs` DDL construct and the + :meth:`_sql.Select.into` method. The new construct allows creating a + table directly from the results of a SELECT statement, with support for + options such as ``TEMPORARY`` and ``IF NOT EXISTS`` where supported by the + target database. Tables defined with :class:`_schema.CreateTableAs` + integrate with :class:`.MetaData` for automated DDL generation and provide + a :class:`.Table` object for querying. Pull request courtesy Greg Jarzab. + + .. seealso:: + + :ref:`change_4950` + + + + .. change:: + :tags: change, sql + :tickets: 5252 + + the :class:`.Numeric` and :class:`.Float` SQL types have been separated out + so that :class:`.Float` no longer inherits from :class:`.Numeric`; instead, + they both extend from a common mixin :class:`.NumericCommon`. This + corrects for some architectural shortcomings where numeric and float types + are typically separate, and establishes more consistency with + :class:`.Integer` also being a distinct type. The change should not have + any end-user implications except for code that may be using + ``isinstance()`` to test for the :class:`.Numeric` datatype; third party + dialects which rely upon specific implementation types for numeric and/or + float may also require adjustment to maintain compatibility. + + .. change:: + :tags: change, sql + :tickets: 7066, 12915 + + Added new implementation for the :meth:`.Select.params` method and that of + similar statements, via a new statement-only + :meth:`.ExecutableStatement.params` method which works more efficiently and + correctly than the previous implementations available from + :class:`.ClauseElement`, by associating the given parameter dictionary with + the statement overall rather than cloning the statement and rewriting its + bound parameters. The :meth:`_sql.ClauseElement.params` and + :meth:`_sql.ClauseElement.unique_params` methods, when called on an object + that does not implement :class:`.ExecutableStatement`, will continue to + work the old way of cloning the object, and will emit a deprecation + warning. This issue both resolves the architectural / performance + concerns of :ticket:`7066` and also provides correct ORM compatibility for + functions like :func:`_orm.aliased`, reported by :ticket:`12915`. + + .. seealso:: + + :ref:`change_7066` + + .. change:: + :tags: usecase, sql + :tickets: 7910 + + Added method :meth:`.TableClause.insert_column` to complement + :meth:`.TableClause.append_column`, which inserts the given column at a + specific index. This can be helpful for prepending primary key columns to + tables, etc. + + + .. change:: + :tags: feature, asyncio + :tickets: 8047 + + The "emulated" exception hierarchies for the asyncio + drivers such as asyncpg, aiomysql, aioodbc, etc. have been standardized + on a common base :class:`.EmulatedDBAPIException`, which is now what's + available from the :attr:`.StatementException.orig` attribute on a + SQLAlchemy :class:`.DBAPIError` object. Within :class:`.EmulatedDBAPIException` + and the subclasses in its hierarchy, the original driver-level exception is + also now available via the :attr:`.EmulatedDBAPIException.orig` attribute, + and is also available from :class:`.DBAPIError` directly using the + :attr:`.DBAPIError.driver_exception` attribute. + + + + .. change:: + :tags: feature, postgresql + :tickets: 8047 + + Added additional emulated error classes for the subclasses of + ``asyncpg.exception.IntegrityError`` including ``RestrictViolationError``, + ``NotNullViolationError``, ``ForeignKeyViolationError``, + ``UniqueViolationError`` ``CheckViolationError``, + ``ExclusionViolationError``. These exceptions are not directly thrown by + SQLAlchemy's asyncio emulation, however are available from the + newly added :attr:`.DBAPIError.driver_exception` attribute when a + :class:`.IntegrityError` is caught. + + .. change:: + :tags: usecase, sql + :tickets: 8579 + + Added support for the pow operator (``**``), with a default SQL + implementation of the ``POW()`` function. On Oracle Database, PostgreSQL + and MSSQL it renders as ``POWER()``. As part of this change, the operator + routes through a new first class ``func`` member :class:`_functions.pow`, + which renders on Oracle Database, PostgreSQL and MSSQL as ``POWER()``. + + .. change:: + :tags: usecase, sql, orm + :tickets: 8601 + + The :meth:`_sql.Select.filter_by`, :meth:`.Update.filter_by` and + :meth:`.Delete.filter_by` methods now search across all entities + present in the statement, rather than limiting their search to only the + last joined entity or the first FROM entity. This allows these methods + to locate attributes unambiguously across multiple joined tables, + resolving issues where changing the order of operations such as + :meth:`_sql.Select.with_only_columns` would cause the method to fail. + + If an attribute name exists in more than one FROM clause entity, an + :class:`_exc.AmbiguousColumnError` is now raised, indicating that + :meth:`_sql.Select.filter` (or :meth:`_sql.Select.where`) should be used + instead with explicit table-qualified column references. + + .. seealso:: + + :ref:`change_8601` - Migration notes + + .. change:: + :tags: change, engine + :tickets: 9647 + + An empty sequence passed to any ``execute()`` method now + raised a deprecation warning, since such an executemany + is invalid. + Pull request courtesy of Carlos Sousa. + + .. change:: + :tags: feature, orm + :tickets: 9809 + + Session autoflush behavior has been simplified to unconditionally flush the + session each time an execution takes place, regardless of whether an ORM + statement or Core statement is being executed. This change eliminates the + previous conditional logic that only flushed when ORM-related statements + were detected, which had become difficult to define clearly with the unified + v2 syntax that allows both Core and ORM execution patterns. The change + provides more consistent and predictable session behavior across all types + of SQL execution. + + .. seealso:: + + :ref:`change_9809` + + .. change:: + :tags: feature, orm + :tickets: 9832 + + Added :class:`_orm.RegistryEvents` event class that allows event listeners + to be established on a :class:`_orm.registry` object. The new class + provides three events: :meth:`_orm.RegistryEvents.resolve_type_annotation` + which allows customization of type annotation resolution that can + supplement or replace the use of the + :paramref:`.registry.type_annotation_map` dictionary, including that it can + be helpful with custom resolution for complex types such as those of + :pep:`695`, as well as :meth:`_orm.RegistryEvents.before_configured` and + :meth:`_orm.RegistryEvents.after_configured`, which are registry-local + forms of the mapper-wide version of these hooks. + + .. seealso:: + + :ref:`change_9832` + + .. change:: + :tags: change, asyncio + + Removed the compatibility ``async_fallback`` mode for async dialects, + since it's no longer used by SQLAlchemy tests. + Also removed the internal function ``await_fallback()`` and renamed + the internal function ``await_only()`` to ``await_()``. + No change is expected to user code. + + .. change:: + :tags: feature, mysql + + Added new construct :func:`_mysql.limit` which can be applied to any + :func:`_sql.update` or :func:`_sql.delete` to provide the LIMIT keyword to + UPDATE and DELETE. This new construct supersedes the use of the + "mysql_limit" dialect keyword argument. + + + .. change:: + :tags: change, tests + + The top-level test runner has been changed to use ``nox``, adding a + ``noxfile.py`` as well as some included modules. The ``tox.ini`` file + remains in place so that ``tox`` runs will continue to function in the near + term, however it will be eventually removed and improvements and + maintenance going forward will be only towards ``noxfile.py``. + + + + .. change:: + :tags: change, platform + + Updated the setup manifest definition to use PEP 621-compliant + pyproject.toml. Also updated the extra install dependency to comply with + PEP-685. Thanks for the help of Matt Oberle and KOLANICH on this change. + + .. change:: + :tags: change, platform + :tickets: 10357, 12029, 12819 + + Python 3.10 or above is now required; support for Python 3.9, 3.8 and 3.7 + is dropped as these versions are EOL. + + .. change:: + :tags: engine, change + + The private method ``Connection._execute_compiled`` is removed. This method may + have been used for some special purposes however the :class:`.SQLCompiler` + object has lots of special state that should be set up for an execute call, + which we don't support. diff --git a/doc/build/changelog/migration_06.rst b/doc/build/changelog/migration_06.rst index 320f34009af..a8fd5f573c7 100644 --- a/doc/build/changelog/migration_06.rst +++ b/doc/build/changelog/migration_06.rst @@ -825,7 +825,7 @@ few changes there: subclasses NUMERIC, FLOAT, DECIMAL don't generate any length or scale unless specified. This also continues to include the controversial ``String`` and ``VARCHAR`` types - (although MySQL dialect will pre-emptively raise when + (although MySQL dialect will preemptively raise when asked to render VARCHAR with no length). No defaults are assumed, and if they are used in a CREATE TABLE statement, an error will be raised if the underlying database does diff --git a/doc/build/changelog/migration_07.rst b/doc/build/changelog/migration_07.rst index 4f1c98be1a8..4fae00d5008 100644 --- a/doc/build/changelog/migration_07.rst +++ b/doc/build/changelog/migration_07.rst @@ -163,7 +163,7 @@ scenarios. Highlights of this release include: ``cursor.execute`` for a large bulk insert of joined- table objects can be cut in half, allowing native DBAPI optimizations to take place for those statements passed - to ``cursor.executemany()`` (such as re-using a prepared + to ``cursor.executemany()`` (such as reusing a prepared statement). * The codepath invoked when accessing a many-to-one @@ -199,7 +199,7 @@ scenarios. Highlights of this release include: * The collection of "bind processors" for a particular ``Compiled`` instance of a statement is also cached on the ``Compiled`` object, taking further advantage of the - "compiled cache" used by the flush process to re-use the + "compiled cache" used by the flush process to reuse the same compiled form of INSERT, UPDATE, DELETE statements. A demonstration of callcount reduction including a sample diff --git a/doc/build/changelog/migration_09.rst b/doc/build/changelog/migration_09.rst index 61cd9a3a307..835b0f43eec 100644 --- a/doc/build/changelog/migration_09.rst +++ b/doc/build/changelog/migration_09.rst @@ -1892,7 +1892,7 @@ Firebird ``fdb`` and ``kinterbasdb`` set ``retaining=False`` by default Both the ``fdb`` and ``kinterbasdb`` DBAPIs support a flag ``retaining=True`` which can be passed to the ``commit()`` and ``rollback()`` methods of its connection. The documented rationale for this flag is so that the DBAPI -can re-use internal transaction state for subsequent transactions, for the +can reuse internal transaction state for subsequent transactions, for the purposes of improving performance. However, newer documentation refers to analyses of Firebird's "garbage collection" which expresses that this flag can have a negative effect on the database's ability to process cleanup diff --git a/doc/build/changelog/migration_10.rst b/doc/build/changelog/migration_10.rst index 1e61b308571..2e975253a27 100644 --- a/doc/build/changelog/migration_10.rst +++ b/doc/build/changelog/migration_10.rst @@ -2117,7 +2117,7 @@ for additional positions: [SQL: u'INSERT INTO my_table (id, data) VALUES (?, ?), (?, ?), (?, ?)'] [parameters: (1, 'd1', 'd2', 'd3')] -And with a "named" dialect, the same value for "id" would be re-used in +And with a "named" dialect, the same value for "id" would be reused in each row (hence this change is backwards-incompatible with a system that relied on this): diff --git a/doc/build/changelog/migration_11.rst b/doc/build/changelog/migration_11.rst index 15ef6fcd0c7..03a0d1202d4 100644 --- a/doc/build/changelog/migration_11.rst +++ b/doc/build/changelog/migration_11.rst @@ -1115,7 +1115,7 @@ Note that upon invalidation, the immediate DBAPI connection used by being used subsequent to the exception raise, will use a new DBAPI connection for subsequent operations upon next use; however, the state of any transaction in progress is lost and the appropriate ``.rollback()`` method -must be called if applicable before this re-use can proceed. +must be called if applicable before this reuse can proceed. In order to identify this change, it was straightforward to demonstrate a pymysql or mysqlclient / MySQL-Python connection moving into a corrupted state when diff --git a/doc/build/changelog/migration_12.rst b/doc/build/changelog/migration_12.rst index cd21d087910..6e0340aa131 100644 --- a/doc/build/changelog/migration_12.rst +++ b/doc/build/changelog/migration_12.rst @@ -202,7 +202,7 @@ are loaded with additional SELECT statements: employee.type AS employee_type, engineer.engineer_name AS engineer_engineer_name FROM employee JOIN engineer ON employee.id = engineer.id - WHERE employee.id IN (?, ?) ORDER BY employee.id + WHERE employee.id IN (?, ?) (1, 2) SELECT @@ -211,7 +211,7 @@ are loaded with additional SELECT statements: employee.type AS employee_type, manager.manager_name AS manager_manager_name FROM employee JOIN manager ON employee.id = manager.id - WHERE employee.id IN (?) ORDER BY employee.id + WHERE employee.id IN (?) (3,) .. seealso:: diff --git a/doc/build/changelog/migration_13.rst b/doc/build/changelog/migration_13.rst index a86e5bc089a..529dee5cecb 100644 --- a/doc/build/changelog/migration_13.rst +++ b/doc/build/changelog/migration_13.rst @@ -468,7 +468,7 @@ descriptor would raise an error. Additionally, it would assume that the first class to be seen by ``__get__()`` would be the only parent class it needed to know about. This is despite the fact that if a particular class has inheriting subclasses, the association proxy is really working on behalf of more than one -parent class even though it was not explicitly re-used. While even with this +parent class even though it was not explicitly reused. While even with this shortcoming, the association proxy would still get pretty far with its current behavior, it still leaves shortcomings in some cases as well as the complex problem of determining the best "owner" class. @@ -1326,7 +1326,7 @@ built-in ``Queue`` class in order to store database connections waiting to be used. The ``Queue`` features first-in-first-out behavior, which is intended to provide a round-robin use of the database connections that are persistently in the pool. However, a potential downside of this is that -when the utilization of the pool is low, the re-use of each connection in series +when the utilization of the pool is low, the reuse of each connection in series means that a server-side timeout strategy that attempts to reduce unused connections is prevented from shutting down these connections. To suit this use case, a new flag :paramref:`_sa.create_engine.pool_use_lifo` is added diff --git a/doc/build/changelog/migration_20.rst b/doc/build/changelog/migration_20.rst index 70dd6c41197..670daac7ef9 100644 --- a/doc/build/changelog/migration_20.rst +++ b/doc/build/changelog/migration_20.rst @@ -2192,7 +2192,7 @@ Therefore the best strategy for migrating from "dynamic" is to **wait until the application is fully running on 2.0**, then migrate directly from :class:`.AppenderQuery`, which is the collection type used by the "dynamic" strategy, to :class:`.WriteOnlyCollection`, which is the collection type -used by hte "write_only" strategy. +used by the "write_only" strategy. Some techniques are available to use ``lazy="dynamic"`` under 1.4 in a more "2.0" style however. There are two ways to achieve 2.0 style querying that's in diff --git a/doc/build/changelog/migration_21.rst b/doc/build/changelog/migration_21.rst index 304f9a5d249..987fbb2a582 100644 --- a/doc/build/changelog/migration_21.rst +++ b/doc/build/changelog/migration_21.rst @@ -1,4 +1,4 @@ -.. _whatsnew_21_toplevel: +.. _migration_21_toplevel: ============================= What's New in SQLAlchemy 2.1? @@ -10,6 +10,648 @@ What's New in SQLAlchemy 2.1? version 2.1. +Introduction +============ + +This guide introduces what's new in SQLAlchemy version 2.1 +and also documents changes which affect users migrating +their applications from the 2.0 series of SQLAlchemy to 2.1. + +Please carefully review the sections on behavioral changes for +potentially backwards-incompatible changes in behavior. + +General +======= + +.. _change_10197: + +Asyncio "greenlet" dependency no longer installs by default +------------------------------------------------------------ + +SQLAlchemy 1.4 and 2.0 used a complex expression to determine if the +``greenlet`` dependency, needed by the :ref:`asyncio ` +extension, could be installed from pypi using a pre-built wheel instead +of having to build from source. This because the source build of ``greenlet`` +is not always trivial on some platforms. + +Disadvantages to this approach included that SQLAlchemy needed to track +exactly which versions of ``greenlet`` were published as wheels on pypi; +the setup expression led to problems with some package management tools +such as ``poetry``; it was not possible to install SQLAlchemy **without** +``greenlet`` being installed, even though this is completely feasible +if the asyncio extension is not used. + +These problems are all solved by keeping ``greenlet`` entirely within the +``[asyncio]`` target. The only downside is that users of the asyncio extension +need to be aware of this extra installation dependency. + +:ticket:`10197` + +New Features and Improvements - ORM +==================================== + + + +.. _change_9809: + +Session autoflush behavior simplified to be unconditional +--------------------------------------------------------- + +Session autoflush behavior has been simplified to unconditionally flush the +session each time an execution takes place, regardless of whether an ORM +statement or Core statement is being executed. This change eliminates the +previous conditional logic that only flushed when ORM-related statements +were detected. + +Previously, the session would only autoflush when executing ORM queries:: + + # 2.0 behavior - autoflush only occurred for ORM statements + session.add(User(name="new user")) + + # This would trigger autoflush + users = session.execute(select(User)).scalars().all() + + # This would NOT trigger autoflush + result = session.execute(text("SELECT * FROM users")) + +In 2.1, autoflush occurs for all statement executions:: + + # 2.1 behavior - autoflush occurs for all executions + session.add(User(name="new user")) + + # Both of these now trigger autoflush + users = session.execute(select(User)).scalars().all() + result = session.execute(text("SELECT * FROM users")) + +This change provides more consistent and predictable session behavior across +all types of SQL execution. + +:ticket:`9809` + + +.. _change_10050: + +ORM Relationship allows callable for back_populates +--------------------------------------------------- + +To help produce code that is more amenable to IDE-level linting and type +checking, the :paramref:`_orm.relationship.back_populates` parameter now +accepts both direct references to a class-bound attribute as well as +lambdas which do the same:: + + class A(Base): + __tablename__ = "a" + + id: Mapped[int] = mapped_column(primary_key=True) + + # use a lambda: to link to B.a directly when it exists + bs: Mapped[list[B]] = relationship(back_populates=lambda: B.a) + + + class B(Base): + __tablename__ = "b" + id: Mapped[int] = mapped_column(primary_key=True) + a_id: Mapped[int] = mapped_column(ForeignKey("a.id")) + + # A.bs already exists, so can link directly + a: Mapped[A] = relationship(back_populates=A.bs) + +:ticket:`10050` + +.. _change_12168: + +ORM Mapped Dataclasses no longer populate implicit ``default``, collection-based ``default_factory`` in ``__dict__`` +-------------------------------------------------------------------------------------------------------------------- + +This behavioral change addresses a widely reported issue with SQLAlchemy's +:ref:`orm_declarative_native_dataclasses` feature that was introduced in 2.0. +SQLAlchemy ORM has always featured a behavior where a particular attribute on +an ORM mapped class will have different behaviors depending on if it has an +actively set value, including if that value is ``None``, versus if the +attribute is not set at all. When Declarative Dataclass Mapping was introduced, the +:paramref:`_orm.mapped_column.default` parameter introduced a new capability +which is to set up a dataclass-level default to be present in the generated +``__init__`` method. This had the unfortunate side effect of breaking various +popular workflows, the most prominent of which is creating an ORM object with +the foreign key value in lieu of a many-to-one reference:: + + class Base(MappedAsDataclass, DeclarativeBase): + pass + + + class Parent(Base): + __tablename__ = "parent" + + id: Mapped[int] = mapped_column(primary_key=True, init=False) + + related_id: Mapped[int | None] = mapped_column(ForeignKey("child.id"), default=None) + related: Mapped[Child | None] = relationship(default=None) + + + class Child(Base): + __tablename__ = "child" + + id: Mapped[int] = mapped_column(primary_key=True, init=False) + +In the above mapping, the ``__init__`` method generated for ``Parent`` +would in Python code look like this:: + + + def __init__(self, related_id=None, related=None): ... + +This means that creating a new ``Parent`` with ``related_id`` only would populate +both ``related_id`` and ``related`` in ``__dict__``:: + + # 2.0 behavior; will INSERT NULL for related_id due to the presence + # of related=None + >>> p1 = Parent(related_id=5) + >>> p1.__dict__ + {'related_id': 5, 'related': None, '_sa_instance_state': ...} + +The ``None`` value for ``'related'`` means that SQLAlchemy favors the non-present +related ``Child`` over the present value for ``'related_id'``, which would be +discarded, and ``NULL`` would be inserted for ``'related_id'`` instead. + +In the new behavior, the ``__init__`` method instead looks like the example below, +using a special constant ``DONT_SET`` indicating a non-present value for ``'related'`` +should be ignored. This allows the class to behave more closely to how +SQLAlchemy ORM mapped classes traditionally operate:: + + def __init__(self, related_id=DONT_SET, related=DONT_SET): ... + +We then get a ``__dict__`` setup that will follow the expected behavior of +omitting ``related`` from ``__dict__`` and later running an INSERT with +``related_id=5``:: + + # 2.1 behavior; will INSERT 5 for related_id + >>> p1 = Parent(related_id=5) + >>> p1.__dict__ + {'related_id': 5, '_sa_instance_state': ...} + +Dataclass defaults are delivered via descriptor instead of __dict__ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The above behavior goes a step further, which is that in order to +honor default values that are something other than ``None``, the value of the +dataclass-level default (i.e. set using any of the +:paramref:`_orm.mapped_column.default`, +:paramref:`_orm.column_property.default`, or :paramref:`_orm.deferred.default` +parameters) is directed to be delivered at the +Python :term:`descriptor` level using mechanisms in SQLAlchemy's attribute +system that normally return ``None`` for un-populated columns, so that even though the default is not +populated into ``__dict__``, it's still delivered when the attribute is +accessed. This behavior is based on what Python dataclasses itself does +when a default is indicated for a field that also includes ``init=False``. + +In the example below, an immutable default ``"default_status"`` +is applied to a column called ``status``:: + + class Base(MappedAsDataclass, DeclarativeBase): + pass + + + class SomeObject(Base): + __tablename__ = "parent" + + id: Mapped[int] = mapped_column(primary_key=True, init=False) + + status: Mapped[str] = mapped_column(default="default_status") + +In the above mapping, constructing ``SomeObject`` with no parameters will +deliver no values inside of ``__dict__``, but will deliver the default +value via descriptor:: + + # object is constructed with no value for ``status`` + >>> s1 = SomeObject() + + # the default value is not placed in ``__dict__`` + >>> s1.__dict__ + {'_sa_instance_state': ...} + + # but the default value is delivered at the object level via descriptor + >>> s1.status + 'default_status' + + # the value still remains unpopulated in ``__dict__`` + >>> s1.__dict__ + {'_sa_instance_state': ...} + +The value passed +as :paramref:`_orm.mapped_column.default` is also assigned as was the +case before to the :paramref:`_schema.Column.default` parameter of the +underlying :class:`_schema.Column`, where it takes +place as a Python-level default for INSERT statements. So while ``__dict__`` +is never populated with the default value on the object, the INSERT +still includes the value in the parameter set. This essentially modifies +the Declarative Dataclass Mapping system to work more like traditional +ORM mapped classes, where a "default" means just that, a column level +default. + +Dataclass defaults are accessible on objects even without init +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As the new behavior makes use of descriptors in a similar way as Python +dataclasses do themselves when ``init=False``, the new feature implements +this behavior as well. This is an all new behavior where an ORM mapped +class can deliver a default value for fields even if they are not part of +the ``__init__()`` method at all. In the mapping below, the ``status`` +field is configured with ``init=False``, meaning it's not part of the +constructor at all:: + + class Base(MappedAsDataclass, DeclarativeBase): + pass + + + class SomeObject(Base): + __tablename__ = "parent" + id: Mapped[int] = mapped_column(primary_key=True, init=False) + status: Mapped[str] = mapped_column(default="default_status", init=False) + +When we construct ``SomeObject()`` with no arguments, the default is accessible +on the instance, delivered via descriptor:: + + >>> so = SomeObject() + >>> so.status + default_status + +default_factory for collection-based relationships internally uses DONT_SET +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A late add to the behavioral change brings equivalent behavior to the +use of the :paramref:`_orm.relationship.default_factory` parameter with +collection-based relationships. This attribute is `documented ` +as being limited to exactly the collection class that's stated on the left side +of the annotation, which is now enforced at mapper configuration time:: + + class Parent(Base): + __tablename__ = "parents" + + id: Mapped[int] = mapped_column(primary_key=True, init=False) + name: Mapped[str] + + children: Mapped[list["Child"]] = relationship(default_factory=list) + +With the above mapping, the actual +:paramref:`_orm.relationship.default_factory` parameter is replaced internally +to instead use the same ``DONT_SET`` constant that's applied to +:paramref:`_orm.relationship.default` for many-to-one relationships. +SQLAlchemy's existing collection-on-attribute access behavior occurs as always +on access:: + + >>> p1 = Parent(name="p1") + >>> p1.children + [] + +This change to :paramref:`_orm.relationship.default_factory` accommodates a +similar merge-based condition where an empty collection would be forced into +a new object that in fact wants a merged collection to arrive. + + +Related Changes +^^^^^^^^^^^^^^^ + +This change includes the following API changes: + +* The :paramref:`_orm.relationship.default` parameter, when present, only + accepts a value of ``None``, and is only accepted when the relationship is + ultimately a many-to-one relationship or one that establishes + :paramref:`_orm.relationship.uselist` as ``False``. +* The :paramref:`_orm.mapped_column.default` and :paramref:`_orm.mapped_column.insert_default` + parameters are mutually exclusive, and only one may be passed at a time. + The behavior of the two parameters is equivalent at the :class:`_schema.Column` + level, however at the Declarative Dataclass Mapping level, only + :paramref:`_orm.mapped_column.default` actually sets the dataclass-level + default with descriptor access; using :paramref:`_orm.mapped_column.insert_default` + will have the effect of the object attribute defaulting to ``None`` on the + instance until the INSERT takes place, in the same way it works on traditional + ORM mapped classes. + +:ticket:`12168` + +.. _change_12496: + +New Hybrid DML hook features +---------------------------- + +To complement the existing :meth:`.hybrid_property.update_expression` decorator, +a new decorator :meth:`.hybrid_property.bulk_dml` is added, which works +specifically with parameter dictionaries passed to :meth:`_orm.Session.execute` +when dealing with ORM-enabled :func:`_dml.insert` or :func:`_dml.update`:: + + from typing import MutableMapping + from dataclasses import dataclass + + + @dataclass + class Point: + x: int + y: int + + + class Location(Base): + __tablename__ = "location" + + id: Mapped[int] = mapped_column(primary_key=True) + x: Mapped[int] + y: Mapped[int] + + @hybrid_property + def coordinates(self) -> Point: + return Point(self.x, self.y) + + @coordinates.inplace.bulk_dml + @classmethod + def _coordinates_bulk_dml( + cls, mapping: MutableMapping[str, Any], value: Point + ) -> None: + mapping["x"] = value.x + mapping["y"] = value.y + +Additionally, a new helper :func:`_sql.from_dml_column` is added, which may be +used with the :meth:`.hybrid_property.update_expression` hook to indicate +reuse of a column expression from elsewhere in the UPDATE statement's SET +clause:: + + from sqlalchemy import from_dml_column + + + class Product(Base): + __tablename__ = "product" + + id: Mapped[int] = mapped_column(primary_key=True) + price: Mapped[float] + tax_rate: Mapped[float] + + @hybrid_property + def total_price(self) -> float: + return self.price * (1 + self.tax_rate) + + @total_price.inplace.update_expression + @classmethod + def _total_price_update_expression(cls, value: Any) -> List[Tuple[Any, Any]]: + return [(cls.price, value / (1 + from_dml_column(cls.tax_rate)))] + +In the above example, if the ``tax_rate`` column is also indicated in the +SET clause of the UPDATE, that expression will be used for the ``total_price`` +expression rather than making use of the previous value of the ``tax_rate`` +column: + +.. sourcecode:: pycon+sql + + >>> from sqlalchemy import update + >>> print(update(Product).values({Product.tax_rate: 0.08, Product.total_price: 125.00})) + {printsql}UPDATE product SET tax_rate=:tax_rate, price=(:param_1 / (:tax_rate + :param_2)) + +When the target column is omitted, :func:`_sql.from_dml_column` falls back to +using the original column expression: + +.. sourcecode:: pycon+sql + + >>> from sqlalchemy import update + >>> print(update(Product).values({Product.total_price: 125.00})) + {printsql}UPDATE product SET price=(:param_1 / (tax_rate + :param_2)) + + +.. seealso:: + + :ref:`hybrid_bulk_update` + +:ticket:`12496` + + +.. _change_12570: + +New rules for None-return for ORM Composites +-------------------------------------------- + +ORM composite attributes configured using :func:`_orm.composite` can now +specify whether or not they should return ``None`` using a new parameter +:paramref:`_orm.composite.return_none_on`. By default, a composite +attribute now returns a non-None object in all cases, whereas previously +under 2.0, a ``None`` value would be returned for a pending object with +``None`` values for all composite columns. + +Given a composite mapping:: + + import dataclasses + + + @dataclasses.dataclass + class Point: + x: int | None + y: int | None + + + class Base(DeclarativeBase): + pass + + + class Vertex(Base): + __tablename__ = "vertices" + + id: Mapped[int] = mapped_column(primary_key=True) + + start: Mapped[Point] = composite(mapped_column("x1"), mapped_column("y1")) + end: Mapped[Point] = composite(mapped_column("x2"), mapped_column("y2")) + +When constructing a pending ``Vertex`` object, the initial value of the +``x1``, ``y1``, ``x2``, ``y2`` columns is ``None``. Under version 2.0, +accessing the composite at this stage would automatically return ``None``:: + + >>> v1 = Vertex() + >>> v1.start + None + +Under 2.1, the default behavior is to return the composite class with attributes +set to ``None``:: + + >>> v1 = Vertex() + >>> v1.start + Point(x=None, y=None) + +This behavior is now consistent with other forms of access, such as accessing +the attribute from a persistent object as well as querying for the attribute +directly. It is also consistent with the mapped annotation ``Mapped[Point]``. + +The behavior can be further controlled by applying the +:paramref:`_orm.composite.return_none_on` parameter, which accepts a callable +that returns True if the composite should be returned as None, given the +arguments that would normally be passed to the composite class. The typical callable +here would return True (i.e. the value should be ``None``) for the case where all +columns are ``None``:: + + class Vertex(Base): + __tablename__ = "vertices" + + id: Mapped[int] = mapped_column(primary_key=True) + + start: Mapped[Point] = composite( + mapped_column("x1"), + mapped_column("y1"), + return_none_on=lambda x, y: x is None and y is None, + ) + end: Mapped[Point] = composite( + mapped_column("x2"), + mapped_column("y2"), + return_none_on=lambda x, y: x is None and y is None, + ) + +For the above class, any ``Vertex`` instance whether pending or persistent will +return ``None`` for ``start`` and ``end`` if both composite columns for the attribute +are ``None``:: + + >>> v1 = Vertex() + >>> v1.start + None + +The :paramref:`_orm.composite.return_none_on` parameter is also set +automatically, if not otherwise set explicitly, when using +:ref:`orm_declarative_mapped_column`; setting the left hand side to +``Optional`` or ``| None`` will assign the above ``None``-handling callable:: + + + class Vertex(Base): + __tablename__ = "vertices" + + id: Mapped[int] = mapped_column(primary_key=True) + + # will apply return_none_on=lambda *args: all(arg is None for arg in args) + start: Mapped[Point | None] = composite(mapped_column("x1"), mapped_column("y1")) + end: Mapped[Point | None] = composite(mapped_column("x2"), mapped_column("y2")) + +The above object will return ``None`` for ``start`` and ``end`` automatically +if the columns are also None:: + + >>> session.scalars( + ... select(Vertex.start).where(Vertex.x1 == None, Vertex.y1 == None) + ... ).first() + None + +If :paramref:`_orm.composite.return_none_on` is set explicitly, that value will +supersede the choice made by ORM Annotated Declarative. This includes that +the parameter may be explicitly set to ``None`` which will disable the ORM +Annotated Declarative setting from taking place. + +:ticket:`12570` + +.. _change_9832: + +New RegistryEvents System for ORM Mapping Customization +-------------------------------------------------------- + +SQLAlchemy 2.1 introduces :class:`.RegistryEvents`, providing for event +hooks that are specific to a :class:`_orm.registry`. These events include +:meth:`_orm.RegistryEvents.before_configured` and :meth:`_orm.RegistryEvents.after_configured` +to complement the same-named events that can be established on a +:class:`_orm.Mapper`, as well as :meth:`_orm.RegistryEvents.resolve_type_annotation` +that allows programmatic access to the ORM Annotated Declarative type resolution +process. Examples are provided illustrating how to define resolution schemes +for any kind of type hierarchy in an automated fashion, including :pep:`695` +type aliases. + +E.g.:: + + from typing import Any + + from sqlalchemy import event + from sqlalchemy.orm import DeclarativeBase + from sqlalchemy.orm import registry as RegistryType + from sqlalchemy.orm import TypeResolve + from sqlalchemy.types import TypeEngine + + + class Base(DeclarativeBase): + pass + + + @event.listens_for(Base, "resolve_type_annotation") + def resolve_custom_type(resolve_type: TypeResolve) -> TypeEngine[Any] | None: + if resolve_type.resolved_type is MyCustomType: + return MyCustomSQLType() + else: + return None + + + @event.listens_for(Base, "after_configured") + def after_base_configured(registry: RegistryType) -> None: + print(f"Registry {registry} fully configured") + +.. seealso:: + + :ref:`orm_declarative_resolve_type_event` - Complete documentation on using + the :meth:`.RegistryEvents.resolve_type_annotation` event + + :class:`.RegistryEvents` - Complete API reference for all registry events + +:ticket:`9832` + +New Features and Improvements - Core +===================================== + +.. _change_12548: + +Template String (t-string) Support for Python 3.14+ +---------------------------------------------------- + +SQLAlchemy 2.1 adds support for Python 3.14+ template strings (t-strings) +via the new :func:`_sql.tstring` construct, as defined in :pep:`750`. +This feature provides a more ergonomic way to construct SQL statements by +automatically interpolating Python values and SQLAlchemy expressions within +template strings. + +The :func:`_sql.tstring` function works similarly to :func:`_sql.text`, but +automatically handles different types of interpolated values: + +* **String literals** from the template are rendered directly as SQL +* **SQLAlchemy expressions** (columns, functions, subqueries, etc.) are + embedded as clause elements +* **Plain Python values** are automatically wrapped with :func:`_sql.literal` + +Example usage:: + + from sqlalchemy import tstring, select, literal, JSON + + # Python values become bound values + user_id = 42 + stmt = tstring(t"SELECT * FROM users WHERE id = {user_id}") + # renders: SELECT * FROM users WHERE id = :param_1 + + # SQLAlchemy expressions are embedded + from sqlalchemy import table, column + + stmt = tstring(t"SELECT {column('q')} FROM {table('t')}") + # renders: SELECT q FROM t + + # Apply explicit SQL types to bound values using literal() + some_json = {"foo": "bar"} + stmt = tstring(t"SELECT {literal(some_json, JSON)}") + +Like :func:`_sql.text`, the :class:`_sql.TString` construct supports the +:meth:`_sql.TString.columns` method to specify return columns and their types:: + + from sqlalchemy import column, Integer, String + + stmt = tstring(t"SELECT id, name FROM users").columns( + column("id", Integer), column("name", String) + ) + + for id, name in connection.execute(stmt): + print(id, name) + +The :func:`_sql.tstring` construct is fully compatible with SQLAlchemy's +statement caching system. Statements with the same structure but different +literal values will share the same cache key, providing optimal performance. + +.. seealso:: + + :func:`_sql.tstring` + + :class:`_sql.TString` + + `PEP 750 `_ - Template Strings + +:ticket:`12548` + .. _change_10635: ``Row`` now represents individual column types directly without ``Tuple`` @@ -79,60 +721,277 @@ up front, which would be verbose and not automatic. :ticket:`10635` +.. _change_13085: -.. _change_10197: +Better type checker integration for Core froms, like Table +---------------------------------------------------------- -Asyncio "greenlet" dependency no longer installs by default ------------------------------------------------------------- +SQLAlchemy 2.1 changes :class:`_schema.Table`, along with most +:class:`_sql.FromClause` subclasses, to be generic on the column collection, +providing the option for better static type checking support. +By declaring the columns using a :class:`_schema.TypedColumns` subclass and +providing it to the :class:`_schema.Table` instance, IDEs and type checkers +can infer the exact types of columns when accessing them via the +:attr:`_schema.Table.c` attribute, enabling better autocomplete and type validation. -SQLAlchemy 1.4 and 2.0 used a complex expression to determine if the -``greenlet`` dependency, needed by the :ref:`asyncio ` -extension, could be installed from pypi using a pre-built wheel instead -of having to build from source. This because the source build of ``greenlet`` -is not always trivial on some platforms. +Example usage:: -Disadvantages to this approach included that SQLAlchemy needed to track -exactly which versions of ``greenlet`` were published as wheels on pypi; -the setup expression led to problems with some package management tools -such as ``poetry``; it was not possible to install SQLAlchemy **without** -``greenlet`` being installed, even though this is completely feasible -if the asyncio extension is not used. + from sqlalchemy import Table, TypedColumns, Column, Integer, MetaData, select -These problems are all solved by keeping ``greenlet`` entirely within the -``[asyncio]`` target. The only downside is that users of the asyncio extension -need to be aware of this extra installation dependency. -:ticket:`10197` + class user_cols(TypedColumns): + id = Column(Integer, primary_key=True) + name: Column[str] + age: Column[int] + # optional, used to infer the select types when selecting the table + __row_pos__: tuple[int, str, int] -.. _change_10050: -ORM Relationship allows callable for back_populates ---------------------------------------------------- + metadata = MetaData() + user = Table("user", metadata, user_cols) + + # Type checkers now understand the column types when selecting single columns + stmt = select(user.c.id, user.c.name) # Inferred as Select[int, str] + + # and also when selecting the whole table, when __row_pos__ is present + stmt = select(user) # Inferred as Select[int, str, int] + +The optional :attr:`sqlalchemy.sql._annotated_cols.HasRowPos.__row_pos__` annotation +is used to infer the types of a select when selecting the table directly. + +Columns can be declared in :class:`.TypedColumns` subclasses by instantiating +them directly or by using only a type annotations, that will be inferred when +generating a :class:`_schema.Table`. + +Other :class:`_sql.FromClause`, like :class:`_sql.Join`, :class:`_sql.CTE`, etc, can be made +generic using the :meth:`_sql.FromClause.with_cols` method:: + + # using with_cols the ``c`` collection of the cte has typed tables + cte = user.select().cte().with_cols(user_cols) + +ORM Integration +^^^^^^^^^^^^^^^ + +This functionality also offers some integration with the ORM, by using +:class:`_orm.MappedColumn` annotated attributes in the ORM model and +:func:`_orm.as_typed_table` to get an annotated :class:`_sql.FromClause`:: + + from sqlalchemy import TypedColumns + from sqlalchemy.orm import DeclarativeBase, mapped_column + from sqlalchemy.orm import MappedColumn, as_typed_table + + + class Base(DeclarativeBase): + pass + + + class A(Base): + __tablename__ = "a" + __typed_cols__: "a_cols" + + id: MappedColumn[int] = mapped_column(primary_key=True) + data: MappedColumn[str] + + + class a_cols(A, TypedColumns): + pass + + + # table_a is annotated as FromClause[a_cols], and is just A.__table__ + table_a = as_typed_table(A) + +For proper typing integration :class:`_orm.MappedColumn` should be used +to annotate the single columns, since it's a more specific annotation than +the usual :class:`_orm.Mapped` used for ORM attributes. + +:ticket:`13085` + +.. _change_8601: + +``filter_by()`` now searches across all FROM clause entities +------------------------------------------------------------- + +The :meth:`_sql.Select.filter_by` method, available for both Core +:class:`_sql.Select` objects and ORM-enabled select statements, has been +enhanced to search for attribute names across **all entities present in the +FROM clause** of the statement, rather than only looking at the last joined +entity or first FROM entity. + +This resolves a long-standing issue where the behavior of +:meth:`_sql.Select.filter_by` was sensitive to the order of operations. For +example, calling :meth:`_sql.Select.with_only_columns` after setting up joins +would reset which entity was searched, causing :meth:`_sql.Select.filter_by` +to fail even though the joined entity was still part of the FROM clause. + +Example - previously failing case now works:: + + from sqlalchemy import select, MetaData, Table, Column, Integer, String, ForeignKey + + metadata = MetaData() + + users = Table( + "users", + metadata, + Column("id", Integer, primary_key=True), + Column("name", String(50)), + ) + + addresses = Table( + "addresses", + metadata, + Column("id", Integer, primary_key=True), + Column("user_id", ForeignKey("users.id")), + Column("email", String(100)), + ) + + # This now works in 2.1 - previously raised an error + stmt = ( + select(users) + .join(addresses) + .with_only_columns(users.c.id) # changes selected columns + .filter_by(email="foo@bar.com") # searches addresses table successfully + ) + +Ambiguous Attribute Names +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When an attribute name exists in more than one entity in the FROM clause, +:meth:`_sql.Select.filter_by` now raises :class:`_exc.AmbiguousColumnError`, +indicating that :meth:`_sql.Select.filter` should be used instead with +explicit column references:: + + # Both users and addresses have 'id' column + stmt = select(users).join(addresses) + + # Raises AmbiguousColumnError in 2.1 + stmt = stmt.filter_by(id=5) + + # Use filter() with explicit qualification instead + stmt = stmt.filter(addresses.c.id == 5) + +The same behavior applies to ORM entities:: + + from sqlalchemy.orm import Session + + stmt = select(User).join(Address) + + # If both User and Address have an 'id' attribute, this raises + # AmbiguousColumnError + stmt = stmt.filter_by(id=5) + + # Use filter() with explicit entity qualification + stmt = stmt.filter(Address.id == 5) + +Legacy Query Use is Unchanged +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The change to :meth:`.Select.filter_by` has **not** been applied to the +:meth:`.Query.filter_by` method of :class:`.Query`; as :class:`.Query` is +a legacy API, its behavior hasn't changed. + +Migration Path +^^^^^^^^^^^^^^ + +Code that was previously working should continue to work without modification +in the vast majority of cases. The only breaking changes would be: + +1. **Ambiguous names that were previously accepted**: If your code had joins + where :meth:`_sql.Select.filter_by` happened to use an ambiguous column + name but it worked because it searched only one entity, this will now + raise :class:`_exc.AmbiguousColumnError`. The fix is to use + :meth:`_sql.Select.filter` with explicit column qualification. + +2. **Different entity selection**: In rare cases where the old behavior of + selecting the "last joined" or "first FROM" entity was being relied upon, + :meth:`_sql.Select.filter_by` might now find the attribute in a different + entity. Review any :meth:`_sql.Select.filter_by` calls in complex + multi-entity queries. + +It's hoped that in most cases, this change will make +:meth:`_sql.Select.filter_by` more intuitive to use. + +:ticket:`8601` + +.. _change_new_syntax_ext: + +New Syntax Extension Feature for Core +------------------------------------- + +Added the ability to create custom SQL constructs that can define new +clauses within SELECT, INSERT, UPDATE, and DELETE statements without +needing to modify the construction or compilation code of +:class:`.Select`, :class:`_dml.Insert`, :class:`.Update`, or :class:`.Delete` +directly. + +Custom extension can be created by subclassing the class +:class:`sqlalchemy.sql.SyntaxExtension`. +For example, support for the ``INTO OUTFILE`` clause of a select +supported by MariaDB and MySQL, can be implemented using syntax extensions +as follows:: + + from sqlalchemy.ext.compiler import compiles + from sqlalchemy.sql import ClauseElement, Select, SyntaxExtension, visitors + + + def into_outfile(name: str) -> "IntoOutFile": + """Return a INTO OUTFILE construct""" + return IntoOutFile(name) + + + class IntoOutFile(SyntaxExtension, ClauseElement): + """Define the INTO OUTFILE class.""" + + _traverse_internals = [("name", visitors.InternalTraversal.dp_string)] + """Structure that defines how SQLAlchemy can cache this element. + Specify ``inherit_cache=False`` to turn off caching. + """ + name: str + + def __init__(self, name: str): + self.name = name + + def apply_to_select(self, select_stmt: Select) -> None: + """Called when the :meth:`.Select.ext` method is called.""" + select_stmt.apply_syntax_extension_point( + self.append_replacing_same_type, "post_body" + ) -To help produce code that is more amenable to IDE-level linting and type -checking, the :paramref:`_orm.relationship.back_populates` parameter now -accepts both direct references to a class-bound attribute as well as -lambdas which do the same:: - class A(Base): - __tablename__ = "a" + @compiles(IntoOutFile) + def _compile_into_outfile(element: IntoOutFile, compiler, **kw): + """a compiles extension that compiles to SQL IntoOutFile""" + name = element.name.replace("'", "''") + return f"INTO OUTFILE '{name}'" - id: Mapped[int] = mapped_column(primary_key=True) +This can then be used in a select using the :meth:`.Select.ext` method: - # use a lambda: to link to B.a directly when it exists - bs: Mapped[list[B]] = relationship(back_populates=lambda: B.a) +.. sourcecode:: pycon+sql + >>> import sqlalchemy as sa - class B(Base): - __tablename__ = "b" - id: Mapped[int] = mapped_column(primary_key=True) - a_id: Mapped[int] = mapped_column(ForeignKey("a.id")) + >>> stmt = ( + ... sa.select(sa.column("a")) + ... .select_from(sa.table("tbl")) + ... .ext(into_outfile("myfile.txt")) + ... ) + >>> print(sql) + {printsql}SELECT a + FROM tbl INTO OUTFILE 'myfile.txt'{stop} - # A.bs already exists, so can link directly - a: Mapped[A] = relationship(back_populates=A.bs) +Several SQLAlchemy features custom to a single backend have been +re-implemented using this new system, including PostgreSQL +:func:`_postgresql.distinct_on` and MySQL :func:`_mysql.limit` functions +that supersede the previous implementations. + +.. seealso:: + + :ref:`examples_syntax_extensions` - A fully documented example of a + ``QUALIFY`` clause implemented using this new feature. + +:ticket:`12195` +:ticket:`12342` -:ticket:`10050` .. _change_11234: @@ -177,6 +1036,7 @@ be escaped in the result, leading to a URL that does not represent the original database portion. Below, `b=c` is part of the query string and not the database portion:: + >>> # pre-2.1 behavior >>> from sqlalchemy import URL >>> u = URL.create("driver", database="a?b=c") >>> str(u) @@ -184,6 +1044,700 @@ not the database portion:: :ticket:`11234` +.. _change_7066: + +Improved ``params()`` implementation for executable statements +-------------------------------------------------------------- + +The :meth:`_sql.ClauseElement.params` and :meth:`_sql.ClauseElement.unique_params` +methods have been deprecated in favor of a new implementation on executable +statements that provides improved performance and better integration with +ORM-enabled statements. + +Executable statement objects like :class:`_sql.Select`, :class:`_sql.CompoundSelect`, +and :class:`_sql.TextClause` now provide an improved :meth:`_sql.ExecutableStatement.params` +method that avoids a full cloned traversal of the statement tree. Instead, parameters +are stored directly on the statement object and efficiently merged during compilation +and/or cache key traversal. + +The new implementation provides several benefits: + +* **Better performance** - Parameters are stored in a simple dictionary rather than + requiring a full statement tree traversal with cloning +* **Proper caching integration** - Parameters are correctly integrated into SQLAlchemy's + cache key system via ``_generate_cache_key()`` +* **ORM statement compatibility** - Works correctly with ORM-enabled statements, including + ORM entities used with :func:`_orm.aliased`, subqueries, CTEs, etc. + +Use of :meth:`_sql.ExecutableStatement.params` is unchanged, provided the given +object is a statement object such as :func:`_sql.select`:: + + stmt = select(table).where(table.c.data == bindparam("x")) + + # Execute with parameter value + result = connection.execute(stmt.params(x=5)) + + # Can be chained and used in subqueries + stmt2 = stmt.params(x=6).subquery().select() + result = connection.execute(stmt2.params(x=7)) # Uses x=7 + +The deprecated :meth:`_sql.ClauseElement.params` and :meth:`_sql.ClauseElement.unique_params` +methods on non-executable elements like :class:`_sql.ColumnElement` and general +:class:`_sql.ClauseElement` instances will continue to work during the deprecation +period but will emit deprecation warnings. + +:ticket:`7066` + + +.. _change_4950: + +CREATE VIEW and CREATE TABLE AS SELECT Support +---------------------------------------------- + +SQLAlchemy 2.1 adds support for the SQL ``CREATE VIEW`` and +``CREATE TABLE ... AS SELECT`` constructs, as well as the ``SELECT ... INTO`` +variant for selected backends. Both DDL statements generate a table +or table-like construct based on the structure and rows represented by a +SELECT statement. The constructs are available via the :class:`.CreateView` +and :class:`_schema.CreateTableAs` DDL classes, as well as the +:meth:`_sql.SelectBase.into` convenience method. + +Both constructs work in exactly the same way, including that a :class:`.Table` +object is automatically generated from a given :class:`.Select`. DDL +can then be emitted by executing the construct directly or by allowing the +:meth:`.MetaData.create_all` or :meth:`.Table.create` sequences to emit the +correct DDL. + +E.g. using :class:`.CreateView`:: + + >>> from sqlalchemy import Table, Column, Integer, String, MetaData + >>> from sqlalchemy import CreateView, select + >>> + >>> metadata_obj = MetaData() + >>> user_table = Table( + ... "user_account", + ... metadata_obj, + ... Column("id", Integer, primary_key=True), + ... Column("name", String(30)), + ... Column("fullname", String), + ... ) + >>> view = CreateView( + ... select(user_table).where(user_table.c.name.like("%spongebob%")), + ... "spongebob_view", + ... metadata=metadata_obj, + ... ) + + +The above ``CreateView`` construct will emit CREATE VIEW when executed directly, +or when a DDL create operation is run. When using :meth:`.MetaData.create_all`, +the view is created after all dependent tables have been created: + +.. sourcecode:: pycon+sql + + >>> from sqlalchemy import create_engine + >>> e = create_engine("sqlite://", echo=True) + >>> metadata_obj.create_all(e) + {opensql}BEGIN (implicit) + + CREATE TABLE user_account ( + id INTEGER NOT NULL, + name VARCHAR(30), + fullname VARCHAR, + PRIMARY KEY (id) + ) + + CREATE VIEW spongebob_view AS + SELECT user_account.id, user_account.name, user_account.fullname + FROM user_account + WHERE user_account.name LIKE '%spongebob%' + + COMMIT + +The view is usable in SQL expressions via the :attr:`.CreateView.table` attribute: + +.. sourcecode:: pycon+sql + + >>> with e.connect() as conn: + ... conn.execute(select(view.table)) + {opensql}BEGIN (implicit) + SELECT spongebob_view.id, spongebob_view.name, spongebob_view.fullname + FROM spongebob_view + + ROLLBACK + +:class:`_schema.CreateTableAs` works in the same way, emitting ``CREATE TABLE AS``:: + + >>> from sqlalchemy import CreateTableAs + >>> select_stmt = select(users.c.id, users.c.name).where(users.c.name == "squidward") + >>> create_table_as = CreateTableAs(select_stmt, "squidward_users") + +In this case, :class:`.CreateTableAs` was not given a :class:`.MetaData` collection. +While a :class:`.MetaData` collection will be created automatically in this case, +the actual ``CREATE TABLE AS`` statement can also be generated by directly +executing the object: + +.. sourcecode:: pycon+sql + + >>> with e.begin() as conn: + ... conn.execute(create_table_as) + {opensql}BEGIN (implicit) + CREATE TABLE squidward_users AS SELECT user_account.id, user_account.name + FROM user_account + WHERE user_account.name = 'squidward' + COMMIT + +Like before, the :class:`.Table` is accessible from :attr:`.CreateTableAs.table`: + +.. sourcecode:: pycon+sql + + >>> with e.connect() as conn: + ... conn.execute(select(create_table_as.table)) + {opensql}BEGIN (implicit) + SELECT squidward_users.id, squidward_users.name + FROM squidward_users + + ROLLBACK + +.. seealso:: + + :ref:`metadata_create_view` - in :ref:`metadata_toplevel` + + :ref:`metadata_create_table_as` - in :ref:`metadata_toplevel` + + :class:`_schema.CreateView` - DDL construct for CREATE VIEW + + :class:`_schema.CreateTableAs` - DDL construct for CREATE TABLE AS + + :meth:`_sql.SelectBase.into` - convenience method on SELECT and UNION + statements + +:ticket:`4950` + + +.. _change_12736: + +Operator classes added to validate operator usage with datatypes +---------------------------------------------------------------- + +SQLAlchemy 2.1 introduces a new "operator classes" system that provides +validation when SQL operators are used with specific datatypes. This feature +helps catch usage of operators that are not appropriate for a given datatype +during the initial construction of expression objects. A simple example is an +integer or numeric column used with a "string match" operator. When an +incompatible operation is used, a deprecation warning is emitted; in a future +major release this will raise :class:`.InvalidRequestError`. + +The initial motivation for this new system is to revise the use of the +:meth:`.ColumnOperators.contains` method when used with :class:`_types.JSON` columns. +The :meth:`.ColumnOperators.contains` method in the case of the :class:`_types.JSON` +datatype makes use of the string-oriented version of the method, that +assumes string data and uses LIKE to match substrings. This is not compatible +with the same-named method that is defined by the PostgreSQL +:class:`_postgresql.JSONB` type, which uses PostgreSQL's native JSONB containment +operators. Because :class:`_types.JSON` data is normally stored as a plain string, +:meth:`.ColumnOperators.contains` would "work", and even in trivial cases +behave similarly to that of :class:`_postgresql.JSONB`. However, since the two +operations are not actually compatible at all, this mis-use can easily lead to +unexpected inconsistencies. + +Code that uses :meth:`.ColumnOperators.contains` with :class:`_types.JSON` columns will +now emit a deprecation warning:: + + from sqlalchemy import JSON, select, Column + from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + + class Base(DeclarativeBase): + pass + + + class MyTable(Base): + __tablename__ = "my_table" + + id: Mapped[int] = mapped_column(primary_key=True) + json_column: Mapped[dict] = mapped_column(JSON) + + + # This will now emit a deprecation warning + select(MyTable).filter(MyTable.json_column.contains("some_value")) + +Above, using :meth:`.ColumnOperators.contains` with :class:`_types.JSON` columns +is considered to be inappropriate, since :meth:`.ColumnOperators.contains` +works as a simple string search without any awareness of JSON structuring. +To explicitly indicate that the JSON data should be searched as a string +using LIKE, the +column should first be cast (using either :func:`_sql.cast` for a full CAST, +or :func:`_sql.type_coerce` for a Python-side cast) to :class:`.String`:: + + from sqlalchemy import type_coerce, String + + # Explicit string-based matching + select(MyTable).filter(type_coerce(MyTable.json_column, String).contains("some_value")) + +This change forces code to distinguish between using string-based "contains" +with a :class:`_types.JSON` column and using PostgreSQL's JSONB containment +operator with :class:`_postgresql.JSONB` columns as separate, explicitly-stated operations. + +The operator class system involves a mapping of SQLAlchemy operators listed +out in :mod:`sqlalchemy.sql.operators` to operator class combinations that come +from the :class:`.OperatorClass` enumeration, which are reconciled at +expression construction time with datatypes using the +:attr:`.TypeEngine.operator_classes` attribute. A custom user defined type +may want to set this attribute to indicate the kinds of operators that make +sense:: + + from sqlalchemy.types import UserDefinedType + from sqlalchemy.sql.sqltypes import OperatorClass + + + class ComplexNumber(UserDefinedType): + operator_classes = OperatorClass.MATH + +The above ``ComplexNumber`` datatype would then validate that operators +used are included in the "math" operator class. By default, user defined +types made with :class:`.UserDefinedType` are left open to accept all +operators by default, whereas classes defined with :class:`.TypeDecorator` +will make use of the operator classes declared by the "impl" type. + +.. seealso:: + + :paramref:`.Operators.op.operator_class` - define an operator class when creating custom operators + + :class:`.OperatorClass` + +:ticket:`12736` + +.. _change_12596: + +Non-integer RANGE window frame clauses now supported +----------------------------------------------------- + +The :func:`_sql.over` clause now supports non-integer values in the +:paramref:`_sql.over.range_` parameter through the new :class:`_sql.FrameClause` +construct. Previously, only integer values were allowed in RANGE clauses, which +limited their use to integer-based ordering columns. + +With this change, applications can now use RANGE with other data types such +as floating-point numbers, dates, and intervals. The new :class:`_sql.FrameClause` +construct provides explicit control over frame boundaries using the +:class:`_sql.FrameClauseType` enum:: + + from datetime import timedelta + from sqlalchemy import FrameClause, FrameClauseType + + # Example: date-based RANGE with a 7-day window + func.sum(my_table.c.amount).over( + order_by=my_table.c.date, + range_=FrameClause( + start=timedelta(days=7), + end=None, + start_frame_type=FrameClauseType.PRECEDING, + end_frame_type=FrameClauseType.CURRENT, + ), + ) + +For backwards compatibility, the traditional tuple-based syntax continues to +work with integer values:: + + # This continues to work unchanged + func.row_number().over(order_by=table.c.col, range_=(None, 10)) + +However, attempting to use non-integer values in the tuple syntax will now +raise an error, directing users to use :class:`_sql.FrameClause` instead. + + +:ticket:`12596` + +.. _change_10300: + +Python float literals now render as DOUBLE in CAST expressions +--------------------------------------------------------------- + +When Python ``float`` values are used as literal bound parameters in SQL +expressions, in the absence of an explicit type being passed to the bound +parameter expression, they now use the :class:`.Double` type instead of +:class:`.Float`. This change affects the SQL keyword that is rendered when +these values appear in CAST expressions. + +The :class:`.Double` and :class:`.Float` types have identical implementations +and behavior at runtime. The only difference is in DDL and CAST rendering: +:class:`.Double` renders as ``DOUBLE`` or ``DOUBLE PRECISION``, while +:class:`.Float` renders as ``FLOAT``. The ``DOUBLE`` keyword better matches +Python's ``float`` datatype, which uses 8-byte double-precision storage. + +This change is most visible in division operations, where a float divisor is +automatically cast to ensure proper floating-point division:: + + from sqlalchemy import table, column, Integer, select + + t = table("t", column("x", Integer)) + + stmt = select(t.c.x / 5.0) + +The above statement will now render with a DOUBLE CAST on most backends: + +.. sourcecode:: sql + + SELECT t.x / CAST(:x_1 AS DOUBLE) AS anon_1 FROM t + +Previously, this would have rendered as ``CAST(:x_1 AS FLOAT)``. + +Notes for Third-Party Dialects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Third-party dialect implementations that don't support the ``DOUBLE`` keyword +will need to ensure they have a ``visit_double()`` method in their type compiler +to render an appropriate alternative keyword for the target database. + +For example, a dialect for a database that doesn't support ``DOUBLE`` might +map it to ``FLOAT``:: + + class MyDialectTypeCompiler(GenericTypeCompiler): + def visit_double(self, type_, **kw): + # Map double to FLOAT for databases that don't support DOUBLE + return "FLOAT" + +The built-in SQLAlchemy dialects (PostgreSQL, MySQL, Oracle, SQL Server, +SQLite) all handle ``visit_double()`` by rendering either ``DOUBLE`` or +``DOUBLE PRECISION``. + +:ticket:`10300` + + +PostgreSQL +========== + +.. _change_13010_postgresql: + +Default PostgreSQL driver changed to psycopg (psycopg 3) +--------------------------------------------------------- + +The default DBAPI driver for the PostgreSQL dialect has been changed from +``psycopg2`` to ``psycopg`` (psycopg version 3). When using a connection URL +of the form ``postgresql://user:pass@host/dbname``, SQLAlchemy will now +attempt to use the ``psycopg`` driver by default. + +The ``psycopg`` (version 3) driver is the modernized successor to +``psycopg2``, featuring improved performance when built with C extensions, +better support for modern PostgreSQL features, and native async support via +the ``psycopg_async`` dialect. The performance characteristics of ``psycopg`` +with C extensions are comparable to ``psycopg2``. + +The ``psycopg2`` driver remains fully supported and can be used by explicitly +specifying it in the connection URL. + +Examples to summarize the change are as follows:: + + # omit the driver portion, will use the psycopg dialect + engine = create_engine("postgresql://user:pass@host/dbname") + + # indicate the psycopg driver/dialect explicitly (preferred) + engine = create_engine("postgresql+psycopg://user:pass@host/dbname") + + # use the legacy psycopg2 driver/dialect + engine = create_engine("postgresql+psycopg2://user:pass@host/dbname") + +The ``psycopg`` DBAPI driver itself can be installed either directly +or via the ``sqlalchemy[postgresql]`` extra:: + +.. sourcecode:: txt + + # install psycopg directly + pip install "psycopg[binary]" + + # or use SQLAlchemy's postgresql extra (now installs psycopg) + pip install sqlalchemy[postgresql] + +.. seealso:: + + :ref:`postgresql_psycopg` - Documentation for the psycopg 3 dialect + +:ticket:`13010` + +.. _change_10594_postgresql: + +Changes to Named Type Handling in PostgreSQL +--------------------------------------------- + +Named types such as :class:`_postgresql.ENUM`, :class:`_postgresql.DOMAIN` and +the dialect-agnostic :class:`._types.Enum` have undergone behavioral changes in +SQLAlchemy 2.1 to better align with how a distinct type object that may +be shared among tables works in practice. + +Named Types are Now Associated with MetaData +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Named types are now more strongly associated with the :class:`_schema.MetaData` +at the top of the table hierarchy and are de-associated with any particular +:class:`_schema.Table` they may be a part of. This better represents how +PostgreSQL named types exist independently of any particular table, and that +they may be used across many tables simultaneously. + +:class:`_types.Enum` and :class:`_postgresql.DOMAIN` now have their +:attr:`~_types.SchemaType.metadata` attribute set as soon as they are +associated with a table, and no longer refer to the :class:`_schema.Table` +or tables they are within (a table of course still refers to the named types +that it uses). + +Schema Inheritance from MetaData +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Named types will now "inherit" the schema of the :class:`_schema.MetaData` +by default. For example, ``MetaData(schema="myschema")`` will cause all +:class:`_types.Enum` and :class:`_postgresql.DOMAIN` to use the schema +"myschema":: + + metadata = MetaData(schema="myschema") + + table = Table( + "mytable", + metadata, + Column("status", Enum("active", "inactive", name="status_enum")), + ) + + # The enum will be created as "myschema.status_enum" + +To have named types use the schema name of an immediate :class:`_schema.Table` +that they are associated with, set the :paramref:`~_types.SchemaType.schema` +parameter of the type to be that same schema name:: + + table = Table( + "mytable", + metadata, + Column( + "status", Enum("active", "inactive", name="status_enum", schema="tableschema") + ), + schema="tableschema", + ) + +The :paramref:`_types.SchemaType.inherit_schema` parameter remains available +for this release but is deprecated for eventual removal, and will emit a +deprecation warning when used. + +Modified Create and Drop Behavior +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The rules by which named types are created and dropped are also modified to +flow more in terms of a :class:`_schema.MetaData`: + +1. :meth:`_schema.MetaData.create_all` and :meth:`_schema.Table.create` will + create any named types needed +2. :meth:`_schema.Table.drop` will not drop any named types +3. :meth:`_schema.MetaData.drop_all` will drop named types after all tables + are dropped + +Refined CheckFirst Behavior +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There is also newly refined "checkfirst" behavior. A new enumeration +:class:`_schema.CheckFirst` is introduced which allows fine-grained control +within :meth:`_schema.MetaData.create_all`, :meth:`_schema.MetaData.drop_all`, +:meth:`_schema.Table.create`, and :meth:`_schema.Table.drop` as to what "check" +queries are emitted, allowing tests for types, sequences etc. to be included +or not:: + + from sqlalchemy import CheckFirst + + # Only check for table existence, skip type checks + metadata.create_all(engine, checkfirst=CheckFirst.TABLES) + + # Check for both tables and types + metadata.create_all(engine, checkfirst=CheckFirst.TABLES | CheckFirst.TYPES) + +inherit_schema is Deprecated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Because named types now inherit the schema of :class:`.MetaData` automatically +and remain agnostic of what :class:`.Table` objects refer to them, the +:paramref:`_types.Enum.inherit_schema` parameter is deprecated. For 2.1 +it still works the old way by associating the type with the parent +:class:`.Table`, however as this binds the type to a single :class:`.Table` +even though the type can be used against any number of tables, it's preferred +to set :paramref:`_types.Enum.schema` directly as desired when the schema +used by the :class:`.MetaData` is not what's desired. + +:ticket:`10594` + +Support for ``VIRTUAL`` computed columns +---------------------------------------- + +The behaviour of :paramref:`.Computed.persisted` has change in SQLAlchemy 2.1 +to no longer indicate ``STORED`` computed columns by default in PostgreSQL.. + +This change aligns SQLAlchemy with PostgreSQL 18+, which has introduced +support for ``VIRTUAL`` computed columns, and has made them the default +type if no qualifier is specified. + +Migration Path +^^^^^^^^^^^^^^ + +To maintain the previous behaviour of ``STORED`` computed columns, +:paramref:`.Computed.persisted` should be set to ``True`` explicitly:: + + from sqlalchemy import Table, Column, MetaData, Computed, Integer + + metadata = MetaData() + + t = Table( + "t", + metadata, + Column("x", Integer), + Column("x^2", Integer, Computed("x * x", persisted=True)), + ) + +.. _change_10556: + +Addition of ``BitString`` subclass for handling postgresql ``BIT`` columns +-------------------------------------------------------------------------- + +Values of :class:`_postgresql.BIT` columns in the PostgreSQL dialect are +returned as instances of a new ``str`` subclass, +:class:`_postgresql.BitString`. Previously, the value of :class:`_postgresql.BIT` +columns was driver dependent, with most drivers returning ``str`` instances +except ``asyncpg``, which used ``asyncpg.BitString``. + +With this change, for the ``psycopg``, ``psycopg2``, and ``pg8000`` drivers, +the new :class:`_postgresql.BitString` type is mostly compatible with ``str``, but +adds methods for bit manipulation and supports bitwise operators. + +As :class:`_postgresql.BitString` is a string subclass, hashability as well +as equality tests continue to work against plain strings. This also leaves +ordering operators intact. + +For implementations using the ``asyncpg`` driver, the new type is incompatible with +the existing ``asyncpg.BitString`` type. + +:ticket:`10556` + +.. _change_12948: + +HSTORE subscripting now uses native PostgreSQL 14+ syntax +---------------------------------------------------------- + +When connected to PostgreSQL 14 or later, HSTORE column subscripting operations +now automatically use PostgreSQL's native subscript notation ``hstore_col['key']`` +instead of the traditional arrow operator ``hstore_col -> 'key'``. This change +applies to both read and write operations and provides better compatibility with +PostgreSQL's native HSTORE subscripting feature introduced in version 14. + +The change is similar to the equivalent change made for JSONB columns in +SQLAlchemy 2.0.42. The change for HSTORE is kept in 2.1 to provide a longer +migration buffer for the issue of indexes which may refer to a subscript +function, which was unanticipated when the JSONB change was made. + +For PostgreSQL versions prior to 14, SQLAlchemy continues to use the arrow +operator syntax automatically, ensuring backward compatibility. + +Example of the new syntax when connected to PostgreSQL 14+:: + + from sqlalchemy import table, column, update + from sqlalchemy.dialects.postgresql import HSTORE + + data = table("data", column("h", HSTORE)) + + stmt1 = select(data.c.h["key"]) + + stmt2 = update(data).values({data.c.h["status"]: "active"}) + +On PostgreSQL 14 and above, the statements above would render as: + +.. sourcecode:: sql + + -- new subscript operator on PostgreSQL 14+ + SELECT data.h['key'] FROM data + + UPDATE data SET h['status'] = 'active' + +On PostgreSQL 13 and earlier, they would render using the +arrow operator: + +.. sourcecode:: sql + + -- Same code on PostgreSQL 13 renders as: + SELECT data.h -> 'key' FROM data + + UPDATE data SET h -> 'status' = 'active' + +Impact on Existing Indexes +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If existing PostgreSQL 14+ databases have expression indexes on HSTORE +subscript operations, those indexes will need to be recreated to match +the new SQL syntax. The index definitions that used the arrow operator +syntax (``h -> 'key'``) will not match the new subscript syntax +(``h['key']``), which may cause index scans to not be used. + +To update an existing index: + +.. sourcecode:: sql + + -- Drop the old index + DROP INDEX IF EXISTS idx_hstore_key; + + -- Create new index with subscript syntax + CREATE INDEX idx_hstore_key ON my_table ((h['key'])); + +:ticket:`12948` + +.. _change_13014_postgresql: + +Support for Server-Side Monotonic Functions such as uuidv7() in Batched INSERT Operations +------------------------------------------------------------------------------------------ + +SQLAlchemy 2.1 adds support for using monotonic server-side functions, such as +PostgreSQL 18's ``uuidv7()`` function, as sentinels in the +:ref:`engine_insertmanyvalues` feature. This allows these functions to work +efficiently with batched INSERT operations while maintaining deterministic row +ordering. + +When using a monotonic function as a default value, the ``monotonic=True`` +parameter must be passed to the function to indicate that it produces +monotonically increasing values. This enables SQLAlchemy to use the function's +values to correlate RETURNING results with input parameter sets:: + + from sqlalchemy import Table, Column, MetaData, UUID, Integer, func + + metadata = MetaData() + + t = Table( + "t", + metadata, + Column("id", UUID, server_default=func.uuidv7(monotonic=True), primary_key=True), + Column("x", Integer), + ) + +With the above configuration, when performing a batched INSERT with RETURNING +on PostgreSQL, SQLAlchemy will generate SQL that properly orders the rows +while allowing the server to generate the UUID values: + +.. sourcecode:: sql + + INSERT INTO t (x) SELECT p0::INTEGER FROM + (VALUES (%(x__0)s, 0), (%(x__1)s, 1), (%(x__2)s, 2), ...) + AS imp_sen(p0, sen_counter) ORDER BY sen_counter + RETURNING t.id, t.id AS id__1 + +The returned rows are then sorted by the monotonically increasing UUID values +to match the order of the input parameters, ensuring that ORM objects and +returned values are properly correlated. + +This feature works with both :paramref:`_schema.Column.server_default` (for +DDL-level defaults) and :paramref:`_schema.Column.default` (for ad-hoc +server-side function calls). + +.. seealso:: + + :ref:`engine_insertmanyvalues_monotonic_functions` - Complete documentation + on using monotonic functions + + :ref:`postgresql_monotonic_functions` - PostgreSQL-specific examples + +:ticket:`13014` + + +Microsoft SQL Server +==================== + .. _change_11250: Potential breaking change to odbc_connect= handling for mssql+pyodbc @@ -206,3 +1760,109 @@ would appear in a valid ODBC connection string (i.e., the same as would be required if using the connection string directly with ``pyodbc.connect()``). :ticket:`11250` + +.. _change_12869: + +Support for mssql-python driver +-------------------------------- + +SQLAlchemy 2.1 adds support for the ``mssql-python`` driver, Microsoft's +official Python driver for SQL Server. This driver represents a modern +alternative to the widely-used ``pyodbc`` driver, with native support for +several SQL Server-specific features. + +The ``mssql-python`` driver can be used by specifying ``mssql+mssqlpython`` +in the connection URL:: + + from sqlalchemy import create_engine + + # Basic connection + engine = create_engine("mssql+mssqlpython://user:password@hostname/database") + + # With Windows Authentication + engine = create_engine( + "mssql+mssqlpython://hostname/database?authentication=ActiveDirectoryIntegrated" + ) + +The ``mssql-python`` driver is available from PyPI: + +.. sourcecode:: text + + pip install mssql-python + +.. seealso:: + + :ref:`mssql_python` - Documentation for the mssql-python dialect + +:ticket:`12869` + + +Oracle Database +=============== + +.. _change_13010_oracle: + +Default Oracle driver changed to python-oracledb +------------------------------------------------- + +The default DBAPI driver for the Oracle dialect has been changed from +``cx_oracle`` to ``oracledb`` (python-oracledb). When using a connection URL +of the form ``oracle://user:pass@host/dbname``, SQLAlchemy will now attempt +to use the ``oracledb`` driver by default. + +The ``oracledb`` driver is the modernized successor to ``cx_oracle``, +actively maintained by Oracle with improved performance characteristics and +ongoing feature development. The documentation for ``cx_oracle`` has been +largely replaced with references to ``oracledb`` at +https://cx-oracle.readthedocs.io/. + +The ``cx_oracle`` driver remains fully supported and can be used by +explicitly specifying it in the connection URL. + +Examples to summarize the change are as follows:: + + # omit the driver portion, will use the oracledb dialect + engine = create_engine("oracle://user:pass@host/dbname") + + # indicate the oracledb driver/dialect explicitly (preferred) + engine = create_engine("oracle+oracledb://user:pass@host/dbname") + + # use the legacy cx_oracle driver/dialect + engine = create_engine("oracle+cx_oracle://user:pass@host/dbname") + +The ``oracledb`` DBAPI driver itself can be installed either directly +or via the ``sqlalchemy[oracle]`` extra: + +.. sourcecode:: text + + # install oracledb directly + pip install oracledb + + # or use SQLAlchemy's oracle extra (now installs oracledb) + pip install sqlalchemy[oracle] + +.. seealso:: + + :ref:`oracledb` - Documentation for the python-oracledb dialect + +:ticket:`13010` + +.. _change_11633: + +Native :class:`.BOOLEAN` support for Oracle 23c and above +---------------------------------------------------------- + +The :class:`.Boolean` emulated datatype will now produce the +DDL ``BOOLEAN`` when Oracle Database 23c or higher is used. +The :class:`.BOOLEAN` exact datatype may also be used with Oracle +Database. For earlier versions, the :class:`.Boolean` emulated +type will still produce ``SMALLINT`` in DDL and convert between Boolean +and integer. An Oracle database that uses ``SMALLINT`` with emulation +on version 23c or above will also function correctly when using +the :class:`.Boolean` datatype. + +.. seealso:: + + :ref:`oracle_boolean_support` + +:ticket:`11633` diff --git a/doc/build/changelog/unreleased_20/12332.rst b/doc/build/changelog/unreleased_20/12332.rst deleted file mode 100644 index a6c1d4e2fb1..00000000000 --- a/doc/build/changelog/unreleased_20/12332.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. change:: - :tags: bug, mysql - :tickets: 12332 - - Support has been re-added for the MySQL-Connector/Python DBAPI using the - ``mysql+mysqlconnector://`` URL scheme. The DBAPI now works against - modern MySQL versions as well as MariaDB versions (in the latter case it's - required to pass charset/collation explicitly). Note however that - server side cursor support is disabled due to unresolved issues with this - driver. diff --git a/doc/build/changelog/unreleased_20/12363.rst b/doc/build/changelog/unreleased_20/12363.rst deleted file mode 100644 index e04e51fe0de..00000000000 --- a/doc/build/changelog/unreleased_20/12363.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. change:: - :tags: bug, sql - :tickets: 12363 - - Fixed issue in :class:`.CTE` constructs involving multiple DDL - :class:`.Insert` statements with multiple VALUES parameter sets where the - bound parameter names generated for these parameter sets would conflict, - generating a compile time error. - diff --git a/doc/build/changelog/unreleased_20/13229.rst b/doc/build/changelog/unreleased_20/13229.rst new file mode 100644 index 00000000000..e02886bed01 --- /dev/null +++ b/doc/build/changelog/unreleased_20/13229.rst @@ -0,0 +1,8 @@ +.. change:: + :tags: postgresql, bug + :tickets: 13229 + + Improve handling of two phase transaction identifiers for PostgreSQL + when the identifier is provided by the user. + As part of this change the psycopg dialect was updated to use the DBAPI + two phase transaction API instead of executing the SQL directly. diff --git a/doc/build/changelog/unreleased_20/13230.rst b/doc/build/changelog/unreleased_20/13230.rst new file mode 100644 index 00000000000..a9cb60e6706 --- /dev/null +++ b/doc/build/changelog/unreleased_20/13230.rst @@ -0,0 +1,5 @@ +.. change:: + :tags: bug, sqlite + :tickets: 13230 + + Escape key and pragma values when utilizing the pysqlcipher dialect. diff --git a/doc/build/changelog/unreleased_20/13241.rst b/doc/build/changelog/unreleased_20/13241.rst new file mode 100644 index 00000000000..3b129450d2e --- /dev/null +++ b/doc/build/changelog/unreleased_20/13241.rst @@ -0,0 +1,9 @@ +.. change:: + :tags: bug, postgresql + :tickets: 13241 + + Fixed issue where the asyncpg driver could throw an insufficiently-handled + exception ``InternalClientError`` under some circumstances, leading to + connections not being properly marked as invalidated. + + diff --git a/doc/build/changelog/unreleased_20/13243.rst b/doc/build/changelog/unreleased_20/13243.rst new file mode 100644 index 00000000000..2a19b9e4748 --- /dev/null +++ b/doc/build/changelog/unreleased_20/13243.rst @@ -0,0 +1,13 @@ +.. change:: + :tags: bug, mysql, reflection + :tickets: 13243 + + Narrowed the scope of the internal workaround for MySQL bugs `#88718 + `_ and `#96365 + `_ so that it is only applied + where needed: MySQL 8.0.1 through 8.0.13 (where bug 88718 is present), and + on systems with ``lower_case_table_names=2`` (where bug 96365 applies, + typically macOS). Previously the workaround was applied unconditionally + for all MySQL 8.0+ versions, which caused a ``KeyError`` during foreign key + reflection when the database user lacked SELECT privileges on referred + tables. diff --git a/doc/build/changelog/unreleased_21/10050.rst b/doc/build/changelog/unreleased_21/10050.rst deleted file mode 100644 index a1c1753a1c1..00000000000 --- a/doc/build/changelog/unreleased_21/10050.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. change:: - :tags: feature, orm - :tickets: 10050 - - The :paramref:`_orm.relationship.back_populates` argument to - :func:`_orm.relationship` may now be passed as a Python callable, which - resolves to either the direct linked ORM attribute, or a string value as - before. ORM attributes are also accepted directly by - :paramref:`_orm.relationship.back_populates`. This change allows type - checkers and IDEs to confirm the argument for - :paramref:`_orm.relationship.back_populates` is valid. Thanks to Priyanshu - Parikh for the help on suggesting and helping to implement this feature. - - .. seealso:: - - :ref:`change_10050` - diff --git a/doc/build/changelog/unreleased_21/10197.rst b/doc/build/changelog/unreleased_21/10197.rst deleted file mode 100644 index f3942383225..00000000000 --- a/doc/build/changelog/unreleased_21/10197.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. change:: - :tags: change, installation - :tickets: 10197 - - The ``greenlet`` dependency used for asyncio support no longer installs - by default. This dependency does not publish wheel files for every architecture - and is not needed for applications that aren't using asyncio features. - Use the ``sqlalchemy[asyncio]`` install target to include this dependency. - - .. seealso:: - - :ref:`change_10197` - - diff --git a/doc/build/changelog/unreleased_21/10236.rst b/doc/build/changelog/unreleased_21/10236.rst deleted file mode 100644 index 96e3b51a730..00000000000 --- a/doc/build/changelog/unreleased_21/10236.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. change:: - :tags: change, sql - :tickets: 10236 - - The ``.c`` and ``.columns`` attributes on the :class:`.Select` and - :class:`.TextualSelect` constructs, which are not instances of - :class:`.FromClause`, have been removed completely, in addition to the - ``.select()`` method as well as other codepaths which would implicitly - generate a subquery from a :class:`.Select` without the need to explicitly - call the :meth:`.Select.subquery` method. - - In the case of ``.c`` and ``.columns``, these attributes were never useful - in practice and have caused a great deal of confusion, hence were - deprecated back in version 1.4, and have emitted warnings since that - version. Accessing the columns that are specific to a :class:`.Select` - construct is done via the :attr:`.Select.selected_columns` attribute, which - was added in version 1.4 to suit the use case that users often expected - ``.c`` to accomplish. In the larger sense, implicit production of - subqueries works against SQLAlchemy's modern practice of making SQL - structure as explicit as possible. - - Note that this is **not related** to the usual :attr:`.FromClause.c` and - :attr:`.FromClause.columns` attributes, common to objects such as - :class:`.Table` and :class:`.Subquery`, which are unaffected by this - change. - - .. seealso:: - - :ref:`change_4617` - original notes from SQLAlchemy 1.4 - diff --git a/doc/build/changelog/unreleased_21/10247.rst b/doc/build/changelog/unreleased_21/10247.rst deleted file mode 100644 index 1024693cabe..00000000000 --- a/doc/build/changelog/unreleased_21/10247.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. change:: - :tags: schema - :tickets: 10247 - - Deprecate Oracle only parameters :paramref:`_schema.Sequence.order`, - :paramref:`_schema.Identity.order` and :paramref:`_schema.Identity.on_null`. - They should be configured using the dialect kwargs ``oracle_order`` and - ``oracle_on_null``. diff --git a/doc/build/changelog/unreleased_21/10296.rst b/doc/build/changelog/unreleased_21/10296.rst deleted file mode 100644 index c58eb856602..00000000000 --- a/doc/build/changelog/unreleased_21/10296.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. change:: - :tags: change, asyncio - :tickets: 10296 - - Added an initialize step to the import of - ``sqlalchemy.ext.asyncio`` so that ``greenlet`` will - be imported only when the asyncio extension is first imported. - Alternatively, the ``greenlet`` library is still imported lazily on - first use to support use case that don't make direct use of the - SQLAlchemy asyncio extension. diff --git a/doc/build/changelog/unreleased_21/10339.rst b/doc/build/changelog/unreleased_21/10339.rst deleted file mode 100644 index 91fe20dad39..00000000000 --- a/doc/build/changelog/unreleased_21/10339.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. change:: - :tags: usecase, mariadb - :tickets: 10339 - - Modified the MariaDB dialect so that when using the :class:`_sqltypes.Uuid` - datatype with MariaDB >= 10.7, leaving the - :paramref:`_sqltypes.Uuid.native_uuid` parameter at its default of True, - the native ``UUID`` datatype will be rendered in DDL and used for database - communication, rather than ``CHAR(32)`` (the non-native UUID type) as was - the case previously. This is a behavioral change since 2.0, where the - generic :class:`_sqltypes.Uuid` datatype delivered ``CHAR(32)`` for all - MySQL and MariaDB variants. Support for all major DBAPIs is implemented - including support for less common "insertmanyvalues" scenarios where UUID - values are generated in different ways for primary keys. Thanks much to - Volodymyr Kochetkov for delivering the PR. - diff --git a/doc/build/changelog/unreleased_21/10357.rst b/doc/build/changelog/unreleased_21/10357.rst deleted file mode 100644 index 22772678fa1..00000000000 --- a/doc/build/changelog/unreleased_21/10357.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. change:: - :tags: change, installation - :tickets: 10357, 12029 - - Python 3.9 or above is now required; support for Python 3.8 and 3.7 is - dropped as these versions are EOL. diff --git a/doc/build/changelog/unreleased_21/10415.rst b/doc/build/changelog/unreleased_21/10415.rst deleted file mode 100644 index ee96c2df5ae..00000000000 --- a/doc/build/changelog/unreleased_21/10415.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. change:: - :tags: change, asyncio - :tickets: 10415 - - Adapted all asyncio dialects, including aiosqlite, aiomysql, asyncmy, - psycopg, asyncpg to use the generic asyncio connection adapter first added - in :ticket:`6521` for the aioodbc DBAPI, allowing these dialects to take - advantage of a common framework. diff --git a/doc/build/changelog/unreleased_21/10497.rst b/doc/build/changelog/unreleased_21/10497.rst deleted file mode 100644 index f3e4a91c524..00000000000 --- a/doc/build/changelog/unreleased_21/10497.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. change:: - :tags: change, orm - :tickets: 10497 - - A sweep through class and function names in the ORM renames many classes - and functions that have no intent of public visibility to be underscored. - This is to reduce ambiguity as to which APIs are intended to be targeted by - third party applications and extensions. Third parties are encouraged to - propose new public APIs in Discussions to the extent they are needed to - replace those that have been clarified as private. diff --git a/doc/build/changelog/unreleased_21/10500.rst b/doc/build/changelog/unreleased_21/10500.rst deleted file mode 100644 index 6a8c62cc767..00000000000 --- a/doc/build/changelog/unreleased_21/10500.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. change:: - :tags: change, orm - :tickets: 10500 - - The ``first_init`` ORM event has been removed. This event was - non-functional throughout the 1.4 and 2.0 series and could not be invoked - without raising an internal error, so it is not expected that there is any - real-world use of this event hook. diff --git a/doc/build/changelog/unreleased_21/10564.rst b/doc/build/changelog/unreleased_21/10564.rst deleted file mode 100644 index cbff04a0d1b..00000000000 --- a/doc/build/changelog/unreleased_21/10564.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. change:: - :tags: bug, orm - :tickets: 10564 - - The :paramref:`_orm.relationship.secondary` parameter no longer uses Python - ``eval()`` to evaluate the given string. This parameter when passed a - string should resolve to a table name that's present in the local - :class:`.MetaData` collection only, and never needs to be any kind of - Python expression otherwise. To use a real deferred callable based on a - name that may not be locally present yet, use a lambda instead. diff --git a/doc/build/changelog/unreleased_21/10635.rst b/doc/build/changelog/unreleased_21/10635.rst deleted file mode 100644 index 81fbba97d8b..00000000000 --- a/doc/build/changelog/unreleased_21/10635.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. change:: - :tags: typing, feature - :tickets: 10635 - - The :class:`.Row` object now no longer makes use of an intermediary - ``Tuple`` in order to represent its individual element types; instead, - the individual element types are present directly, via new :pep:`646` - integration, now available in more recent versions of Mypy. Mypy - 1.7 or greater is now required for statements, results and rows - to be correctly typed. Pull request courtesy Yurii Karabas. - - .. seealso:: - - :ref:`change_10635` diff --git a/doc/build/changelog/unreleased_21/10646.rst b/doc/build/changelog/unreleased_21/10646.rst deleted file mode 100644 index 7d82138f98d..00000000000 --- a/doc/build/changelog/unreleased_21/10646.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. change:: - :tags: typing - :tickets: 10646 - - The default implementation of :attr:`_sql.TypeEngine.python_type` now - returns ``object`` instead of ``NotImplementedError``, since that's the - base for all types in Python3. - The ``python_type`` of :class:`_sql.JSON` no longer returns ``dict``, - but instead fallbacks to the generic implementation. diff --git a/doc/build/changelog/unreleased_21/10721.rst b/doc/build/changelog/unreleased_21/10721.rst deleted file mode 100644 index 5ec405748f2..00000000000 --- a/doc/build/changelog/unreleased_21/10721.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. change:: - :tags: change, orm - :tickets: 10721 - - Removed legacy signatures dating back to 0.9 release from the - :meth:`_orm.SessionEvents.after_bulk_update` and - :meth:`_orm.SessionEvents.after_bulk_delete`. diff --git a/doc/build/changelog/unreleased_21/10788.rst b/doc/build/changelog/unreleased_21/10788.rst deleted file mode 100644 index 63f6af86e6d..00000000000 --- a/doc/build/changelog/unreleased_21/10788.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. change:: - :tags: bug, sql - :tickets: 10788 - - Fixed issue in name normalization (e.g. "uppercase" backends like Oracle) - where using a :class:`.TextualSelect` would not properly maintain as - uppercase column names that were quoted as uppercase, even though - the :class:`.TextualSelect` includes a :class:`.Column` that explicitly - holds this uppercase name. diff --git a/doc/build/changelog/unreleased_21/10789.rst b/doc/build/changelog/unreleased_21/10789.rst deleted file mode 100644 index af3b301b545..00000000000 --- a/doc/build/changelog/unreleased_21/10789.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. change:: - :tags: usecase, engine - :tickets: 10789 - - Added new execution option - :paramref:`_engine.Connection.execution_options.driver_column_names`. This - option disables the "name normalize" step that takes place against the - DBAPI ``cursor.description`` for uppercase-default backends like Oracle, - and will cause the keys of a result set (e.g. named tuple names, dictionary - keys in :attr:`.Row._mapping`, etc.) to be exactly what was delivered in - cursor.description. This is mostly useful for plain textual statements - using :func:`_sql.text` or :meth:`_engine.Connection.exec_driver_sql`. diff --git a/doc/build/changelog/unreleased_21/10816.rst b/doc/build/changelog/unreleased_21/10816.rst deleted file mode 100644 index 1b037bcb31e..00000000000 --- a/doc/build/changelog/unreleased_21/10816.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. change:: - :tags: usecase, orm - :tickets: 10816 - - The :paramref:`_orm.Session.flush.objects` parameter is now - deprecated. \ No newline at end of file diff --git a/doc/build/changelog/unreleased_21/11045.rst b/doc/build/changelog/unreleased_21/11045.rst deleted file mode 100644 index 8788d33d790..00000000000 --- a/doc/build/changelog/unreleased_21/11045.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. change:: - :tags: orm - :tickets: 11045 - - The :func:`_orm.noload` relationship loader option and related - ``lazy='noload'`` setting is deprecated and will be removed in a future - release. This option was originally intended for custom loader patterns - that are no longer applicable in modern SQLAlchemy. diff --git a/doc/build/changelog/unreleased_21/11163.rst b/doc/build/changelog/unreleased_21/11163.rst deleted file mode 100644 index c8355714587..00000000000 --- a/doc/build/changelog/unreleased_21/11163.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. change:: - :tags: orm - :tickets: 11163 - - Ignore :paramref:`_orm.Session.join_transaction_mode` in all cases when - the bind provided to the :class:`_orm.Session` is an - :class:`_engine.Engine`. - Previously if an event that executed before the session logic, - like :meth:`_engine.ConnectionEvents.engine_connect`, - left the connection with an active transaction, the - :paramref:`_orm.Session.join_transaction_mode` behavior took - place, leading to a surprising behavior. diff --git a/doc/build/changelog/unreleased_21/11234.rst b/doc/build/changelog/unreleased_21/11234.rst deleted file mode 100644 index f168714e891..00000000000 --- a/doc/build/changelog/unreleased_21/11234.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. change:: - :tags: bug, engine - :tickets: 11234 - - Adjusted URL parsing and stringification to apply url quoting to the - "database" portion of the URL. This allows a URL where the "database" - portion includes special characters such as question marks to be - accommodated. - - .. seealso:: - - :ref:`change_11234` diff --git a/doc/build/changelog/unreleased_21/11250.rst b/doc/build/changelog/unreleased_21/11250.rst deleted file mode 100644 index ba1fc14b739..00000000000 --- a/doc/build/changelog/unreleased_21/11250.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. change:: - :tags: bug, mssql - :tickets: 11250 - - Fix mssql+pyodbc issue where valid plus signs in an already-unquoted - ``odbc_connect=`` (raw DBAPI) connection string are replaced with spaces. - - The pyodbc connector would unconditionally pass the odbc_connect value - to unquote_plus(), even if it was not required. So, if the (unquoted) - odbc_connect value contained ``PWD=pass+word`` that would get changed to - ``PWD=pass word``, and the login would fail. One workaround was to quote - just the plus sign — ``PWD=pass%2Bword`` — which would then get unquoted - to ``PWD=pass+word``. diff --git a/doc/build/changelog/unreleased_21/11349.rst b/doc/build/changelog/unreleased_21/11349.rst deleted file mode 100644 index 244713e9e3f..00000000000 --- a/doc/build/changelog/unreleased_21/11349.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. change:: - :tags: bug, orm - :tickets: 11349 - - Revised the set "binary" operators for the association proxy ``set()`` - interface to correctly raise ``TypeError`` for invalid use of the ``|``, - ``&``, ``^``, and ``-`` operators, as well as the in-place mutation - versions of these methods, to match the behavior of standard Python - ``set()`` as well as SQLAlchemy ORM's "intstrumented" set implementation. - - diff --git a/doc/build/changelog/unreleased_21/11450.rst b/doc/build/changelog/unreleased_21/11450.rst new file mode 100644 index 00000000000..371f86dec18 --- /dev/null +++ b/doc/build/changelog/unreleased_21/11450.rst @@ -0,0 +1,7 @@ +.. change:: + :tags: feature, orm + :tickets: 11450 + + Added :paramref:`.selectinload.chunksize` parameter to :func`.selectinload` + allowing users to configure the number of primary keys sent per IN clause + when loading reltaionships. Pull request courtesy bekapono. diff --git a/doc/build/changelog/unreleased_21/11515.rst b/doc/build/changelog/unreleased_21/11515.rst deleted file mode 100644 index 8d551a078db..00000000000 --- a/doc/build/changelog/unreleased_21/11515.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. change:: - :tags: bug, sql - :tickets: 11515 - - Enhanced the caching structure of the :paramref:`_expression.over.rows` - and :paramref:`_expression.over.range` so that different numerical - values for the rows / - range fields are cached on the same cache key, to the extent that the - underlying SQL does not actually change (i.e. "unbounded", "current row", - negative/positive status will still change the cache key). This prevents - the use of many different numerical range/rows value for a query that is - otherwise identical from filling up the SQL cache. - - Note that the semi-private compiler method ``_format_frame_clause()`` - is removed by this fix, replaced with a new method - ``visit_frame_clause()``. Third party dialects which may have referred - to this method will need to change the name and revise the approach to - rendering the correct SQL for that dialect. - diff --git a/doc/build/changelog/unreleased_21/11776.rst b/doc/build/changelog/unreleased_21/11776.rst deleted file mode 100644 index 446c5e17173..00000000000 --- a/doc/build/changelog/unreleased_21/11776.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. change:: - :tags: orm, usecase - :tickets: 11776 - - Added the utility method :meth:`_orm.Session.merge_all` and - :meth:`_orm.Session.delete_all` that operate on a collection - of instances. diff --git a/doc/build/changelog/unreleased_21/11811.rst b/doc/build/changelog/unreleased_21/11811.rst deleted file mode 100644 index 34d0683dd9d..00000000000 --- a/doc/build/changelog/unreleased_21/11811.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. change:: - :tags: bug, schema - :tickets: 11811 - - The :class:`.Float` and :class:`.Numeric` types are no longer automatically - considered as auto-incrementing columns when the - :paramref:`_schema.Column.autoincrement` parameter is left at its default - of ``"auto"`` on a :class:`_schema.Column` that is part of the primary key. - When the parameter is set to ``True``, a :class:`.Numeric` type will be - accepted as an auto-incrementing datatype for primary key columns, but only - if its scale is explicitly given as zero; otherwise, an error is raised. - This is a change from 2.0 where all numeric types including floats were - automatically considered as "autoincrement" for primary key columns. diff --git a/doc/build/changelog/unreleased_21/12195.rst b/doc/build/changelog/unreleased_21/12195.rst deleted file mode 100644 index a36d1bc8a87..00000000000 --- a/doc/build/changelog/unreleased_21/12195.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. change:: - :tags: feature, sql - :tickets: 12195 - - Added the ability to create custom SQL constructs that can define new - clauses within SELECT, INSERT, UPDATE, and DELETE statements without - needing to modify the construction or compilation code of of - :class:`.Select`, :class:`.Insert`, :class:`.Update`, or :class:`.Delete` - directly. Support for testing these constructs, including caching support, - is present along with an example test suite. The use case for these - constructs is expected to be third party dialects for analytical SQL - (so-called NewSQL) or other novel styles of database that introduce new - clauses to these statements. A new example suite is included which - illustrates the ``QUALIFY`` SQL construct used by several NewSQL databases - which includes a cachable implementation as well as a test suite. - - .. seealso:: - - :ref:`examples.syntax_extensions` - diff --git a/doc/build/changelog/unreleased_21/12293.rst b/doc/build/changelog/unreleased_21/12293.rst deleted file mode 100644 index 321a0761da1..00000000000 --- a/doc/build/changelog/unreleased_21/12293.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. change:: - :tags: typing, orm - :tickets: 12293 - - Removed the deprecated mypy plugin. - The plugin was non-functional with newer version of mypy and it's no - longer needed with modern SQLAlchemy declarative style. diff --git a/doc/build/changelog/unreleased_21/12395.rst b/doc/build/changelog/unreleased_21/12395.rst deleted file mode 100644 index 8515db06b53..00000000000 --- a/doc/build/changelog/unreleased_21/12395.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. change:: - :tags: bug, orm - :tickets: 12395 - - The behavior of :func:`_orm.with_polymorphic` when used with a single - inheritance mapping has been changed such that its behavior should match as - closely as possible to that of an equivalent joined inheritance mapping. - Specifically this means that the base class specified in the - :func:`_orm.with_polymorphic` construct will be the basemost class that is - loaded, as well as all descendant classes of that basemost class. - The change includes that the descendant classes named will no longer be - exclusively indicated in "WHERE polymorphic_col IN" criteria; instead, the - whole hierarchy starting with the given basemost class will be loaded. If - the query indicates that rows should only be instances of a specific - subclass within the polymorphic hierarchy, an error is raised if an - incompatible superclass is loaded in the result since it cannot be made to - match the requested class; this behavior is the same as what joined - inheritance has done for many years. The change also allows a single result - set to include column-level results from multiple sibling classes at once - which was not previously possible with single table inheritance. diff --git a/doc/build/changelog/unreleased_21/5252.rst b/doc/build/changelog/unreleased_21/5252.rst deleted file mode 100644 index 79d77b4623e..00000000000 --- a/doc/build/changelog/unreleased_21/5252.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. change:: - :tags: change, sql - :tickets: 5252 - - the :class:`.Numeric` and :class:`.Float` SQL types have been separated out - so that :class:`.Float` no longer inherits from :class:`.Numeric`; instead, - they both extend from a common mixin :class:`.NumericCommon`. This - corrects for some architectural shortcomings where numeric and float types - are typically separate, and establishes more consistency with - :class:`.Integer` also being a distinct type. The change should not have - any end-user implications except for code that may be using - ``isinstance()`` to test for the :class:`.Numeric` datatype; third party - dialects which rely upon specific implementation types for numeric and/or - float may also require adjustment to maintain compatibility. diff --git a/doc/build/changelog/unreleased_21/9647.rst b/doc/build/changelog/unreleased_21/9647.rst deleted file mode 100644 index f933b083b3b..00000000000 --- a/doc/build/changelog/unreleased_21/9647.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. change:: - :tags: change, engine - :tickets: 9647 - - An empty sequence passed to any ``execute()`` method now - raised a deprecation warning, since such an executemany - is invalid. - Pull request courtesy of Carlos Sousa. diff --git a/doc/build/changelog/unreleased_21/async_fallback.rst b/doc/build/changelog/unreleased_21/async_fallback.rst deleted file mode 100644 index 44b91d21565..00000000000 --- a/doc/build/changelog/unreleased_21/async_fallback.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. change:: - :tags: change, asyncio - - Removed the compatibility ``async_fallback`` mode for async dialects, - since it's no longer used by SQLAlchemy tests. - Also removed the internal function ``await_fallback()`` and renamed - the internal function ``await_only()`` to ``await_()``. - No change is expected to user code. diff --git a/doc/build/changelog/unreleased_21/mysql_limit.rst b/doc/build/changelog/unreleased_21/mysql_limit.rst deleted file mode 100644 index cf74e97a44c..00000000000 --- a/doc/build/changelog/unreleased_21/mysql_limit.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. change:: - :tags: feature, mysql - - Added new construct :func:`_mysql.limit` which can be applied to any - :func:`_sql.update` or :func:`_sql.delete` to provide the LIMIT keyword to - UPDATE and DELETE. This new construct supersedes the use of the - "mysql_limit" dialect keyword argument. - diff --git a/doc/build/changelog/unreleased_21/pep_621.rst b/doc/build/changelog/unreleased_21/pep_621.rst deleted file mode 100644 index 473c17ee961..00000000000 --- a/doc/build/changelog/unreleased_21/pep_621.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. change:: - :tags: change, setup - - Updated the setup manifest definition to use PEP 621-compliant - pyproject.toml. - Also updated the extra install dependency to comply with PEP-685. - Thanks for the help of Matt Oberle and KOLANICH on this change. diff --git a/doc/build/changelog/whatsnew_20.rst b/doc/build/changelog/whatsnew_20.rst index f7c2b74f031..8603fcb3039 100644 --- a/doc/build/changelog/whatsnew_20.rst +++ b/doc/build/changelog/whatsnew_20.rst @@ -682,7 +682,7 @@ example below adds additional ``Annotated`` types in addition to our Above, columns that are mapped with ``Mapped[str50]``, ``Mapped[intpk]``, or ``Mapped[user_fk]`` draw from both the :paramref:`_orm.registry.type_annotation_map` as well as the -``Annotated`` construct directly in order to re-use pre-established typing +``Annotated`` construct directly in order to reuse pre-established typing and column configurations. Optional step - turn mapped classes into dataclasses_ diff --git a/doc/build/conf.py b/doc/build/conf.py index d667781e17e..c5f8862e48e 100644 --- a/doc/build/conf.py +++ b/doc/build/conf.py @@ -40,7 +40,7 @@ "sphinx_paramlinks", "sphinx_copybutton", ] -needs_extensions = {"zzzeeksphinx": "1.2.1"} +needs_extensions = {"zzzeeksphinx": "1.6.1"} # Add any paths that contain templates here, relative to this directory. # not sure why abspath() is needed here, some users @@ -167,11 +167,6 @@ "sqlalchemy.orm.util": "sqlalchemy.orm", } -autodocmods_convert_modname_w_class = { - ("sqlalchemy.engine.interfaces", "Connectable"): "sqlalchemy.engine", - ("sqlalchemy.sql.base", "DialectKWArgs"): "sqlalchemy.sql.base", -} - # on the referencing side, a newer zzzeeksphinx extension # applies shorthand symbols to references so that we can have short # names that are still using absolute references. @@ -233,7 +228,7 @@ # General information about the project. project = "SQLAlchemy" -copyright = "2007-2025, the SQLAlchemy authors and contributors" # noqa +copyright = "2007-2026, the SQLAlchemy authors and contributors" # noqa # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -242,9 +237,9 @@ # The short X.Y version. version = "2.1" # The full version, including alpha/beta/rc tags. -release = "2.1.0b1" +release = "2.1.0b2" -release_date = None +release_date = "April 16, 2026" site_base = os.environ.get("RTD_SITE_BASE", "https://www.sqlalchemy.org") site_adapter_template = "docs_adapter.mako" @@ -482,3 +477,17 @@ # Allow duplicate toc entries. # epub_tocdup = True + + +def setup(app): # noqa: U100 + """Sphinx setup hook to configure documentation build.""" + + # delete class attributes with a value, where the value has ``__doc__`` + # defined, but we want to see only the docstring under the attribute + # itself. + try: + from sqlalchemy.ext.asyncio import AsyncSession + except ImportError: + pass + else: + del AsyncSession.sync_session_class diff --git a/doc/build/copyright.rst b/doc/build/copyright.rst index 54535474c42..6f41046769e 100644 --- a/doc/build/copyright.rst +++ b/doc/build/copyright.rst @@ -6,7 +6,7 @@ Appendix: Copyright This is the MIT license: ``_ -Copyright (c) 2005-2025 Michael Bayer and contributors. +Copyright (c) 2005-2026 Michael Bayer and contributors. SQLAlchemy is a trademark of Michael Bayer. Permission is hereby granted, free of charge, to any person obtaining a copy of this diff --git a/doc/build/core/connections.rst b/doc/build/core/connections.rst index 030d41cd3b3..6b634ab3bbf 100644 --- a/doc/build/core/connections.rst +++ b/doc/build/core/connections.rst @@ -69,7 +69,7 @@ When the :class:`_engine.Connection` is closed at the end of the ``with:`` block referenced DBAPI connection is :term:`released` to the connection pool. From the perspective of the database itself, the connection pool will not actually "close" the connection assuming the pool has room to store this connection for -the next use. When the connection is returned to the pool for re-use, the +the next use. When the connection is returned to the pool for reuse, the pooling mechanism issues a ``rollback()`` call on the DBAPI connection so that any transactional state or locks are removed (this is known as :ref:`pool_reset_on_return`), and the connection is ready for its next use. @@ -285,7 +285,7 @@ that loses not only "read committed" but also loses atomicity. :ref:`dbapi_autocommit_understanding`, that "autocommit" isolation level like any other isolation level does **not** affect the "transactional" behavior of the :class:`_engine.Connection` object, which continues to call upon DBAPI - ``.commit()`` and ``.rollback()`` methods (they just have no effect under + ``.commit()`` and ``.rollback()`` methods (they just have no net effect under autocommit), and for which the ``.begin()`` method assumes the DBAPI will start a transaction implicitly (which means that SQLAlchemy's "begin" **does not change autocommit mode**). @@ -340,6 +340,8 @@ begin a transaction:: set at this level. This because the option must be set on a DBAPI connection on a per-transaction basis. +.. _dbapi_autocommit_engine: + Setting Isolation Level or DBAPI Autocommit for an Engine ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -358,14 +360,20 @@ With the above setting, each new DBAPI connection the moment it's created will be set to use a ``"REPEATABLE READ"`` isolation level setting for all subsequent operations. +.. tip:: + + Prefer to set frequently used isolation levels engine wide as illustrated + above compared to using per-engine or per-connection execution options for + maximum performance. + .. _dbapi_autocommit_multiple: Maintaining Multiple Isolation Levels for a Single Engine ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The isolation level may also be set per engine, with a potentially greater -level of flexibility, using either the -:paramref:`_sa.create_engine.execution_options` parameter to +level of flexibility but with a small per-connection performance overhead, +using either the :paramref:`_sa.create_engine.execution_options` parameter to :func:`_sa.create_engine` or the :meth:`_engine.Engine.execution_options` method, the latter of which will create a copy of the :class:`.Engine` that shares the dialect and connection pool of the original engine, but has its own @@ -408,6 +416,14 @@ copy of the original :class:`_engine.Engine`. Both ``eng`` and The isolation level setting, regardless of which one it is, is unconditionally reverted when a connection is returned to the connection pool. +.. note:: + + The execution options approach, whether used engine wide or per connection, + incurs a small performance penalty as isolation level instructions + are sent on connection acquire as well as connection release. Consider + the engine-wide isolation setting at :ref:`dbapi_autocommit_engine` so + that connections are configured at the target isolation level permanently + as they are pooled. .. seealso:: @@ -457,8 +473,9 @@ committed, this rollback has no change on the state of the database. It is important to note that "autocommit" mode persists even when the :meth:`_engine.Connection.begin` method is called; -the DBAPI will not emit any BEGIN to the database, nor will it emit -COMMIT when :meth:`_engine.Connection.commit` is called. This usage is also +the DBAPI will not emit any BEGIN to the database. When +:meth:`_engine.Connection.commit` is called, the DBAPI may still emit the +"COMMIT" instruction, but this is a no-op at the database level. This usage is also not an error scenario, as it is expected that the "autocommit" isolation level may be applied to code that otherwise was written assuming a transactional context; the "isolation level" is, after all, a configurational detail of the transaction @@ -483,7 +500,7 @@ it probably will have no effect due to autocommit mode: INFO sqlalchemy.engine.Engine BEGIN (implicit) ... - INFO sqlalchemy.engine.Engine COMMIT using DBAPI connection.commit(), DBAPI should ignore due to autocommit mode + INFO sqlalchemy.engine.Engine COMMIT using DBAPI connection.commit(), has no effect due to autocommit mode At the same time, even though we are using "DBAPI autocommit", SQLAlchemy's transactional semantics, that is, the in-Python behavior of :meth:`_engine.Connection.begin` @@ -514,6 +531,43 @@ maintain a completely consistent usage pattern with the :class:`_engine.Connection` where DBAPI-autocommit mode can be changed independently without indicating any code changes elsewhere. +.. _dbapi_autocommit_skip_rollback: + +Fully preventing ROLLBACK calls under autocommit +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.0.43 + +A common use case is to use AUTOCOMMIT isolation mode to improve performance, +and this is a particularly common practice on MySQL / MariaDB databases. +When seeking this pattern, it should be preferred to set AUTOCOMMIT engine +wide using the :paramref:`.create_engine.isolation_level` so that pooled +connections are permanently set in autocommit mode. The SQLAlchemy connection +pool as well as the :class:`.Connection` will still seek to invoke the DBAPI +``.rollback()`` method upon connection :term:`release`, as their behavior +remains agnostic of the isolation level that's configured on the connection. +As this rollback still incurs a network round trip under most if not all +DBAPI drivers, this additional network trip may be disabled using the +:paramref:`.create_engine.skip_autocommit_rollback` parameter, which will +apply a rule at the basemost portion of the dialect that invokes DBAPI +``.rollback()`` to first check if the connection is configured in autocommit, +using a method of detection that does not itself incur network overhead:: + + autocommit_engine = create_engine( + "mysql+mysqldb://scott:tiger@mysql80/test", + skip_autocommit_rollback=True, + isolation_level="AUTOCOMMIT", + ) + +When DBAPI connections are returned to the pool by the :class:`.Connection`, +whether the :class:`.Connection` or the pool attempts to reset the +"transaction", the underlying DBAPI ``.rollback()`` method will be blocked +based on a positive test of "autocommit". + +If the dialect in use does not support a no-network means of detecting +autocommit, the dialect will raise ``NotImplementedError`` when a connection +release is attempted. + Changing Between Isolation Levels ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -715,7 +769,7 @@ for further background on using .. versionadded:: 1.4.40 Added :paramref:`_engine.Connection.execution_options.yield_per` as a Core level execution option to conveniently set streaming results, - buffer size, and partition size all at once in a manner that is transferrable + buffer size, and partition size all at once in a manner that is transferable to that of the ORM's similar use case. .. _engine_stream_results_sr: @@ -1210,7 +1264,7 @@ strings that are safe to reuse for many statement invocations, given a particular cache key that is keyed to that SQL string. This means that any literal values in a statement, such as the LIMIT/OFFSET values for a SELECT, can not be hardcoded in the dialect's compilation scheme, as -the compiled string will not be re-usable. SQLAlchemy supports rendered +the compiled string will not be reusable. SQLAlchemy supports rendered bound parameters using the :meth:`_sql.BindParameter.render_literal_execute` method which can be applied to the existing ``Select._limit_clause`` and ``Select._offset_clause`` attributes by a custom compiler, which @@ -1771,10 +1825,10 @@ performance example. including sample performance tests .. tip:: The :term:`insertmanyvalues` feature is a **transparently available** - performance feature which requires no end-user intervention in order for - it to take place as needed. This section describes the architecture - of the feature as well as how to measure its performance and tune its - behavior in order to optimize the speed of bulk INSERT statements, + performance feature which typically requires no end-user intervention in + order for it to take place as needed. This section describes the + architecture of the feature as well as how to measure its performance and + tune its behavior in order to optimize the speed of bulk INSERT statements, particularly as used by the ORM. As more databases have added support for INSERT..RETURNING, SQLAlchemy has @@ -1933,7 +1987,7 @@ degrade to "non-batched" mode which runs individual INSERT statements for each parameter set. For example, on SQL Server when an auto incrementing ``IDENTITY`` column is -used as the primary key, the following SQL form is used: +used as the primary key, the following SQL form is used [#]_: .. sourcecode:: sql @@ -1985,27 +2039,31 @@ of **non-batched** mode when guaranteed RETURNING ordering is requested. .. seealso:: + .. [#] - * Microsoft SQL Server rationale + * Microsoft SQL Server rationale - "INSERT queries that use SELECT with ORDER BY to populate rows guarantees - how identity values are computed but not the order in which the rows are inserted." - https://learn.microsoft.com/en-us/sql/t-sql/statements/insert-transact-sql?view=sql-server-ver16#limitations-and-restrictions + "INSERT queries that use SELECT with ORDER BY to populate rows guarantees + how identity values are computed but not the order in which the rows are inserted." + https://learn.microsoft.com/en-us/sql/t-sql/statements/insert-transact-sql?view=sql-server-ver16#limitations-and-restrictions - * PostgreSQL batched INSERT Discussion + .. [#] - Original description in 2018 https://www.postgresql.org/message-id/29386.1528813619@sss.pgh.pa.us + * PostgreSQL batched INSERT Discussion - Follow up in 2023 - https://www.postgresql.org/message-id/be108555-da2a-4abc-a46b-acbe8b55bd25%40app.fastmail.com + Original description in 2018 https://www.postgresql.org/message-id/29386.1528813619@sss.pgh.pa.us + + Follow up in 2023 - https://www.postgresql.org/message-id/be108555-da2a-4abc-a46b-acbe8b55bd25%40app.fastmail.com .. [#] - * MariaDB AUTO_INCREMENT behavior (using the same InnoDB engine as MySQL): + * MariaDB AUTO_INCREMENT behavior (using the same InnoDB engine as MySQL) + + https://dev.mysql.com/doc/refman/8.0/en/innodb-auto-increment-handling.html - https://dev.mysql.com/doc/refman/8.0/en/innodb-auto-increment-handling.html + https://dba.stackexchange.com/a/72099 - https://dba.stackexchange.com/a/72099 .. _engine_insertmanyvalues_non_batch: @@ -2042,12 +2100,10 @@ also individually passed along to event listeners such as below). - - .. _engine_insertmanyvalues_sentinel_columns: Configuring Sentinel Columns -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In typical cases, the "insertmanyvalues" feature in order to provide INSERT..RETURNING with deterministic row order will automatically determine a @@ -2184,6 +2240,90 @@ In the example above, both "my_table" and "sub_table" will have an additional integer column named "_sentinel" that can be used by the "insertmanyvalues" feature to help optimize bulk inserts used by the ORM. +.. _engine_insertmanyvalues_monotonic_functions: + +Configuring Monotonic Functions such as UUIDV7 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Using a monotonic function such as uuidv7 is supported by the "insertmanyvalues" +feature most easily by establishing the function as a client side callable, +e.g. using Python's built-in ``uuid.uuid7()`` call by providing the callable +to the :paramref:`_schema.Connection.default` parameter:: + + import uuid + + from sqlalchemy import UUID, Integer + + t = Table( + "t", + metadata, + Column("id", UUID, default=uuid.uuid7, primary_key=True), + Column("x", Integer), + ) + +In the above example, SQLAlchemy will invoke Python's ``uuid.uuid7()`` function +to create new primary key identifiers, which will be batchable by the +"insertmanyvalues" feature. + +However, some databases like PostgreSQL provide a server-side function for +uuid7 called ``uuidv7()``; in SQLAlchemy, this would be available from the +:data:`_sql.func` namespace as ``func.uuidv7()``, and may be configured on a +:class:`.Column` using either :paramref:`_schema.Connection.default` to allow +it to be called as needed, or :paramref:`_schema.Connection.server_default` to +establish it as part of the table's DDL. However, for full batched "insertmanyvalues" +behavior including support for sorted RETURNING (as would allow the ORM to +most effectively batch INSERT statements), an additional directive must be +included indicating that the function produces +monotonically increasing values, which is the ``monotonic=True`` directive. +This is illustrated below as a DDL server default using +:paramref:`_schema.Connection.server_default`:: + + from sqlalchemy import func, Integer + + t = Table( + "t", + metadata, + Column("id", UUID, server_default=func.uuidv7(monotonic=True), primary_key=True), + Column("x", Integer), + ) + +Using the above form, a batched INSERT...RETURNING on PostgreSQL with +:paramref:`.UpdateBase.returning.sort_by_parameter_order` set to True will +look like: + +.. sourcecode:: sql + + INSERT INTO t (x) SELECT p0::INTEGER FROM + (VALUES (%(x__0)s, 0), (%(x__1)s, 1), (%(x__2)s, 2), ...) + AS imp_sen(p0, sen_counter) ORDER BY sen_counter + RETURNING t.id, t.id AS id__1 + +Similarly if the function is configured as an ad-hoc server side function +using :paramref:`_schema.Connection.default`:: + + t = Table( + "t", + metadata, + Column("id", UUID, default=func.uuidv7(monotonic=True), primary_key=True), + Column("x", Integer), + ) + +The function will then be rendered in the SQL statement explicitly: + +.. sourcecode:: sql + + INSERT INTO t (id, x) SELECT uuidv7(), p1::INTEGER FROM + (VALUES (%(x__0)s, 0), (%(x__1)s, 1), (%(x__2)s, 2), ...) + AS imp_sen(p1, sen_counter) ORDER BY sen_counter + RETURNING t.id, t.id AS id__1 + +.. versionadded:: 2.1 Added support for explicit monotonic server side functions + using ``monotonic=True`` with any :class:`.Function`. + +.. seealso:: + + :ref:`postgresql_monotonic_functions` + .. _engine_insertmanyvalues_page_size: diff --git a/doc/build/core/constraints.rst b/doc/build/core/constraints.rst index c63ad858e2c..83b7e6eb9d6 100644 --- a/doc/build/core/constraints.rst +++ b/doc/build/core/constraints.rst @@ -308,8 +308,12 @@ arguments. The value is any string which will be output after the appropriate ), ) -Note that these clauses require ``InnoDB`` tables when used with MySQL. -They may also not be supported on other databases. +Note that some backends have special requirements for cascades to function: + +* MySQL / MariaDB - the ``InnoDB`` storage engine should be used (this is + typically the default in modern databases) +* SQLite - constraints are not enabled by default. + See :ref:`sqlite_foreign_keys` .. seealso:: @@ -320,6 +324,12 @@ They may also not be supported on other databases. :ref:`passive_deletes_many_to_many` + :ref:`postgresql_constraint_options` - indicates additional options + available for foreign key cascades such as column lists + + :ref:`sqlite_foreign_keys` - background on enabling foreign key support + with SQLite + .. _schema_unique_constraint: UNIQUE Constraint @@ -645,11 +655,6 @@ name as follows:: `The Importance of Naming Constraints `_ - in the Alembic documentation. - -.. versionadded:: 1.3.0 added multi-column naming tokens such as ``%(column_0_N_name)s``. - Generated names that go beyond the character limit for the target database will be - deterministically truncated. - .. _naming_check_constraints: Naming CHECK Constraints diff --git a/doc/build/core/custom_types.rst b/doc/build/core/custom_types.rst index 5390824dda8..ea930367105 100644 --- a/doc/build/core/custom_types.rst +++ b/doc/build/core/custom_types.rst @@ -15,7 +15,7 @@ A frequent need is to force the "string" version of a type, that is the one rendered in a CREATE TABLE statement or other SQL function like CAST, to be changed. For example, an application may want to force the rendering of ``BINARY`` for all platforms -except for one, in which is wants ``BLOB`` to be rendered. Usage +except for one, in which it wants ``BLOB`` to be rendered. Usage of an existing generic type, in this case :class:`.LargeBinary`, is preferred for most use cases. But to control types more accurately, a compilation directive that is per-dialect @@ -176,7 +176,7 @@ Backend-agnostic GUID Type just as an example of a type decorator that receives and returns python objects. -Receives and returns Python uuid() objects. +Receives and returns Python uuid() objects. Uses the PG UUID type when using PostgreSQL, UNIQUEIDENTIFIER when using MSSQL, CHAR(32) on other backends, storing them in stringified format. The ``GUIDHyphens`` version stores the value with hyphens instead of just the hex @@ -405,16 +405,32 @@ to coerce incoming and outgoing data between an application and persistence form Examples include using database-defined encryption/decryption functions, as well as stored procedures that handle geographic data. -Any :class:`.TypeEngine`, :class:`.UserDefinedType` or :class:`.TypeDecorator` subclass -can include implementations of -:meth:`.TypeEngine.bind_expression` and/or :meth:`.TypeEngine.column_expression`, which -when defined to return a non-``None`` value should return a :class:`_expression.ColumnElement` -expression to be injected into the SQL statement, either surrounding -bound parameters or a column expression. For example, to build a ``Geometry`` -type which will apply the PostGIS function ``ST_GeomFromText`` to all outgoing -values and the function ``ST_AsText`` to all incoming data, we can create -our own subclass of :class:`.UserDefinedType` which provides these methods -in conjunction with :data:`~.sqlalchemy.sql.expression.func`:: +Any :class:`.TypeEngine`, :class:`.UserDefinedType` or :class:`.TypeDecorator` +subclass can include implementations of :meth:`.TypeEngine.bind_expression` +and/or :meth:`.TypeEngine.column_expression`, which when defined to return a +non-``None`` value should return a :class:`_expression.ColumnElement` +expression to be injected into the SQL statement, either surrounding bound +parameters or a column expression. + +.. tip:: As SQL-level result processing features are intended to assist with + coercing data from a SELECT statement into result rows in Python, the + :meth:`.TypeEngine.column_expression` conversion method is applied only to + the **outermost** columns clause in a SELECT; it does **not** apply to + columns rendered inside of subqueries, as these column expressions are not + directly delivered to a result. The expression should not be applied to + both, as this would lead to double-conversion of columns, and the + "outermost" level rather than the "innermost" level is used so that + conversion routines don't interfere with the internal expressions used by + the statement, and so that only data that's outgoing to a result row is + actually subject to conversion, which is consistent with the result + row processing functionality provided by + :meth:`.TypeDecorator.process_result_value`. + +For example, to build a ``Geometry`` type which will apply the PostGIS function +``ST_GeomFromText`` to all outgoing values and the function ``ST_AsText`` to +all incoming data, we can create our own subclass of :class:`.UserDefinedType` +which provides these methods in conjunction with +:data:`~.sqlalchemy.sql.expression.func`:: from sqlalchemy import func from sqlalchemy.types import UserDefinedType diff --git a/doc/build/core/ddl.rst b/doc/build/core/ddl.rst index 1e323dea2b0..831ea7bc4bb 100644 --- a/doc/build/core/ddl.rst +++ b/doc/build/core/ddl.rst @@ -321,24 +321,28 @@ DDL Expression Constructs API .. autoclass:: DDL :members: +.. autoclass:: CheckFirst + :members: + .. autoclass:: _CreateDropBase .. autoclass:: CreateTable :members: - .. autoclass:: DropTable :members: +.. autoclass:: DropView + :members: .. autoclass:: CreateColumn :members: +.. autofunction:: insert_sentinel .. autoclass:: CreateSequence :members: - .. autoclass:: DropSequence :members: diff --git a/doc/build/core/defaults.rst b/doc/build/core/defaults.rst index 586f0531438..e4e5fb78b9d 100644 --- a/doc/build/core/defaults.rst +++ b/doc/build/core/defaults.rst @@ -171,14 +171,6 @@ multi-valued INSERT construct, the subset of parameters that corresponds to the individual VALUES clause is isolated from the full parameter dictionary and returned alone. -.. versionadded:: 1.2 - - Added :meth:`.DefaultExecutionContext.get_current_parameters` method, - which improves upon the still-present - :attr:`.DefaultExecutionContext.current_parameters` attribute - by offering the service of organizing multiple VALUES clauses - into individual parameter dictionaries. - .. _defaults_client_invoked_sql: Client-Invoked SQL Expressions @@ -634,8 +626,6 @@ including the default schema, if any. Computed Columns (GENERATED ALWAYS AS) -------------------------------------- -.. versionadded:: 1.3.11 - The :class:`.Computed` construct allows a :class:`_schema.Column` to be declared in DDL as a "GENERATED ALWAYS AS" column, that is, one which has a value that is computed by the database server. The construct accepts a SQL expression @@ -659,7 +649,7 @@ Example:: Column("perimeter", Integer, Computed("4 * side")), ) -The DDL for the ``square`` table when run on a PostgreSQL 12 backend will look +The DDL for the ``square`` table when run on a PostgreSQL 18 backend [#pgnote]_ will look like: .. sourcecode:: sql @@ -667,8 +657,8 @@ like: CREATE TABLE square ( id SERIAL NOT NULL, side INTEGER, - area INTEGER GENERATED ALWAYS AS (side * side) STORED, - perimeter INTEGER GENERATED ALWAYS AS (4 * side) STORED, + area INTEGER GENERATED ALWAYS AS (side * side), + perimeter INTEGER GENERATED ALWAYS AS (4 * side), PRIMARY KEY (id) ) @@ -702,7 +692,7 @@ eagerly fetched. * MariaDB 10.x series and onwards -* PostgreSQL as of version 12 +* PostgreSQL as of version 12 [#pgnote]_ * Oracle Database - with the caveat that RETURNING does not work correctly with UPDATE (a warning will be emitted to this effect when the UPDATE..RETURNING @@ -721,7 +711,10 @@ DDL is emitted to the database. .. seealso:: - :class:`.Computed` + :class:`.Computed` - produces a GENERATED ALWAYS AS phrase for :class:`.Column` + + .. [#pgnote] :ref:`postgresql_computed_column_notes` - notes for GENERATED ALWAYS AS + on PostgreSQL including behavioral changes as of PostgreSQL 18 .. _identity_ddl: diff --git a/doc/build/core/dml.rst b/doc/build/core/dml.rst index 1724dd6985c..a316aa71d60 100644 --- a/doc/build/core/dml.rst +++ b/doc/build/core/dml.rst @@ -32,10 +32,18 @@ Class documentation for the constructors listed at .. automethod:: Delete.where + .. automethod:: Delete.filter + + .. automethod:: Delete.filter_by + .. automethod:: Delete.with_dialect_options .. automethod:: Delete.returning + .. automethod:: Delete.ext + + .. automethod:: Delete.apply_syntax_extension_point + .. autoclass:: Insert :members: @@ -45,6 +53,10 @@ Class documentation for the constructors listed at .. automethod:: Insert.returning + .. automethod:: Insert.ext + + .. automethod:: Insert.apply_syntax_extension_point + .. autoclass:: Update :members: @@ -52,10 +64,18 @@ Class documentation for the constructors listed at .. automethod:: Update.where + .. automethod:: Update.filter + + .. automethod:: Update.filter_by + .. automethod:: Update.with_dialect_options .. automethod:: Update.values + .. automethod:: Update.ext + + .. automethod:: Update.apply_syntax_extension_point + .. autoclass:: sqlalchemy.sql.expression.UpdateBase :members: diff --git a/doc/build/core/engines.rst b/doc/build/core/engines.rst index 8ac57cdaaf3..838e126a718 100644 --- a/doc/build/core/engines.rst +++ b/doc/build/core/engines.rst @@ -596,7 +596,7 @@ using the :paramref:`_sa.create_engine.pool_logging_name` parameters with :func:`sqlalchemy.create_engine`; the name will be appended to existing class-qualified logging name. This use is recommended for applications that -make use of multiple global :class:`.Engine` instances simultaenously, so +make use of multiple global :class:`.Engine` instances simultaneously, so that they may be distinguished in logging:: >>> import logging diff --git a/doc/build/core/functions.rst b/doc/build/core/functions.rst index 9771ffeedd9..26c59a0bdda 100644 --- a/doc/build/core/functions.rst +++ b/doc/build/core/functions.rst @@ -124,6 +124,9 @@ return types are in use. .. autoclass:: percentile_disc :no-members: +.. autoclass:: pow + :no-members: + .. autoclass:: random :no-members: diff --git a/doc/build/core/internals.rst b/doc/build/core/internals.rst index 5146ef4af43..eeb2800fdc6 100644 --- a/doc/build/core/internals.rst +++ b/doc/build/core/internals.rst @@ -39,7 +39,6 @@ Some key internal constructs are listed here. .. autoclass:: sqlalchemy.engine.default.DefaultExecutionContext :members: - .. autoclass:: sqlalchemy.engine.ExecutionContext :members: diff --git a/doc/build/core/metadata.rst b/doc/build/core/metadata.rst index 318509bbdac..9dd5e99af67 100644 --- a/doc/build/core/metadata.rst +++ b/doc/build/core/metadata.rst @@ -153,6 +153,7 @@ table include:: ``employees.c["some column"]``. See :class:`_sql.ColumnCollection` for further information. +.. _metadata_creating_and_dropping: Creating and Dropping Database Tables ------------------------------------- @@ -558,9 +559,309 @@ The schema feature of SQLAlchemy interacts with the table reflection feature introduced at :ref:`metadata_reflection_toplevel`. See the section :ref:`metadata_reflection_schemas` for additional details on how this works. +.. _metadata_alt_create_forms: + +Alternate CREATE TABLE forms: CREATE VIEW, CREATE TABLE AS +---------------------------------------------------------- + +.. versionadded:: 2.1 SQLAlchemy 2.1 introduces new table creation DDL + sequences CREATE VIEW and CREATE TABLE AS, both of which create a + table or table-like object derived from a SELECT statement + +The :meth:`.MetaData.create_all` sequence discussed at +:ref:`metadata_creating_and_dropping` makes use of a :class:`.DDL` construct +called :class:`.CreateTable` in order to emit the actual ``CREATE TABLE`` +statement. SQLAlchemy 2.1 features additional DDL constructs that can create +tables and views from SELECT statements: :class:`.CreateTableAs` and +:class:`.CreateView`. Both classes are constructed with a :func:`_sql.select` +object that serves as the source of data. Once constructed, they each provide +access to a dynamically-generated :class:`.Table` object that contains the +correct name and :class:`.Column` configuration; this :class:`.Table` can +then be used in subsequent :func:`_sql.select` statements to query the new +table or view. To emit the actual ``CREATE TABLE AS`` or ``CREATE VIEW`` +statement to a database, the :class:`.CreateTableAs` or :class:`.CreateView` +objects may be invoked directly via :meth:`.Connection.execute`, or they +will be invoked automatically via the :meth:`.Table.create` method or +:meth:`.MetaData.create_all` if a :class:`.MetaData` is provided to the +constructor. + + +.. _metadata_create_view: + +Using :class:`.CreateView` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The :class:`.CreateView` construct provides support for the ``CREATE VIEW`` +DDL construct, which allows the creation of database views that represent +the result of a SELECT statement. Unlike a table, a view does not store data +directly; instead, it dynamically evaluates the underlying SELECT query +whenever the view is accessed. A compatible SQL syntax is supported by all +included SQLAlchemy backends. + +A :class:`.CreateView` expression may be produced from a +:func:`_sql.select` created against any combinations of tables:: + + >>> from sqlalchemy.sql.ddl import CreateView + >>> select_stmt = select(user.c.user_id, user.c.user_name).where(user.c.status == "active") + >>> create_view = CreateView(select_stmt, "active_users") + +Stringifying this construct illustrates the ``CREATE VIEW`` syntax:: + + >>> print(create_view) + CREATE VIEW active_users AS SELECT "user".user_id, "user".user_name + FROM "user" + WHERE "user".status = 'active' + +A :class:`.Table` object corresponding to the structure of the view that would +be created can be accessed via the :attr:`.CreateView.table` attribute as soon +as the object is constructed. New Core :func:`_sql.select` +objects can use this :class:`.Table` like any other selectable:: + + >>> view_stmt = select(create_view.table).where(create_view.table.c.user_id > 5) + >>> print(view_stmt) + SELECT active_users.user_id, active_users.user_name + FROM active_users + WHERE active_users.user_id > :user_id_1 + +The DDL for :class:`.CreateView` may be executed in a database either +by calling standard :meth:`.Table.create` or :meth:`.MetaData.create_all` +methods, or by executing the construct directly: -Backend-Specific Options ------------------------- +.. sourcecode:: pycon+sql + + >>> with engine.begin() as connection: + ... connection.execute(create_view) + {opensql}BEGIN (implicit) + CREATE VIEW active_users AS SELECT user.user_id, user.user_name + FROM user + WHERE user.status = 'active' + COMMIT + +The database now has a new view ``active_users`` which will dynamically +evaluate the SELECT statement whenever the view is queried. + +:class:`.CreateView` interacts with a :class:`.MetaData` collection; an +explicit :class:`.MetaData` may be passed using the +:paramref:`.CreateView.metadata` parameter, where operations like +:meth:`.MetaData.create_all` and :meth:`.MetaData.drop_all` may be used to +emit a CREATE / DROP DDL within larger DDL sequences. :class:`.CreateView` +includes itself in the new :class:`.Table` via the :meth:`.Table.set_creator_ddl` +method and also applies :class:`.DropView` to the :meth:`.Table.set_dropper_ddl` +elements, so that ``CREATE VIEW`` and ``DROP VIEW`` will be emitted for the +:class:`.Table`: + +.. sourcecode:: pycon+sql + + >>> create_view = CreateView(select_stmt, "active_users", metadata=metadata_obj) + >>> metadata_obj.create_all(engine) + {opensql}BEGIN (implicit) + PRAGMA main.table_info("active_users") + ... + CREATE VIEW active_users AS SELECT user.user_id, user.user_name + FROM user + WHERE user.status = 'active' + COMMIT + +DROP may be emitted for this view alone using :meth:`.Table.drop` +against :attr:`.CreateView.table`, just like it would be used for +any other table; the :class:`.DropView` DDL construct will be invoked: + +.. sourcecode:: pycon+sql + + >>> create_view.table.drop(engine) + {opensql}DROP VIEW active_users + COMMIT + +:class:`.CreateView` supports optional flags such as ``TEMPORARY``, +``OR REPLACE``, and ``MATERIALIZED`` where supported by the target +database:: + + >>> # Create a view with OR REPLACE + >>> stmt = CreateView( + ... select(user.c.user_id, user.c.user_name), + ... "user_snapshot", + ... or_replace=True, + ... ) + >>> print(stmt) + CREATE OR REPLACE VIEW user_snapshot AS SELECT user.user_id, user.user_name + FROM user + +The ``OR REPLACE`` clause renders in all forms, including a simple use +of :meth:`.Table.create`, which does not use a "checkfirst" query by default:: + + >>> stmt.table.create(engine) + BEGIN (implicit) + CREATE OR REPLACE VIEW user_snapshot AS SELECT user.user_id, user.user_name + FROM user + COMMIT + +.. tip:: + + The exact phrase ``OR REPLACE`` is supported by PostgreSQL, Oracle + Database, MySQL and MariaDB. When :class:`.CreateView` with + :paramref:`.CreateView.or_replace` is used on Microsoft SQL Server, the + equivalent keywords ``OR ALTER`` is emitted instead. The remaining + SQLAlchemy-native dialect, SQLite, remains an outlier - for SQLite, the + dialect-specific parameter ``sqlite_if_not_exists`` may be used to create a + view with a check for already existing:: + + stmt = CreateView( + select(user.c.user_id, user.c.user_name), + "user_snapshot", + sqlite_if_not_exists=True, + ) + + ``sqlite_if_not_exists`` is separate from :paramref:`.CreateView.or_replace` + since it has a different meaning, leaving an existing view unmodified + whereas :paramref:`.CreateView.or_replace` will update the definition of + an existing view. + +The ``MATERIALIZED`` keyword may be emitted by specifying :paramref:`.CreateView.materialized`:: + + >>> stmt = CreateView( + ... select(user.c.user_id, user.c.user_name), + ... "user_snapshot", + ... materialized=True, + ... ) + >>> print(stmt) + CREATE MATERIALIZED VIEW user_snapshot AS SELECT user.user_id, user.user_name + FROM user + +Materialized views store the query results physically and can offer performance +benefits for complex queries, though they typically need to be refreshed periodically +using database-specific commands. The Oracle and PostgreSQL backends currently +support ``MATERIALIZED``; however it may be the case that ``MATERIALIZED`` cannot be +combined with ``OR REPLACE``. + + +.. _metadata_create_table_as: + +Using :class:`.CreateTableAs` or :meth:`.Select.into` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + +The :class:`.CreateTableAs` construct, along with a complementing method +:meth:`.Select.into`, provides support for the "CREATE TABLE AS" / "SELECT INTO" +DDL constructs, which allows the creation of new tables in the database that +represent the contents of an arbitrary SELECT statement. A compatible SQL +syntax is supported by all included SQLAlchemy backends. + +A :class:`_schema.CreateTableAs` expression may be produced from a +:func:`_sql.select` created against any combinations of tables:: + + >>> from sqlalchemy import select, CreateTableAs + >>> select_stmt = select(user.c.user_id, user.c.user_name).where( + ... user.c.user_name.like("sponge%") + ... ) + >>> create_table_as = CreateTableAs(select_stmt, "spongebob_users") + +The equivalent :meth:`.Select.into` method may also be used; this creates +a :class:`.CreateTableAs` construct as well:: + + >>> create_table_as = select_stmt.into("spongebob_users") + +Stringifying this construct on most backends illustrates the ``CREATE TABLE AS`` syntax:: + + >>> print(create_table_as) + CREATE TABLE spongebob_users AS SELECT "user".user_id, "user".user_name + FROM "user" + WHERE "user".user_name LIKE 'sponge%' + +On Microsoft SQL Server, SELECT INTO is generated instead:: + + >>> from sqlalchemy.dialects import mssql + >>> print(create_table_as.compile(dialect=mssql.dialect())) + SELECT [user].user_id, [user].user_name INTO spongebob_users + FROM [user] + WHERE [user].user_name LIKE 'sponge%' + +A :class:`.Table` object corresponding to the structure of the view that would +be created can be accessed via the :attr:`.CreateTableAs.table` attribute as soon +as the object is constructed. New Core :func:`_sql.select` +objects can use this :class:`.Table` like any other selectable:: + + >>> ctas_stmt = select(create_table_as.table).where(create_table_as.table.c.user_id > 5) + >>> print(ctas_stmt) + SELECT spongebob_users.user_id, spongebob_users.user_name + FROM spongebob_users + WHERE spongebob_users.user_id > :user_id_1 + +The DDL for :class:`.CreateTableAs` may be executed in a database either +by calling standard :meth:`.Table.create` or :meth:`.MetaData.create_all` +methods, or by executing the construct directly: + +.. sourcecode:: pycon+sql + + >>> with engine.begin() as connection: + ... connection.execute(create_table_as) + {opensql}BEGIN (implicit) + CREATE TABLE spongebob_users AS SELECT user.user_id, user.user_name + FROM user + WHERE user.user_name LIKE 'sponge%' + COMMIT + +The database now has a new table ``spongebob_users`` which contains all the +columns and rows that would be returned by the SELECT statement. This is a +real table in the database that will remain until we drop it (unless it's a +temporary table that automatically drops, or if transactional DDL is rolled +back). + +Like :class:`.CreateView`, :class:`.CreateTableAs` interacts +with a :class:`.MetaData` collection; an explicit :class:`.MetaData` may be +passed using the :paramref:`.CreateTableAs.metadata` parameter, where +operations like :meth:`.MetaData.create_all` and :meth:`.MetaData.drop_all` may +be used to emit a CREATE / DROP DDL within larger DDL sequences. :class:`.CreateView` +includes itself in the new :class:`.Table` via the :meth:`.Table.set_creator_ddl` +method, so that ``CREATE TABLE AS `` will be emitted for the +:class:`.Table`: + +.. sourcecode:: pycon+sql + + >>> create_table_as = CreateTableAs(select_stmt, "spongebob_users", metadata=metadata_obj) + >>> metadata_obj.create_all(engine) + {opensql}BEGIN (implicit) + PRAGMA main.table_info("spongebob_users") + ... + CREATE TABLE spongebob_users AS SELECT user.user_id, user.user_name + FROM user + WHERE user.user_name LIKE 'sponge%' + COMMIT + + +DROP may be emitted for this table alone using :meth:`.Table.drop` +against :attr:`.CreateTableAs.table`, just like it would be used for +any other table: + +.. sourcecode:: pycon+sql + + >>> create_table_as.table.drop(engine) + {opensql}DROP TABLE spongebob_users + COMMIT + +:class:`.CreateTableAs` and :meth:`.Select.into` both support optional flags +such as ``TEMPORARY`` and ``IF NOT EXISTS`` where supported by the target +database:: + + >>> # Create a temporary table with IF NOT EXISTS + >>> stmt = select(user.c.user_id, user.c.user_name).into( + ... "temp_snapshot", temporary=True, if_not_exists=True + ... ) + >>> print(stmt) + CREATE TEMPORARY TABLE IF NOT EXISTS temp_snapshot AS SELECT user_account.id, user_account.name + FROM user_account + +The ``IF NOT EXISTS`` clause renders in all forms, including a simple use +of :meth:`.Table.create`, which does not use a "checkfirst" query by default:: + + >>> stmt.table.create(engine) + BEGIN (implicit) + CREATE TEMPORARY TABLE IF NOT EXISTS temp_snapshot AS SELECT user.user_id, user.user_name + FROM user + COMMIT + + +Backend-Specific Options for :class:`.Table` +-------------------------------------------- :class:`~sqlalchemy.schema.Table` supports database-specific options. For example, MySQL has different table backend types, including "MyISAM" and @@ -597,6 +898,11 @@ Column, Table, MetaData API :members: :inherited-members: +.. autoclass:: CreateTableAs + :members: + +.. autoclass:: CreateView + :members: .. autoclass:: MetaData :members: @@ -607,8 +913,16 @@ Column, Table, MetaData API .. autoclass:: SchemaItem :members: -.. autofunction:: insert_sentinel - .. autoclass:: Table :members: :inherited-members: + +.. autoclass:: TypedColumns + :members: + +.. autoclass:: Named + :members: + +.. autoclass:: sqlalchemy.sql._annotated_cols.HasRowPos + :special-members: __row_pos__ + diff --git a/doc/build/core/operators.rst b/doc/build/core/operators.rst index 35c25fe75c3..d6493a4d632 100644 --- a/doc/build/core/operators.rst +++ b/doc/build/core/operators.rst @@ -1,5 +1,7 @@ .. highlight:: pycon+sql +.. module:: sqlalchemy.sql.operators + Operator Reference =============================== @@ -425,7 +427,7 @@ behaviors and results on different databases: >>> from sqlalchemy.dialects import postgresql >>> print(column("x").regexp_match("word").compile(dialect=postgresql.dialect())) - {printsql}x ~ %(x_1)s + {printsql}x ~ %(x_1)s::VARCHAR Or MySQL:: @@ -469,7 +471,7 @@ String Alteration REPLACE equivalent for the backends which support it:: >>> print(column("x").regexp_replace("foo", "bar").compile(dialect=postgresql.dialect())) - {printsql}REGEXP_REPLACE(x, %(x_1)s, %(x_2)s) + {printsql}REGEXP_REPLACE(x, %(x_1)s::VARCHAR, %(x_2)s::VARCHAR) .. @@ -636,7 +638,7 @@ boolean operators. >>> from sqlalchemy.dialects import postgresql >>> print(column("x").bitwise_xor(5).compile(dialect=postgresql.dialect())) - x # %(x_1)s + x # %(x_1)s::INTEGER .. @@ -757,6 +759,49 @@ The above conjunction functions :func:`_sql.and_`, :func:`_sql.or_`, .. +.. _operators_parentheses: + +Parentheses and Grouping +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Parenthesization of expressions is rendered based on operator precedence, +not the placement of parentheses in Python code, since there is no means of +detecting parentheses from interpreted Python expressions. So an expression +like:: + + >>> expr = or_( + ... User.name == "squidward", and_(Address.user_id == User.id, User.name == "sandy") + ... ) + +won't include parentheses, because the AND operator takes natural precedence over OR:: + + >>> print(expr) + user_account.name = :name_1 OR address.user_id = user_account.id AND user_account.name = :name_2 + +Whereas this one, where OR would otherwise not be evaluated before the AND, does:: + + >>> expr = and_( + ... Address.user_id == User.id, or_(User.name == "squidward", User.name == "sandy") + ... ) + >>> print(expr) + address.user_id = user_account.id AND (user_account.name = :name_1 OR user_account.name = :name_2) + +The same behavior takes effect for math operators. In the parenthesized +Python expression below, the multiplication operator naturally takes precedence over +the addition operator, therefore the SQL will not include parentheses:: + + >>> print(column("q") + (column("x") * column("y"))) + {printsql}q + x * y{stop} + +Whereas this one, where the addition operator would not otherwise occur before +the multiplication operator, does get parentheses:: + + >>> print(column("q") * (column("x") + column("y"))) + {printsql}q * (x + y){stop} + +More background on this is in the FAQ at :ref:`faq_sql_expression_paren_rules`. + + .. Setup code, not for display >>> conn.close() diff --git a/doc/build/core/pooling.rst b/doc/build/core/pooling.rst index 1a4865ba2b9..991287ea303 100644 --- a/doc/build/core/pooling.rst +++ b/doc/build/core/pooling.rst @@ -6,7 +6,7 @@ Connection Pooling .. module:: sqlalchemy.pool A connection pool is a standard technique used to maintain -long running connections in memory for efficient re-use, +long running connections in memory for efficient reuse, as well as to provide management for the total number of connections an application might use simultaneously. @@ -134,8 +134,14 @@ The pool includes "reset on return" behavior which will call the ``rollback()`` method of the DBAPI connection when the connection is returned to the pool. This is so that any existing transactional state is removed from the connection, which includes not just uncommitted data but table and row locks as -well. For most DBAPIs, the call to ``rollback()`` is inexpensive, and if the -DBAPI has already completed a transaction, the method should be a no-op. +well. For most DBAPIs, the call to ``rollback()`` is relatively inexpensive. + +The "reset on return" feature takes place when a connection is :term:`released` +back to the connection pool. In modern SQLAlchemy, this reset on return +behavior is shared between the :class:`.Connection` and the :class:`.Pool`, +where the :class:`.Connection` itself, if it releases its transaction upon close, +considers ``.rollback()`` to have been called, and instructs the pool to skip +this step. Disabling Reset on Return for non-transactional connections @@ -146,24 +152,39 @@ using a connection that is configured for :ref:`autocommit ` or when using a database that has no ACID capabilities such as the MyISAM engine of MySQL, the reset-on-return behavior can be disabled, which is typically done for -performance reasons. This can be affected by using the +performance reasons. + +As of SQLAlchemy 2.0.43, the :paramref:`.create_engine.skip_autocommit_rollback` +parameter of :func:`.create_engine` provides the most complete means of +preventing ROLLBACK from being emitted while under autocommit mode, as it +blocks the DBAPI ``.rollback()`` method from being called by the dialect +completely:: + + autocommit_engine = create_engine( + "mysql+mysqldb://scott:tiger@mysql80/test", + skip_autocommit_rollback=True, + isolation_level="AUTOCOMMIT", + ) + +Detail on this pattern is at :ref:`dbapi_autocommit_skip_rollback`. + +The :class:`_pool.Pool` itself also has a parameter that can control its +"reset on return" behavior, noting that in modern SQLAlchemy this is not +the only path by which the DBAPI transaction is released, which is the :paramref:`_pool.Pool.reset_on_return` parameter of :class:`_pool.Pool`, which is also available from :func:`_sa.create_engine` as :paramref:`_sa.create_engine.pool_reset_on_return`, passing a value of ``None``. -This is illustrated in the example below, in conjunction with the -:paramref:`.create_engine.isolation_level` parameter setting of -``AUTOCOMMIT``:: +This pattern looks as below:: - non_acid_engine = create_engine( - "mysql://scott:tiger@host/db", + autocommit_engine = create_engine( + "mysql+mysqldb://scott:tiger@mysql80/test", pool_reset_on_return=None, isolation_level="AUTOCOMMIT", ) -The above engine won't actually perform ROLLBACK when connections are returned -to the pool; since AUTOCOMMIT is enabled, the driver will also not perform -any BEGIN operation. - +The above pattern will still see ROLLBACKs occur however as the :class:`.Connection` +object implicitly starts transaction blocks in the SQLAlchemy 2.0 series, +which still emit ROLLBACK independently of the pool's reset sequence. Custom Reset-on-Return Schemes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -566,8 +587,6 @@ handled by the connection pool and replaced with a new connection. Note that the flag only applies to :class:`.QueuePool` use. -.. versionadded:: 1.3 - .. seealso:: :ref:`pool_disconnects` diff --git a/doc/build/core/selectable.rst b/doc/build/core/selectable.rst index e81c88cc494..38d47aa6573 100644 --- a/doc/build/core/selectable.rst +++ b/doc/build/core/selectable.rst @@ -89,6 +89,9 @@ The classes here are generated using the constructors listed at .. autoclass:: Executable :members: +.. autoclass:: ExecutableStatement + :members: + .. autoclass:: Exists :members: @@ -123,7 +126,7 @@ The classes here are generated using the constructors listed at .. autoclass:: Select :members: :inherited-members: ClauseElement - :exclude-members: memoized_attribute, memoized_instancemethod, append_correlation, append_column, append_prefix, append_whereclause, append_having, append_from, append_order_by, append_group_by + :exclude-members: __new__, memoized_attribute, memoized_instancemethod, append_correlation, append_column, append_prefix, append_whereclause, append_having, append_from, append_order_by, append_group_by .. autoclass:: Selectable @@ -133,7 +136,7 @@ The classes here are generated using the constructors listed at .. autoclass:: SelectBase :members: :inherited-members: ClauseElement - :exclude-members: memoized_attribute, memoized_instancemethod + :exclude-members: __new__, memoized_attribute, memoized_instancemethod .. autoclass:: Subquery :members: @@ -154,6 +157,7 @@ The classes here are generated using the constructors listed at .. autoclass:: Values :members: + :inherited-members: ClauseElement, FromClause, HasTraverseInternals, Selectable .. autoclass:: ScalarValues :members: diff --git a/doc/build/core/sqlelement.rst b/doc/build/core/sqlelement.rst index 9481bf5d9f5..047d8594cda 100644 --- a/doc/build/core/sqlelement.rst +++ b/doc/build/core/sqlelement.rst @@ -22,6 +22,8 @@ Column Element Foundational Constructors Standalone functions imported from the ``sqlalchemy`` namespace which are used when building up SQLAlchemy Expression Language constructs. +.. autofunction:: aggregate_order_by + .. autofunction:: and_ .. autofunction:: bindparam @@ -43,6 +45,8 @@ used when building up SQLAlchemy Expression Language constructs. .. autofunction:: false +.. autofunction:: from_dml_column + .. autodata:: func .. autofunction:: lambda_stmt @@ -61,6 +65,8 @@ used when building up SQLAlchemy Expression Language constructs. .. autofunction:: text +.. autofunction:: tstring + .. autofunction:: true .. autofunction:: try_cast @@ -168,12 +174,15 @@ The classes here are generated using the constructors listed at well as ORM-mapped attributes that will have a ``__clause_element__()`` method. +.. autoclass:: AggregateOrderBy + :members: .. autoclass:: ColumnOperators :members: :special-members: :inherited-members: +.. autoclass:: DMLTargetCopy .. autoclass:: Extract :members: @@ -190,17 +199,35 @@ The classes here are generated using the constructors listed at .. autoclass:: Null :members: +.. autoclass:: OperatorClass + :members: + :undoc-members: + .. autoclass:: Operators :members: :special-members: +.. autoclass:: OrderByList + :members: + .. autoclass:: Over :members: +.. autoclass:: FrameClause + :members: + +.. autoclass:: FrameClauseType + :members: + .. autoclass:: SQLColumnExpression .. autoclass:: TextClause :members: + :inherited-members: columns + +.. autoclass:: TString + :members: + :inherited-members: columns .. autoclass:: TryCast :members: diff --git a/doc/build/dialects/index.rst b/doc/build/dialects/index.rst index 9f18cbba22e..809265aefaf 100644 --- a/doc/build/dialects/index.rst +++ b/doc/build/dialects/index.rst @@ -66,6 +66,10 @@ Currently maintained external dialect projects for SQLAlchemy include: +------------------------------------------------+---------------------------------------+ | Amazon Athena | pyathena_ | +------------------------------------------------+---------------------------------------+ +| Amazon Aurora DSQL | aurora-dsql-sqlalchemy_ | ++------------------------------------------------+---------------------------------------+ +| Amazon DynamoDB | pydynamodb_ | ++------------------------------------------------+---------------------------------------+ | Amazon Redshift (via psycopg2) | sqlalchemy-redshift_ | +------------------------------------------------+---------------------------------------+ | Apache Drill | sqlalchemy-drill_ | @@ -86,6 +90,8 @@ Currently maintained external dialect projects for SQLAlchemy include: +------------------------------------------------+---------------------------------------+ | Databricks | databricks_ | +------------------------------------------------+---------------------------------------+ +| Denodo | denodo-sqlalchemy_ | ++------------------------------------------------+---------------------------------------+ | EXASolution | sqlalchemy_exasol_ | +------------------------------------------------+---------------------------------------+ | Elasticsearch (readonly) | elasticsearch-dbapi_ | @@ -116,15 +122,21 @@ Currently maintained external dialect projects for SQLAlchemy include: +------------------------------------------------+---------------------------------------+ | Microsoft SQL Server (via turbodbc) | sqlalchemy-turbodbc_ | +------------------------------------------------+---------------------------------------+ +| Mimer SQL | sqlalchemy-mimer_ | ++------------------------------------------------+---------------------------------------+ | MonetDB | sqlalchemy-monetdb_ | +------------------------------------------------+---------------------------------------+ +| MongoDB | pymongosql_ | ++------------------------------------------------+---------------------------------------+ +| OceanBase | oceanbase-sqlalchemy_ | ++------------------------------------------------+---------------------------------------+ | OpenGauss | openGauss-sqlalchemy_ | +------------------------------------------------+---------------------------------------+ | Rockset | rockset-sqlalchemy_ | +------------------------------------------------+---------------------------------------+ | SAP ASE (fork of former Sybase dialect) | sqlalchemy-sybase_ | +------------------------------------------------+---------------------------------------+ -| SAP Hana [1]_ | sqlalchemy-hana_ | +| SAP HANA | sqlalchemy-hana_ | +------------------------------------------------+---------------------------------------+ | SAP Sybase SQL Anywhere | sqlalchemy-sqlany_ | +------------------------------------------------+---------------------------------------+ @@ -141,7 +153,7 @@ Currently maintained external dialect projects for SQLAlchemy include: .. [1] Supports version 1.3.x only at the moment. -.. _openGauss-sqlalchemy: https://gitee.com/opengauss/openGauss-sqlalchemy +.. _openGauss-sqlalchemy: https://pypi.org/project/opengauss-sqlalchemy .. _rockset-sqlalchemy: https://pypi.org/project/rockset-sqlalchemy .. _sqlalchemy-ingres: https://github.com/ActianCorp/sqlalchemy-ingres .. _nzalchemy: https://pypi.org/project/nzalchemy/ @@ -155,6 +167,7 @@ Currently maintained external dialect projects for SQLAlchemy include: .. _sqlalchemy-solr: https://github.com/aadel/sqlalchemy-solr .. _sqlalchemy_exasol: https://github.com/blue-yonder/sqlalchemy_exasol .. _sqlalchemy-sqlany: https://github.com/sqlanywhere/sqlalchemy-sqlany +.. _sqlalchemy-mimer: https://pypi.org/project/sqlalchemy-mimer .. _sqlalchemy-monetdb: https://github.com/MonetDB/sqlalchemy-monetdb .. _snowflake-sqlalchemy: https://github.com/snowflakedb/snowflake-sqlalchemy .. _sqlalchemy-pytds: https://pypi.org/project/sqlalchemy-pytds/ @@ -176,6 +189,11 @@ Currently maintained external dialect projects for SQLAlchemy include: .. _sqlalchemy-hsqldb: https://pypi.org/project/sqlalchemy-hsqldb/ .. _databricks: https://docs.databricks.com/en/dev-tools/sqlalchemy.html .. _clickhouse-sqlalchemy: https://pypi.org/project/clickhouse-sqlalchemy/ +.. _oceanbase-sqlalchemy: https://github.com/oceanbase/ecology-plugins/tree/main/oceanbase-sqlalchemy-plugin .. _sqlalchemy-kinetica: https://github.com/kineticadb/sqlalchemy-kinetica/ .. _sqlalchemy-tidb: https://github.com/pingcap/sqlalchemy-tidb .. _ydb-sqlalchemy: https://github.com/ydb-platform/ydb-sqlalchemy/ +.. _denodo-sqlalchemy: https://pypi.org/project/denodo-sqlalchemy/ +.. _aurora-dsql-sqlalchemy: https://pypi.org/project/aurora-dsql-sqlalchemy/ +.. _pydynamodb: https://pypi.org/project/pydynamodb/ +.. _pymongosql: https://pypi.org/project/pymongosql/ diff --git a/doc/build/dialects/mssql.rst b/doc/build/dialects/mssql.rst index b4ea496905e..5d7d35395e4 100644 --- a/doc/build/dialects/mssql.rst +++ b/doc/build/dialects/mssql.rst @@ -161,6 +161,12 @@ PyODBC ------ .. automodule:: sqlalchemy.dialects.mssql.pyodbc +.. _mssql_python: + +mssql-python +------------ +.. automodule:: sqlalchemy.dialects.mssql.mssqlpython + pymssql ------- .. automodule:: sqlalchemy.dialects.mssql.pymssql diff --git a/doc/build/dialects/mysql.rst b/doc/build/dialects/mysql.rst index 657cd2a4189..d00d30e9de7 100644 --- a/doc/build/dialects/mysql.rst +++ b/doc/build/dialects/mysql.rst @@ -223,6 +223,8 @@ MySQL DML Constructs .. autoclass:: sqlalchemy.dialects.mysql.Insert :members: +.. autofunction:: sqlalchemy.dialects.mysql.limit + mysqlclient (fork of MySQL-Python) diff --git a/doc/build/dialects/oracle.rst b/doc/build/dialects/oracle.rst index b3d44858ced..d01db28c904 100644 --- a/doc/build/dialects/oracle.rst +++ b/doc/build/dialects/oracle.rst @@ -15,6 +15,7 @@ originate from :mod:`sqlalchemy.types` or from the local dialect:: from sqlalchemy.dialects.oracle import ( BFILE, BLOB, + BOOLEAN, CHAR, CLOB, DATE, @@ -31,11 +32,9 @@ originate from :mod:`sqlalchemy.types` or from the local dialect:: TIMESTAMP, VARCHAR, VARCHAR2, + VECTOR, ) -.. versionadded:: 1.2.19 Added :class:`_types.NCHAR` to the list of datatypes - exported by the Oracle dialect. - Types which are specific to Oracle Database, or have Oracle-specific construction arguments, are as follows: @@ -50,6 +49,10 @@ construction arguments, are as follows: .. autoclass:: BINARY_FLOAT :members: __init__ +.. autoclass:: BOOLEAN + :members: __init__ + :noindex: + .. autoclass:: DATE :members: __init__ @@ -59,6 +62,9 @@ construction arguments, are as follows: .. autoclass:: INTERVAL :members: __init__ +.. autoclass:: JSON + :members: __init__ + .. autoclass:: NCLOB :members: __init__ @@ -80,11 +86,38 @@ construction arguments, are as follows: .. autoclass:: TIMESTAMP :members: __init__ +.. autoclass:: VECTOR + :members: __init__ + +.. autoclass:: VectorIndexType + :members: + +.. autoclass:: VectorIndexConfig + :members: + :undoc-members: + +.. autoclass:: VectorStorageFormat + :members: + +.. autoclass:: VectorDistanceType + :members: + +.. autoclass:: VectorStorageType + :members: + +.. autoclass:: SparseVector + :members: + + .. _oracledb: python-oracledb --------------- +.. versionchanged:: 2.1 + ``oracledb`` is now the default Oracle dialect when no specific dialect + is specified in the URL (e.g. ``oracle://...``). + .. automodule:: sqlalchemy.dialects.oracle.oracledb .. _cx_oracle: @@ -92,4 +125,8 @@ python-oracledb cx_Oracle --------- +.. versionchanged:: 2.1 + ``cx_oracle`` is no longer the default Oracle dialect. To explicitly use + ``cx_oracle``, specify ``oracle+cx_oracle://...`` in the URL. + .. automodule:: sqlalchemy.dialects.oracle.cx_oracle diff --git a/doc/build/dialects/postgresql.rst b/doc/build/dialects/postgresql.rst index 2d377e3623e..79a04b30460 100644 --- a/doc/build/dialects/postgresql.rst +++ b/doc/build/dialects/postgresql.rst @@ -17,8 +17,27 @@ as well as array literals: * :func:`_postgresql.array_agg` - ARRAY_AGG SQL function -* :class:`_postgresql.aggregate_order_by` - helper for PG's ORDER BY aggregate - function syntax. +* :meth:`_functions.FunctionElement.aggregate_order_by` - dialect-agnostic ORDER BY + for aggregate functions + +* :class:`_postgresql.aggregate_order_by` - legacy helper specific to PostgreSQL + +BIT type +-------- + +PostgreSQL's BIT type is a so-called "bit string" that stores a string of +ones and zeroes. SQLAlchemy provides the :class:`_postgresql.BIT` type +to represent columns and expressions of this type, as well as the +:class:`_postgresql.BitString` value type which is a richly featured ``str`` +subclass that works with :class:`_postgresql.BIT`. + +* :class:`_postgresql.BIT` - the PostgreSQL BIT type + +* :class:`_postgresql.BitString` - Rich-featured ``str`` subclass returned + and accepted for columns and expressions that use :class:`_postgresql.BIT`. + +.. versionchanged:: 2.1 :class:`_postgresql.BIT` now works with the newly + added :class:`_postgresql.BitString` value type. .. _postgresql_json_types: @@ -69,9 +88,6 @@ The combination of ENUM and ARRAY is not directly supported by backend DBAPIs at this time. Prior to SQLAlchemy 1.3.17, a special workaround was needed in order to allow this combination to work, described below. -.. versionchanged:: 1.3.17 The combination of ENUM and ARRAY is now directly - handled by SQLAlchemy's implementation without any workarounds needed. - .. sourcecode:: python from sqlalchemy import TypeDecorator @@ -120,10 +136,6 @@ Similar to using ENUM, prior to SQLAlchemy 1.3.17, for an ARRAY of JSON/JSONB we need to render the appropriate CAST. Current psycopg2 drivers accommodate the result set correctly without any special steps. -.. versionchanged:: 1.3.17 The combination of JSON/JSONB and ARRAY is now - directly handled by SQLAlchemy's implementation without any workarounds - needed. - .. sourcecode:: python class CastingArray(ARRAY): @@ -462,6 +474,9 @@ construction arguments, are as follows: .. autoclass:: BIT +.. autoclass:: BitString + :members: + .. autoclass:: BYTEA :members: __init__ @@ -597,6 +612,8 @@ PostgreSQL SQL Elements and Functions .. autoclass:: ts_headline +.. autofunction:: distinct_on + PostgreSQL Constraint Types --------------------------- @@ -627,19 +644,27 @@ PostgreSQL DML Constructs .. autoclass:: sqlalchemy.dialects.postgresql.Insert :members: -.. _postgresql_psycopg2: +.. _postgresql_psycopg: -psycopg2 +psycopg -------- -.. automodule:: sqlalchemy.dialects.postgresql.psycopg2 +.. versionchanged:: 2.1 + ``psycopg`` (psycopg 3) is now the default PostgreSQL dialect when no + specific dialect is specified in the URL (e.g. ``postgresql://...``). -.. _postgresql_psycopg: +.. automodule:: sqlalchemy.dialects.postgresql.psycopg -psycopg +.. _postgresql_psycopg2: + +psycopg2 -------- -.. automodule:: sqlalchemy.dialects.postgresql.psycopg +.. versionchanged:: 2.1 + ``psycopg2`` is no longer the default PostgreSQL dialect. To explicitly + use ``psycopg2``, specify ``postgresql+psycopg2://...`` in the URL. + +.. automodule:: sqlalchemy.dialects.postgresql.psycopg2 .. _postgresql_pg8000: diff --git a/doc/build/errors.rst b/doc/build/errors.rst index e3ba5cce8f1..8dce4555b7a 100644 --- a/doc/build/errors.rst +++ b/doc/build/errors.rst @@ -65,7 +65,7 @@ familiar with. does not necessarily establish a new connection to the database at the moment the connection object is acquired; it instead consults the connection pool for a connection, which will often retrieve an existing - connection from the pool to be re-used. If no connections are available, + connection from the pool to be reused. If no connections are available, the pool will create a new database connection, but only if the pool has not surpassed a configured capacity. @@ -136,7 +136,7 @@ What causes an application to use up all the connections that it has available? upon to release resources in a timely manner. A common reason this can occur is that the application uses ORM sessions and - does not call :meth:`.Session.close` upon them one the work involving that + does not call :meth:`.Session.close` upon them once the work involving that session is complete. Solution is to make sure ORM sessions if using the ORM, or engine-bound :class:`_engine.Connection` objects if using Core, are explicitly closed at the end of the work being done, either via the appropriate @@ -176,6 +176,27 @@ What causes an application to use up all the connections that it has available? * Threading errors, such as mutexes in a mutual deadlock, or calling upon an already locked mutex in the same thread +* **The application's worker threads are not yielding control** - In + applications that use a fixed thread pool for handling concurrent requests, + such as web applications using a threaded server, if individual request + threads fail to yield control back to the Python interpreter on a regular + basis, other threads that are waiting for a database connection from the + pool may time out even though there are no actual issues with connection + availability or database performance. This condition is known as "thread + starvation" and typically occurs when application code includes CPU-intensive + operations, tight loops without I/O operations, or calls to C extensions + that do not release the Python Global Interpreter Lock (GIL). In such cases, + threads that hold checked-out connections may monopolize CPU time, preventing + the connection pool from serving other threads that are blocked waiting for + connections to become available. Solutions include: + + * Breaking up long-running CPU-intensive operations into smaller chunks + * Ensuring regular I/O operations occur within request handlers + * Explicitly yielding control periodically using ``time.sleep(0)`` or + ``os.sched_yield()`` + * Moving CPU-intensive work to separate background processes or workers + that do not hold database connections + Keep in mind an alternative to using pooling is to turn off pooling entirely. See the section :ref:`pool_switching` for background on this. However, note that when this error message is occurring, it is **always** due to a bigger @@ -1142,11 +1163,6 @@ Overall, "delete-orphan" cascade is usually applied on the "one" side of a one-to-many relationship so that it deletes objects in the "many" side, and not the other way around. -.. versionchanged:: 1.3.18 The text of the "delete-orphan" error message - when used on a many-to-one or many-to-many relationship has been updated - to be more descriptive. - - .. seealso:: :ref:`unitofwork_cascades` @@ -1402,14 +1418,13 @@ notes at :ref:`migration_20_step_six` for an example. When transforming to a dataclass, attribute(s) originate from superclass which is not a dataclass. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This warning occurs when using the SQLAlchemy ORM Mapped Dataclasses feature +This error occurs when using the SQLAlchemy ORM Mapped Dataclasses feature described at :ref:`orm_declarative_native_dataclasses` in conjunction with any mixin class or abstract base that is not itself declared as a dataclass, such as in the example below:: from __future__ import annotations - import inspect from typing import Optional from uuid import uuid4 @@ -1439,18 +1454,17 @@ dataclass, such as in the example below:: email: Mapped[str] = mapped_column() Above, since ``Mixin`` does not itself extend from :class:`_orm.MappedAsDataclass`, -the following warning is generated: +the following error is generated: .. sourcecode:: none - SADeprecationWarning: When transforming to a - dataclass, attribute(s) "create_user", "update_user" originates from - superclass , which is not a dataclass. This usage is deprecated and - will raise an error in SQLAlchemy 2.1. When declaring SQLAlchemy - Declarative Dataclasses, ensure that all mixin classes and other - superclasses which include attributes are also a subclass of - MappedAsDataclass. + sqlalchemy.exc.InvalidRequestError: When transforming to a dataclass, attribute(s) 'create_user', 'update_user' + originates from superclass , which is not a + dataclass. When declaring SQLAlchemy Declarative Dataclasses, ensure that + all mixin classes and other superclasses which include attributes are also + a subclass of MappedAsDataclass or make use of the @unmapped_dataclass + decorator. The fix is to add :class:`_orm.MappedAsDataclass` to the signature of ``Mixin`` as well:: @@ -1459,6 +1473,41 @@ The fix is to add :class:`_orm.MappedAsDataclass` to the signature of create_user: Mapped[int] = mapped_column() update_user: Mapped[Optional[int]] = mapped_column(default=None, init=False) +When using decorators like :func:`_orm.mapped_as_dataclass` to map, the +:func:`_orm.unmapped_dataclass` may be used to indicate mixins:: + + from __future__ import annotations + + from typing import Optional + from uuid import uuid4 + + from sqlalchemy import String + from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_as_dataclass + from sqlalchemy.orm import mapped_column + from sqlalchemy.orm import registry + from sqlalchemy.orm import unmapped_dataclass + + + @unmapped_dataclass + class Mixin: + create_user: Mapped[int] = mapped_column() + update_user: Mapped[Optional[int]] = mapped_column(default=None, init=False) + + + reg = registry() + + + @mapped_as_dataclass(reg) + class User(Mixin): + __tablename__ = "sys_user" + + uid: Mapped[str] = mapped_column( + String(50), init=False, default_factory=uuid4, primary_key=True + ) + username: Mapped[str] = mapped_column() + email: Mapped[str] = mapped_column() + Python's :pep:`681` specification does not accommodate for attributes declared on superclasses of dataclasses that are not themselves dataclasses; per the behavior of Python dataclasses, such fields are ignored, as in the following @@ -1487,14 +1536,12 @@ Above, the ``User`` class will not include ``create_user`` in its constructor nor will it attempt to interpret ``update_user`` as a dataclass attribute. This is because ``Mixin`` is not a dataclass. -SQLAlchemy's dataclasses feature within the 2.0 series does not honor this -behavior correctly; instead, attributes on non-dataclass mixins and -superclasses are treated as part of the final dataclass configuration. However -type checkers such as Pyright and Mypy will not consider these fields as -part of the dataclass constructor as they are to be ignored per :pep:`681`. -Since their presence is ambiguous otherwise, SQLAlchemy 2.1 will require that +Since type checkers such as Pyright and Mypy will not consider these fields as +part of the dataclass constructor as they are to be ignored per :pep:`681`, +their presence becomes ambiguous. Therefore SQLAlchemy requires that mixin classes which have SQLAlchemy mapped attributes within a dataclass -hierarchy have to themselves be dataclasses. +hierarchy have to themselves be dataclasses using SQLAlchemy's unmapped +dataclass feature. .. _error_dcte: diff --git a/doc/build/faq/connections.rst b/doc/build/faq/connections.rst index 1f3bf1ba140..cc95c059256 100644 --- a/doc/build/faq/connections.rst +++ b/doc/build/faq/connections.rst @@ -258,7 +258,9 @@ statement executions:: fn(cursor_obj, statement, context=context, *arg) except engine.dialect.dbapi.Error as raw_dbapi_err: connection = context.root_connection - if engine.dialect.is_disconnect(raw_dbapi_err, connection, cursor_obj): + if engine.dialect.is_disconnect( + raw_dbapi_err, connection.connection.dbapi_connection, cursor_obj + ): engine.logger.error( "disconnection error, attempt %d/%d", retry + 1, @@ -342,7 +344,7 @@ reconnect operation: ping: 1 ... -.. versionadded: 1.4 the above recipe makes use of 1.4-specific behaviors and will +.. versionadded:: 1.4 the above recipe makes use of 1.4-specific behaviors and will not work as given on previous SQLAlchemy versions. The above recipe is tested for SQLAlchemy 1.4. diff --git a/doc/build/faq/installation.rst b/doc/build/faq/installation.rst index 72b4fc15915..51491cd29d9 100644 --- a/doc/build/faq/installation.rst +++ b/doc/build/faq/installation.rst @@ -11,10 +11,9 @@ Installation I'm getting an error about greenlet not being installed when I try to use asyncio ---------------------------------------------------------------------------------- -The ``greenlet`` dependency does not install by default for CPU architectures -for which ``greenlet`` does not supply a `pre-built binary wheel `_. -Notably, **this includes Apple M1**. To install including ``greenlet``, -add the ``asyncio`` `setuptools extra `_ +The ``greenlet`` dependency is not install by default in the 2.1 series. +To install including ``greenlet``, you need to add the ``asyncio`` +`setuptools extra `_ to the ``pip install`` command: .. sourcecode:: text diff --git a/doc/build/faq/ormconfiguration.rst b/doc/build/faq/ormconfiguration.rst index bfcf117ae09..0653e9044a1 100644 --- a/doc/build/faq/ormconfiguration.rst +++ b/doc/build/faq/ormconfiguration.rst @@ -110,11 +110,11 @@ such as: * :attr:`_orm.Mapper.columns` - A namespace of :class:`_schema.Column` objects and other named SQL expressions associated with the mapping. -* :attr:`_orm.Mapper.mapped_table` - The :class:`_schema.Table` or other selectable to which +* :attr:`_orm.Mapper.persist_selectable` - The :class:`_schema.Table` or other selectable to which this mapper is mapped. * :attr:`_orm.Mapper.local_table` - The :class:`_schema.Table` that is "local" to this mapper; - this differs from :attr:`_orm.Mapper.mapped_table` in the case of a mapper mapped + this differs from :attr:`_orm.Mapper.persist_selectable` in the case of a mapper mapped using inheritance to a composed selectable. .. _faq_combining_columns: @@ -389,29 +389,57 @@ parameters are **synonymous**. Part Two - Using Dataclasses support with MappedAsDataclass ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. versionchanged:: 2.1 The behavior of column level defaults when using + dataclasses has changed to use an approach that uses class-level descriptors + to provide class behavior, in conjunction with Core-level column defaults + to provide the correct INSERT behavior. See :ref:`change_12168` for + background. + When you **are** using :class:`_orm.MappedAsDataclass`, that is, the specific form of mapping used at :ref:`orm_declarative_native_dataclasses`, the meaning of the :paramref:`_orm.mapped_column.default` keyword changes. We recognize that it's not ideal that this name changes its behavior, however there was no alternative as PEP-681 requires :paramref:`_orm.mapped_column.default` to take on this meaning. -When dataclasses are used, the :paramref:`_orm.mapped_column.default` parameter must -be used the way it's described at -`Python Dataclasses `_ - it refers -to a constant value like a string or a number, and **is applied to your object -immediately when constructed**. It is also at the moment also applied to the -:paramref:`_orm.mapped_column.default` parameter of :class:`_schema.Column` where -it would be used in an ``INSERT`` statement automatically even if not present -on the object. If you instead want to use a callable for your dataclass, -which will be applied to the object when constructed, you would use -:paramref:`_orm.mapped_column.default_factory`. - -To get access to the ``INSERT``-only behavior of :paramref:`_orm.mapped_column.default` -that is described in part one above, you would use the -:paramref:`_orm.mapped_column.insert_default` parameter instead. -:paramref:`_orm.mapped_column.insert_default` when dataclasses are used continues -to be a direct route to the Core-level "default" process where the parameter can -be a static value or callable. +When dataclasses are used, the :paramref:`_orm.mapped_column.default` parameter +must be used the way it's described at `Python Dataclasses +`_ - it refers to a +constant value like a string or a number, and **is available on your object +immediately when constructed**. As of SQLAlchemy 2.1, the value is delivered +using a descriptor if not otherwise set, without the value actually being +placed in ``__dict__`` unless it were passed to the constructor explicitly. + +The value used for :paramref:`_orm.mapped_column.default` is also applied to the +:paramref:`_schema.Column.default` parameter of :class:`_schema.Column`. +This is so that the value used as the dataclass default is also applied in +an ORM INSERT statement for a mapped object where the value was not +explicitly passed. Using this parameter is **mutually exclusive** against the +:paramref:`_schema.Column.insert_default` parameter, meaning that both cannot +be used at the same time. + +The :paramref:`_orm.mapped_column.default` and +:paramref:`_orm.mapped_column.insert_default` parameters may also be used +(one or the other, not both) +for a SQLAlchemy-mapped dataclass field, or for a dataclass overall, +that indicates ``init=False``. +In this usage, if :paramref:`_orm.mapped_column.default` is used, the default +value will be available on the constructed object immediately as well as +used within the INSERT statement. If :paramref:`_orm.mapped_column.insert_default` +is used, the constructed object will return ``None`` for the attribute value, +but the default value will still be used for the INSERT statement. + +For the specific case of using a callable to generate defaults, the situation +changes a bit; the :paramref:`_orm.mapped_column.default_factory` parameter is +a **dataclass only** parameter that may be used to generate new default values +for instances of the class, but **only takes place when the object is +constructed**. That is, it is **not** equivalent to +:paramref:`_orm.mapped_column.insert_default` with a callable as it **will not +take effect** for a plain ``insert()`` statement that does not actually +construct the object; it only is useful for objects that are inserted using +:term:`unit of work` patterns, i.e. using :meth:`_orm.Session.add` with +:meth:`_orm.Session.flush` / :meth:`_orm.Session.commit`. For defaults that +should apply to INSERT statements regardless of how they are invoked, use +:paramref:`_orm.mapped_column.insert_default` instead. .. list-table:: Summary Chart :header-rows: 1 @@ -421,22 +449,26 @@ be a static value or callable. - Works without dataclasses? - Accepts scalar? - Accepts callable? - - Populates object immediately? + - Available on object immediately? + - Used in INSERT statements? * - :paramref:`_orm.mapped_column.default` - ✔ - ✔ - ✔ - Only if no dataclasses - Only if dataclasses - * - :paramref:`_orm.mapped_column.insert_default` - ✔ + * - :paramref:`_orm.mapped_column.insert_default` + - ✔ (only if no ``default``) - ✔ - ✔ - ✔ - ✖ + - ✔ * - :paramref:`_orm.mapped_column.default_factory` - ✔ - ✖ - ✖ - ✔ - Only if dataclasses + - ✖ (unit of work only) diff --git a/doc/build/faq/sqlexpressions.rst b/doc/build/faq/sqlexpressions.rst index 7a03bdb0362..e09fda4a272 100644 --- a/doc/build/faq/sqlexpressions.rst +++ b/doc/build/faq/sqlexpressions.rst @@ -486,6 +486,8 @@ an expression that has left/right operands and an operator) using the >>> print((column("q1") + column("q2")).self_group().op("->")(column("p"))) {printsql}(q1 + q2) -> p +.. _faq_sql_expression_paren_rules: + Why are the parentheses rules like this? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -555,3 +557,6 @@ Perhaps this change can be made at some point, however for the time being keeping the parenthesization rules more internally consistent seems to be the safer approach. +.. seealso:: + + :ref:`operators_parentheses` - in the Operator Reference diff --git a/doc/build/index.rst b/doc/build/index.rst index 6846a00e898..b5e70727dc8 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -55,7 +55,7 @@ SQLAlchemy Documentation .. container:: - Users upgrading to SQLAlchemy version 2.0 will want to read: + Users upgrading to SQLAlchemy version 2.1 will want to read: * :doc:`What's New in SQLAlchemy 2.1? ` - New features and behaviors in version 2.1 @@ -193,4 +193,4 @@ SQLAlchemy Documentation errors * :doc:`Complete table of of contents ` - Full list of available documentation - * :ref:`Index ` - Index for easy lookup of documentation topics \ No newline at end of file + * :ref:`Index ` - Index for easy lookup of documentation topics diff --git a/doc/build/intro.rst b/doc/build/intro.rst index cba95ab69e7..2c68b5489a9 100644 --- a/doc/build/intro.rst +++ b/doc/build/intro.rst @@ -96,11 +96,11 @@ Supported Platforms SQLAlchemy 2.1 supports the following platforms: -* cPython 3.9 and higher +* cPython 3.10 and higher * Python-3 compatible versions of `PyPy `_ .. versionchanged:: 2.1 - SQLAlchemy now targets Python 3.9 and above. + SQLAlchemy now targets Python 3.10 and above. Supported Installation Methods diff --git a/doc/build/orm/backref.rst b/doc/build/orm/backref.rst index 01f4c90736d..63a45d0ee6c 100644 --- a/doc/build/orm/backref.rst +++ b/doc/build/orm/backref.rst @@ -146,7 +146,7 @@ condition is copied to the new :func:`_orm.relationship` as well:: "user".id = address.user_id AND address.email LIKE :email_1 || '%%' >>> -Other arguments that are transferrable include the +Other arguments that are transferable include the :paramref:`_orm.relationship.secondary` parameter that refers to a many-to-many association table, as well as the "join" arguments :paramref:`_orm.relationship.primaryjoin` and diff --git a/doc/build/orm/basic_relationships.rst b/doc/build/orm/basic_relationships.rst index a1bdb0525c3..a0a541fb982 100644 --- a/doc/build/orm/basic_relationships.rst +++ b/doc/build/orm/basic_relationships.rst @@ -248,8 +248,8 @@ In the preceding example, the ``Parent.child`` relationship is not typed as allowing ``None``; this follows from the ``Parent.child_id`` column itself not being nullable, as it is typed with ``Mapped[int]``. If we wanted ``Parent.child`` to be a **nullable** many-to-one, we can set both -``Parent.child_id`` and ``Parent.child`` to be ``Optional[]``, in which -case the configuration would look like:: +``Parent.child_id`` and ``Parent.child`` to be ``Optional[]`` (or its +equivalent), in which case the configuration would look like:: from typing import Optional @@ -804,10 +804,10 @@ and ``Child.parent_associations -> Association.parent``:: ) extra_data: Mapped[Optional[str]] - # association between Assocation -> Child + # association between Association -> Child child: Mapped["Child"] = relationship(back_populates="parent_associations") - # association between Assocation -> Parent + # association between Association -> Parent parent: Mapped["Parent"] = relationship(back_populates="child_associations") @@ -1018,7 +1018,7 @@ within any of these string expressions:: In an example like the above, the string passed to :class:`_orm.Mapped` can be disambiguated from a specific class argument by passing the class -location string directly to :paramref:`_orm.relationship.argument` as well. +location string directly to the first positional parameter (:paramref:`_orm.relationship.argument`) as well. Below illustrates a typing-only import for ``Child``, combined with a runtime specifier for the target class that will search for the correct name within the :class:`_orm.registry`:: diff --git a/doc/build/orm/cascades.rst b/doc/build/orm/cascades.rst index 20f96001e33..62b0168f1aa 100644 --- a/doc/build/orm/cascades.rst +++ b/doc/build/orm/cascades.rst @@ -221,7 +221,7 @@ that it's now pending within a :class:`_orm.Session`, and there would frequently be subsequent issues where autoflush would prematurely flush the object and cause errors, in those cases where the given object was still being constructed and wasn't in a ready state to be flushed. The option to select between -uni-directional and bi-directional behvaiors was also removed, as this option +uni-directional and bi-directional behaviors was also removed, as this option created two slightly different ways of working, adding to the overall learning curve of the ORM as well as to the documentation and user support burden. diff --git a/doc/build/orm/collection_api.rst b/doc/build/orm/collection_api.rst index 442e88c9810..a04160edee2 100644 --- a/doc/build/orm/collection_api.rst +++ b/doc/build/orm/collection_api.rst @@ -180,7 +180,7 @@ may be appropriately parametrized:: >>> item = Item() >>> item.notes["a"] = Note("a", "atext") - >>> item.notes.items() + >>> item.notes {'a': <__main__.Note object at 0x2eaaf0>} :func:`.attribute_keyed_dict` will ensure that @@ -220,7 +220,7 @@ of the ``Note.text`` field:: keyword: Mapped[str] text: Mapped[str] - item: Mapped["Item"] = relationship() + item: Mapped["Item"] = relationship(back_populates="notes") @property def note_key(self): diff --git a/doc/build/orm/composites.rst b/doc/build/orm/composites.rst index 2fc62cbfd01..4bd11d75406 100644 --- a/doc/build/orm/composites.rst +++ b/doc/build/orm/composites.rst @@ -178,6 +178,69 @@ well as with instances of the ``Vertex`` class, where the ``.start`` and :ref:`mutable_toplevel` extension must be used. See the section :ref:`mutable_composites` for examples. +Returning None for a Composite +------------------------------- + +The composite attribute by default always returns an object when accessed, +regardless of the values of its columns. In the example below, a new +``Vertex`` is created with no parameters; all column attributes ``x1``, ``y1``, +``x2``, and ``y2`` start out as ``None``. A ``Point`` object with ``None`` +values will be returned on access:: + + >>> v1 = Vertex() + >>> v1.start + Point(x=None, y=None) + >>> v1.end + Point(x=None, y=None) + +This behavior is consistent with persistent objects and individual attribute +queries as well:: + + >>> start = session.scalars( + ... select(Point.start).where(Point.x1 == None, Point.y1 == None) + ... ).first() + >>> start + Point(x=None, y=None) + +To support an optional ``Point`` field, we can make use +of the :paramref:`_orm.composite.return_none_on` parameter, which allows +the behavior to be customized with a lambda; this parameter is set automatically if we +declare our composite fields as optional:: + + class Vertex(Base): + __tablename__ = "vertices" + + id: Mapped[int] = mapped_column(primary_key=True) + start: Mapped[Point | None] = composite(mapped_column("x1"), mapped_column("y1")) + end: Mapped[Point | None] = composite(mapped_column("x2"), mapped_column("y2")) + +Above, the :paramref:`_orm.composite.return_none_on` parameter is set equivalently as:: + + composite( + mapped_column("x1"), + mapped_column("y1"), + return_none_on=lambda *args: all(arg is None for arg in args), + ) + +With the above setting, a value of ``None`` is returned if the columns themselves +are both ``None``:: + + >>> v1 = Vertex() + >>> v1.start + None + + >>> start = session.scalars( + ... select(Point.start).where(Point.x1 == None, Point.y1 == None) + ... ).first() + >>> start + None + +.. versionchanged:: 2.1 - added the :paramref:`_orm.composite.return_none_on` parameter with + ORM Annotated Declarative support. + + .. seealso:: + + :ref:`change_12570` .. _orm_composite_other_forms: diff --git a/doc/build/orm/dataclasses.rst b/doc/build/orm/dataclasses.rst index 7f6c2670d96..ec2dbec2c9c 100644 --- a/doc/build/orm/dataclasses.rst +++ b/doc/build/orm/dataclasses.rst @@ -52,7 +52,8 @@ decorator. Dataclass conversion may be added to any Declarative class either by adding the :class:`_orm.MappedAsDataclass` mixin to a :class:`_orm.DeclarativeBase` class hierarchy, or for decorator mapping by using the -:meth:`_orm.registry.mapped_as_dataclass` class decorator. +:meth:`_orm.registry.mapped_as_dataclass` class decorator or its +functional variant :func:`_orm.mapped_as_dataclass`. The :class:`_orm.MappedAsDataclass` mixin may be applied either to the Declarative ``Base`` class or any superclass, as in the example @@ -95,7 +96,7 @@ Or may be applied directly to classes that extend from the Declarative base:: id: Mapped[int] = mapped_column(init=False, primary_key=True) name: Mapped[str] -When using the decorator form, only the :meth:`_orm.registry.mapped_as_dataclass` +When using the decorator form, the :meth:`_orm.registry.mapped_as_dataclass` decorator is supported:: from sqlalchemy.orm import Mapped @@ -113,6 +114,28 @@ decorator is supported:: id: Mapped[int] = mapped_column(init=False, primary_key=True) name: Mapped[str] +The same method is available in a standalone function form, which may +have better compatibility with some versions of the mypy type checker:: + + from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_as_dataclass + from sqlalchemy.orm import mapped_column + from sqlalchemy.orm import registry + + + reg = registry() + + + @mapped_as_dataclass(reg) + class User: + __tablename__ = "user_account" + + id: Mapped[int] = mapped_column(init=False, primary_key=True) + name: Mapped[str] + +.. versionadded:: 2.0.44 Added :func:`_orm.mapped_as_dataclass` after observing + mypy compatibility issues with the method form of the same feature + Class level feature configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -142,8 +165,9 @@ class configuration arguments are passed as class-level parameters:: id: Mapped[int] = mapped_column(init=False, primary_key=True) name: Mapped[str] -When using the decorator form with :meth:`_orm.registry.mapped_as_dataclass`, -class configuration arguments are passed to the decorator directly:: +When using the decorator form with :meth:`_orm.registry.mapped_as_dataclass` or +:func:`_orm.mapped_as_dataclass`, class configuration arguments are passed to +the decorator directly:: from sqlalchemy.orm import registry from sqlalchemy.orm import Mapped @@ -208,13 +232,14 @@ and ``fullname`` is optional. The ``id`` field, which we expect to be database-generated, is not part of the constructor at all:: from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_as_dataclass from sqlalchemy.orm import mapped_column from sqlalchemy.orm import registry reg = registry() - @reg.mapped_as_dataclass + @mapped_as_dataclass(reg) class User: __tablename__ = "user_account" @@ -245,13 +270,14 @@ but where the parameter is optional in the constructor:: from sqlalchemy import func from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_as_dataclass from sqlalchemy.orm import mapped_column from sqlalchemy.orm import registry reg = registry() - @reg.mapped_as_dataclass + @mapped_as_dataclass(reg) class User: __tablename__ = "user_account" @@ -280,7 +306,7 @@ Integration with Annotated The approach introduced at :ref:`orm_declarative_mapped_column_pep593` illustrates how to use :pep:`593` ``Annotated`` objects to package whole -:func:`_orm.mapped_column` constructs for re-use. While ``Annotated`` objects +:func:`_orm.mapped_column` constructs for reuse. While ``Annotated`` objects can be combined with the use of dataclasses, **dataclass-specific keyword arguments unfortunately cannot be used within the Annotated construct**. This includes :pep:`681`-specific arguments ``init``, ``default``, ``repr``, and @@ -300,6 +326,7 @@ emit a deprecation warning:: from typing import Annotated from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_as_dataclass from sqlalchemy.orm import mapped_column from sqlalchemy.orm import registry @@ -309,7 +336,7 @@ emit a deprecation warning:: reg = registry() - @reg.mapped_as_dataclass + @mapped_as_dataclass(reg) class User: __tablename__ = "user_account" id: Mapped[intpk] @@ -325,6 +352,7 @@ the other arguments can remain within the ``Annotated`` construct:: from typing import Annotated from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_as_dataclass from sqlalchemy.orm import mapped_column from sqlalchemy.orm import registry @@ -333,7 +361,7 @@ the other arguments can remain within the ``Annotated`` construct:: reg = registry() - @reg.mapped_as_dataclass + @mapped_as_dataclass(reg) class User: __tablename__ = "user_account" @@ -348,15 +376,19 @@ the other arguments can remain within the ``Annotated`` construct:: Using mixins and abstract superclasses ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Any mixins or base classes that are used in a :class:`_orm.MappedAsDataclass` -mapped class which include :class:`_orm.Mapped` attributes must themselves be -part of a :class:`_orm.MappedAsDataclass` -hierarchy, such as in the example below using a mixin:: +Mixin and abstract superclass are supported with the Declarative Dataclass +Mapping by defining classes that are part of the :class:`_orm.MappedAsDataclass` +hierarchy, either without including a declarative base or by setting +``__abstract__ = True``. The example below illustrates a class ``Mixin`` that is +not itself mapped, but serves as part of the base for a mapped class:: + + from sqlalchemy.orm import DeclarativeBase + from sqlalchemy.orm import MappedAsDataclass class Mixin(MappedAsDataclass): create_user: Mapped[int] = mapped_column() - update_user: Mapped[Optional[int]] = mapped_column(default=None, init=False) + update_user: Mapped[Optional[int]] = mapped_column(default=None) class Base(DeclarativeBase, MappedAsDataclass): @@ -372,22 +404,79 @@ hierarchy, such as in the example below using a mixin:: username: Mapped[str] = mapped_column() email: Mapped[str] = mapped_column() -Python type checkers which support :pep:`681` will otherwise not consider -attributes from non-dataclass mixins to be part of the dataclass. +.. tip:: -.. deprecated:: 2.0.8 Using mixins and abstract bases within - :class:`_orm.MappedAsDataclass` or - :meth:`_orm.registry.mapped_as_dataclass` hierarchies which are not - themselves dataclasses is deprecated, as these fields are not supported - by :pep:`681` as belonging to the dataclass. A warning is emitted for this - case which will later be an error. + When using :class:`_orm.MappedAsDataclass` without a declarative base in + the hierarchy, the target class is still turned into a real Python dataclass, + so that it may properly serve as a base for a mapped dataclass. Using + :class:`_orm.MappedAsDataclass` (or the :func:`_orm.unmapped_dataclass` decorator + described later in this section) is required in order for the class to be correctly + recognized by type checkers as SQLAlchemy-enabled dataclasses. Declarative + itself will reject mixins / abstract classes that are not themselves + Declarative Dataclasses (e.g. they can't be plain classes nor can they be + plain ``@dataclass`` classes). - .. seealso:: + .. seealso:: + + :ref:`error_dcmx` - further background + +Another example, where an abstract base combines :class:`_orm.MappedAsDataclass` +with ``__abstract__ = True``:: + + from sqlalchemy.orm import DeclarativeBase + from sqlalchemy.orm import MappedAsDataclass + + + class Base(DeclarativeBase, MappedAsDataclass): + pass + + + class AbstractUser(Base): + __abstract__ = True + + create_user: Mapped[int] = mapped_column() + update_user: Mapped[Optional[int]] = mapped_column(default=None) + + + class User(AbstractUser): + __tablename__ = "sys_user" + + uid: Mapped[str] = mapped_column( + String(50), init=False, default_factory=uuid4, primary_key=True + ) + username: Mapped[str] = mapped_column() + email: Mapped[str] = mapped_column() + +Finally, for a hierarchy that's based on use of the :func:`_orm.mapped_as_dataclass` +decorator, mixins may be defined using the :func:`_orm.unmapped_dataclass` decorator:: - :ref:`error_dcmx` - background on rationale + from sqlalchemy.orm import registry + from sqlalchemy.orm import mapped_as_dataclass + from sqlalchemy.orm import unmapped_dataclass + + + @unmapped_dataclass() + class Mixin: + create_user: Mapped[int] = mapped_column() + update_user: Mapped[Optional[int]] = mapped_column(default=None, init=False) + + + reg = registry() + @mapped_as_dataclass(reg) + class User(Mixin): + __tablename__ = "sys_user" + + uid: Mapped[str] = mapped_column( + String(50), init=False, default_factory=uuid4, primary_key=True + ) + username: Mapped[str] = mapped_column() + email: Mapped[str] = mapped_column() + +.. versionadded:: 2.1 Added :func:`_orm.unmapped_dataclass` +.. _orm_declarative_dc_relationships: Relationship Configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -405,6 +494,7 @@ scalar object references may make use of from sqlalchemy import ForeignKey from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_as_dataclass from sqlalchemy.orm import mapped_column from sqlalchemy.orm import registry from sqlalchemy.orm import relationship @@ -412,7 +502,7 @@ scalar object references may make use of reg = registry() - @reg.mapped_as_dataclass + @mapped_as_dataclass(reg) class Parent: __tablename__ = "parent" id: Mapped[int] = mapped_column(primary_key=True) @@ -421,7 +511,7 @@ scalar object references may make use of ) - @reg.mapped_as_dataclass + @mapped_as_dataclass(reg) class Child: __tablename__ = "child" id: Mapped[int] = mapped_column(primary_key=True) @@ -454,13 +544,14 @@ of the object, but will not be persisted by the ORM:: from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_as_dataclass from sqlalchemy.orm import mapped_column from sqlalchemy.orm import registry reg = registry() - @reg.mapped_as_dataclass + @mapped_as_dataclass(reg) class Data: __tablename__ = "data" @@ -481,7 +572,7 @@ In the example below, the ``User`` class is declared using ``id``, ``name`` and ``password_hash`` as mapped features, but makes use of init-only ``password`` and ``repeat_password`` fields to represent the user creation process (note: to run this example, replace -the function ``your_crypt_function_here()`` with a third party crypt +the function ``your_hash_function_here()`` with a third party hash function, such as `bcrypt `_ or `argon2-cffi `_):: @@ -489,13 +580,14 @@ function, such as `bcrypt `_ or from typing import Optional from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_as_dataclass from sqlalchemy.orm import mapped_column from sqlalchemy.orm import registry reg = registry() - @reg.mapped_as_dataclass + @mapped_as_dataclass(reg) class User: __tablename__ = "user_account" @@ -511,7 +603,7 @@ function, such as `bcrypt `_ or if password != repeat_password: raise ValueError("passwords do not match") - self.password_hash = your_crypt_function_here(password) + self.password_hash = your_hash_function_here(password) The above object is created with parameters ``password`` and ``repeat_password``, which are consumed up front so that the ``password_hash`` @@ -519,7 +611,7 @@ variable may be generated:: >>> u1 = User(name="some_user", password="xyz", repeat_password="xyz") >>> u1.password_hash - '$6$9ppc... (example crypted string....)' + '$6$9ppc... (example hashed string....)' .. versionchanged:: 2.0.0rc1 When using :meth:`_orm.registry.mapped_as_dataclass` or :class:`.MappedAsDataclass`, fields that do not include the @@ -547,7 +639,8 @@ Integrating with Alternate Dataclass Providers such as Pydantic details which **explicitly resolve** these incompatibilities. SQLAlchemy's :class:`_orm.MappedAsDataclass` class -and :meth:`_orm.registry.mapped_as_dataclass` method call directly into +:meth:`_orm.registry.mapped_as_dataclass` method, and +:func:`_orm.mapped_as_dataclass` functions call directly into the Python standard library ``dataclasses.dataclass`` class decorator, after the declarative mapping process has been applied to the class. This function call may be swapped out for alternateive dataclasses providers, @@ -933,6 +1026,11 @@ applies when using this mapping style. Applying ORM mappings to an existing attrs class ------------------------------------------------- +.. warning:: The ``attrs`` library is not part of SQLAlchemy's continuous + integration testing, and compatibility with this library may change without + notice due to incompatibilities introduced by either side. + + The attrs_ library is a popular third party library that provides similar features as dataclasses, with many additional features provided not found in ordinary dataclasses. @@ -942,103 +1040,27 @@ initiates a process to scan the class for attributes that define the class' behavior, which are then used to generate methods, documentation, and annotations. -The SQLAlchemy ORM supports mapping an attrs_ class using **Declarative with -Imperative Table** or **Imperative** mapping. The general form of these two -styles is fully equivalent to the -:ref:`orm_declarative_dataclasses_declarative_table` and -:ref:`orm_declarative_dataclasses_imperative_table` mapping forms used with -dataclasses, where the inline attribute directives used by dataclasses or attrs -are unchanged, and SQLAlchemy's table-oriented instrumentation is applied at -runtime. +The SQLAlchemy ORM supports mapping an attrs_ class using **Imperative** mapping. +The general form of this style is equivalent to the +:ref:`orm_imperative_dataclasses` mapping form used with +dataclasses, where the class construction uses ``attrs`` alone, with ORM mappings +applied after the fact without any class attribute scanning. The ``@define`` decorator of attrs_ by default replaces the annotated class with a new __slots__ based class, which is not supported. When using the old style annotation ``@attr.s`` or using ``define(slots=False)``, the class -does not get replaced. Furthermore attrs removes its own class-bound attributes +does not get replaced. Furthermore ``attrs`` removes its own class-bound attributes after the decorator runs, so that SQLAlchemy's mapping process takes over these attributes without any issue. Both decorators, ``@attr.s`` and ``@define(slots=False)`` work with SQLAlchemy. -Mapping attrs with Declarative "Imperative Table" -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In the "Declarative with Imperative Table" style, a :class:`_schema.Table` -object is declared inline with the declarative class. The -``@define`` decorator is applied to the class first, then the -:meth:`_orm.registry.mapped` decorator second:: - - from __future__ import annotations - - from typing import List - from typing import Optional - - from attrs import define - from sqlalchemy import Column - from sqlalchemy import ForeignKey - from sqlalchemy import Integer - from sqlalchemy import MetaData - from sqlalchemy import String - from sqlalchemy import Table - from sqlalchemy.orm import Mapped - from sqlalchemy.orm import registry - from sqlalchemy.orm import relationship - - mapper_registry = registry() - - - @mapper_registry.mapped - @define(slots=False) - class User: - __table__ = Table( - "user", - mapper_registry.metadata, - Column("id", Integer, primary_key=True), - Column("name", String(50)), - Column("FullName", String(50), key="fullname"), - Column("nickname", String(12)), - ) - id: Mapped[int] - name: Mapped[str] - fullname: Mapped[str] - nickname: Mapped[str] - addresses: Mapped[List[Address]] - - __mapper_args__ = { # type: ignore - "properties": { - "addresses": relationship("Address"), - } - } - - - @mapper_registry.mapped - @define(slots=False) - class Address: - __table__ = Table( - "address", - mapper_registry.metadata, - Column("id", Integer, primary_key=True), - Column("user_id", Integer, ForeignKey("user.id")), - Column("email_address", String(50)), - ) - id: Mapped[int] - user_id: Mapped[int] - email_address: Mapped[Optional[str]] - -.. note:: The ``attrs`` ``slots=True`` option, which enables ``__slots__`` on - a mapped class, cannot be used with SQLAlchemy mappings without fully - implementing alternative - :ref:`attribute instrumentation `, as mapped - classes normally rely upon direct access to ``__dict__`` for state storage. - Behavior is undefined when this option is present. - - - -Mapping attrs with Imperative Mapping -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. versionchanged:: 2.0 SQLAlchemy integration with ``attrs`` works only + with imperative mapping style, that is, not using Declarative. + The introduction of ORM Annotated Declarative style is not cross-compatible + with ``attrs``. -Just as is the case with dataclasses, we can make use of -:meth:`_orm.registry.map_imperatively` to map an existing ``attrs`` class -as well:: +The ``attrs`` class is built first. The SQLAlchemy ORM mapping can be +applied after the fact using :meth:`_orm.registry.map_imperatively`:: from __future__ import annotations @@ -1102,11 +1124,6 @@ as well:: mapper_registry.map_imperatively(Address, address) -The above form is equivalent to the previous example using -Declarative with Imperative Table. - - - .. _dataclass: https://docs.python.org/3/library/dataclasses.html .. _dataclasses: https://docs.python.org/3/library/dataclasses.html .. _attrs: https://pypi.org/project/attrs/ diff --git a/doc/build/orm/declarative_mixins.rst b/doc/build/orm/declarative_mixins.rst index 1c6179809a2..8087276d912 100644 --- a/doc/build/orm/declarative_mixins.rst +++ b/doc/build/orm/declarative_mixins.rst @@ -724,7 +724,7 @@ define on the class itself. The here to create user-defined collation routines that pull from multiple collections:: - from sqlalchemy.orm import declarative_mixin, declared_attr + from sqlalchemy.orm import declared_attr class MySQLSettings: diff --git a/doc/build/orm/declarative_tables.rst b/doc/build/orm/declarative_tables.rst index bbac1ea101a..9064a4da695 100644 --- a/doc/build/orm/declarative_tables.rst +++ b/doc/build/orm/declarative_tables.rst @@ -108,7 +108,7 @@ further at :ref:`orm_declarative_metadata`. The :func:`_orm.mapped_column` construct accepts all arguments that are accepted by the :class:`_schema.Column` construct, as well as additional -ORM-specific arguments. The :paramref:`_orm.mapped_column.__name` field, +ORM-specific arguments. The :paramref:`_orm.mapped_column.__name` positional parameter, indicating the name of the database column, is typically omitted, as the Declarative process will make use of the attribute name given to the construct and assign this as the name of the column (in the above example, this refers to @@ -133,22 +133,345 @@ itself (more on this at :ref:`mapper_column_distinct_names`). :ref:`mapping_columns_toplevel` - contains additional notes on affecting how :class:`_orm.Mapper` interprets incoming :class:`.Column` objects. -.. _orm_declarative_mapped_column: +ORM Annotated Declarative - Automated Mapping with Type Annotations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Using Annotated Declarative Table (Type Annotated Forms for ``mapped_column()``) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The :func:`_orm.mapped_column` construct in modern Python is normally augmented +by the use of :pep:`484` Python type annotations, where it is capable of +deriving its column-configuration information from type annotations associated +with the attribute as declared in the Declarative mapped class. These type +annotations, if used, must be present within a special SQLAlchemy type called +:class:`.Mapped`, which is a generic type that indicates a specific Python type +within it. -The :func:`_orm.mapped_column` construct is capable of deriving its column-configuration -information from :pep:`484` type annotations associated with the attribute -as declared in the Declarative mapped class. These type annotations, -if used, **must** -be present within a special SQLAlchemy type called :class:`_orm.Mapped`, which -is a generic_ type that then indicates a specific Python type within it. +Using this technique, the example in the previous section can be written +more succinctly as below:: -Below illustrates the mapping from the previous section, adding the use of -:class:`_orm.Mapped`:: + from sqlalchemy import String + from sqlalchemy.orm import DeclarativeBase + from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_column - from typing import Optional + + class Base(DeclarativeBase): + pass + + + class User(Base): + __tablename__ = "user" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(50)) + fullname: Mapped[str | None] + nickname: Mapped[str | None] = mapped_column(String(30)) + +The example above demonstrates that if a class attribute is type-hinted with +:class:`.Mapped` but doesn't have an explicit :func:`_orm.mapped_column` assigned +to it, SQLAlchemy will automatically create one. Furthermore, details like the +column's datatype and whether it can be null (nullability) are inferred from +the :class:`.Mapped` annotation. However, you can always explicitly provide these +arguments to :func:`_orm.mapped_column` to override these automatically-derived +settings. + +For complete details on using the ORM Annotated Declarative system, see +:ref:`orm_declarative_mapped_column` later in this chapter. + +.. seealso:: + + :ref:`orm_declarative_mapped_column` - complete reference for ORM Annotated Declarative + +Dataclass features in ``mapped_column()`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The :func:`_orm.mapped_column` construct integrates with SQLAlchemy's +"native dataclasses" feature, discussed at +:ref:`orm_declarative_native_dataclasses`. See that section for current +background on additional directives supported by :func:`_orm.mapped_column`. + + + + +.. _orm_declarative_metadata: + +Accessing Table and Metadata +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A declaratively mapped class will always include an attribute called +``__table__``; when the above configuration using ``__tablename__`` is +complete, the declarative process makes the :class:`_schema.Table` +available via the ``__table__`` attribute:: + + + # access the Table + user_table = User.__table__ + +The above table is ultimately the same one that corresponds to the +:attr:`_orm.Mapper.local_table` attribute, which we can see through the +:ref:`runtime inspection system `:: + + from sqlalchemy import inspect + + user_table = inspect(User).local_table + +The :class:`_schema.MetaData` collection associated with both the declarative +:class:`_orm.registry` as well as the base class is frequently necessary in +order to run DDL operations such as CREATE, as well as in use with migration +tools such as Alembic. This object is available via the ``.metadata`` +attribute of :class:`_orm.registry` as well as the declarative base class. +Below, for a small script we may wish to emit a CREATE for all tables against a +SQLite database:: + + engine = create_engine("sqlite://") + + Base.metadata.create_all(engine) + +.. _orm_declarative_table_configuration: + +Declarative Table Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When using Declarative Table configuration with the ``__tablename__`` +declarative class attribute, additional arguments to be supplied to the +:class:`_schema.Table` constructor should be provided using the +``__table_args__`` declarative class attribute. + +This attribute accommodates both positional as well as keyword +arguments that are normally sent to the +:class:`_schema.Table` constructor. +The attribute can be specified in one of two forms. One is as a +dictionary:: + + class MyClass(Base): + __tablename__ = "sometable" + __table_args__ = {"mysql_engine": "InnoDB"} + +The other, a tuple, where each argument is positional +(usually constraints):: + + class MyClass(Base): + __tablename__ = "sometable" + __table_args__ = ( + ForeignKeyConstraint(["id"], ["remote_table.id"]), + UniqueConstraint("foo"), + ) + +Keyword arguments can be specified with the above form by +specifying the last argument as a dictionary:: + + class MyClass(Base): + __tablename__ = "sometable" + __table_args__ = ( + ForeignKeyConstraint(["id"], ["remote_table.id"]), + UniqueConstraint("foo"), + {"autoload": True}, + ) + +A class may also specify the ``__table_args__`` declarative attribute, +as well as the ``__tablename__`` attribute, in a dynamic style using the +:func:`_orm.declared_attr` method decorator. See +:ref:`orm_mixins_toplevel` for background. + +.. _orm_declarative_table_schema_name: + +Explicit Schema Name with Declarative Table +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The schema name for a :class:`_schema.Table` as documented at +:ref:`schema_table_schema_name` is applied to an individual :class:`_schema.Table` +using the :paramref:`_schema.Table.schema` argument. When using Declarative +tables, this option is passed like any other to the ``__table_args__`` +dictionary:: + + from sqlalchemy.orm import DeclarativeBase + + + class Base(DeclarativeBase): + pass + + + class MyClass(Base): + __tablename__ = "sometable" + __table_args__ = {"schema": "some_schema"} + +The schema name can also be applied to all :class:`_schema.Table` objects +globally by using the :paramref:`_schema.MetaData.schema` parameter documented +at :ref:`schema_metadata_schema_name`. The :class:`_schema.MetaData` object +may be constructed separately and associated with a :class:`_orm.DeclarativeBase` +subclass by assigning to the ``metadata`` attribute directly:: + + from sqlalchemy import MetaData + from sqlalchemy.orm import DeclarativeBase + + metadata_obj = MetaData(schema="some_schema") + + + class Base(DeclarativeBase): + metadata = metadata_obj + + + class MyClass(Base): + # will use "some_schema" by default + __tablename__ = "sometable" + +.. seealso:: + + :ref:`schema_table_schema_name` - in the :ref:`metadata_toplevel` documentation. + +.. _orm_declarative_column_options: + +Setting Load and Persistence Options for Declarative Mapped Columns +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The :func:`_orm.mapped_column` construct accepts additional ORM-specific +arguments that affect how the generated :class:`_schema.Column` is +mapped, affecting its load and persistence-time behavior. Options +that are commonly used include: + +* **deferred column loading** - The :paramref:`_orm.mapped_column.deferred` + boolean establishes the :class:`_schema.Column` using + :ref:`deferred column loading ` by default. In the example + below, the ``User.bio`` column will not be loaded by default, but only + when accessed:: + + class User(Base): + __tablename__ = "user" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] + bio: Mapped[str] = mapped_column(Text, deferred=True) + + .. seealso:: + + :ref:`orm_queryguide_column_deferral` - full description of deferred column loading + +* **active history** - The :paramref:`_orm.mapped_column.active_history` + ensures that upon change of value for the attribute, the previous value + will have been loaded and made part of the :attr:`.AttributeState.history` + collection when inspecting the history of the attribute. This may incur + additional SQL statements:: + + class User(Base): + __tablename__ = "user" + + id: Mapped[int] = mapped_column(primary_key=True) + important_identifier: Mapped[str] = mapped_column(active_history=True) + +See the docstring for :func:`_orm.mapped_column` for a list of supported +parameters. + +.. seealso:: + + :ref:`orm_imperative_table_column_options` - describes using + :func:`_orm.column_property` and :func:`_orm.deferred` for use with + Imperative Table configuration + +.. _mapper_column_distinct_names: + +.. _orm_declarative_table_column_naming: + +Naming Declarative Mapped Columns Explicitly +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +All of the examples thus far feature the :func:`_orm.mapped_column` construct +linked to an ORM mapped attribute, where the Python attribute name given +to the :func:`_orm.mapped_column` is also that of the column as we see in +CREATE TABLE statements as well as queries. The name for a column as +expressed in SQL may be indicated by passing the string positional argument +:paramref:`_orm.mapped_column.__name` as the first positional argument. +In the example below, the ``User`` class is mapped with alternate names +given to the columns themselves:: + + class User(Base): + __tablename__ = "user" + + id: Mapped[int] = mapped_column("user_id", primary_key=True) + name: Mapped[str] = mapped_column("user_name") + +Where above ``User.id`` resolves to a column named ``user_id`` +and ``User.name`` resolves to a column named ``user_name``. We +may write a :func:`_sql.select` statement using our Python attribute names +and will see the SQL names generated: + +.. sourcecode:: pycon+sql + + >>> from sqlalchemy import select + >>> print(select(User.id, User.name).where(User.name == "x")) + {printsql}SELECT "user".user_id, "user".user_name + FROM "user" + WHERE "user".user_name = :user_name_1 + + +.. seealso:: + + :ref:`orm_imperative_table_column_naming` - applies to Imperative Table + +.. _orm_declarative_table_adding_columns: + +Appending additional columns to an existing Declarative mapped class +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A declarative table configuration allows the addition of new +:class:`_schema.Column` objects to an existing mapping after the :class:`.Table` +metadata has already been generated. + +For a declarative class that is declared using a declarative base class, +the underlying metaclass :class:`.DeclarativeMeta` includes a ``__setattr__()`` +method that will intercept additional :func:`_orm.mapped_column` or Core +:class:`.Column` objects and +add them to both the :class:`.Table` using :meth:`.Table.append_column` +as well as to the existing :class:`.Mapper` using :meth:`.Mapper.add_property`:: + + MyClass.some_new_column = mapped_column(String) + +Using core :class:`_schema.Column`:: + + MyClass.some_new_column = Column(String) + +All arguments are supported including an alternate name, such as +``MyClass.some_new_column = mapped_column("some_name", String)``. However, +the SQL type must be passed to the :func:`_orm.mapped_column` or +:class:`_schema.Column` object explicitly, as in the above examples where +the :class:`_sqltypes.String` type is passed. There's no capability for +the :class:`_orm.Mapped` annotation type to take part in the operation. + +Additional :class:`_schema.Column` objects may also be added to a mapping +in the specific circumstance of using single table inheritance, where +additional columns are present on mapped subclasses that have +no :class:`.Table` of their own. This is illustrated in the section +:ref:`single_inheritance`. + +.. seealso:: + + :ref:`orm_declarative_table_adding_relationship` - similar examples for :func:`_orm.relationship` + +.. note:: Assignment of mapped + properties to an already mapped class will only + function correctly if the "declarative base" class is used, meaning + the user-defined subclass of :class:`_orm.DeclarativeBase` or the + dynamically generated class returned by :func:`_orm.declarative_base` + or :meth:`_orm.registry.generate_base`. This "base" class includes + a Python metaclass which implements a special ``__setattr__()`` method + that intercepts these operations. + + Runtime assignment of class-mapped attributes to a mapped class will **not** work + if the class is mapped using decorators like :meth:`_orm.registry.mapped` + or imperative functions like :meth:`_orm.registry.map_imperatively`. + + +.. _orm_declarative_mapped_column: + +ORM Annotated Declarative - Complete Guide +------------------------------------------ + +The :func:`_orm.mapped_column` construct is capable of deriving its +column-configuration information from :pep:`484` type annotations associated +with the attribute as declared in the Declarative mapped class. These type +annotations, if used, must be present within a special SQLAlchemy type called +:class:`_orm.Mapped`, which is a generic_ type that then indicates a specific +Python type within it. + +Using this technique, the ``User`` example from previous sections may be +written as below:: from sqlalchemy import String from sqlalchemy.orm import DeclarativeBase @@ -165,8 +488,8 @@ Below illustrates the mapping from the previous section, adding the use of id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column(String(50)) - fullname: Mapped[Optional[str]] - nickname: Mapped[Optional[str]] = mapped_column(String(30)) + fullname: Mapped[str | None] + nickname: Mapped[str | None] = mapped_column(String(30)) Above, when Declarative processes each class attribute, each :func:`_orm.mapped_column` will derive additional arguments from the @@ -182,7 +505,7 @@ annotation present. .. _orm_declarative_mapped_column_nullability: ``mapped_column()`` derives the datatype and nullability from the ``Mapped`` annotation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The two qualities that :func:`_orm.mapped_column` derives from the :class:`_orm.Mapped` annotation are: @@ -235,10 +558,11 @@ The two qualities that :func:`_orm.mapped_column` derives from the ``True``, that will also imply that the column should be ``NOT NULL``. In the absence of **both** of these parameters, the presence of - ``typing.Optional[]`` within the :class:`_orm.Mapped` type annotation will be - used to determine nullability, where ``typing.Optional[]`` means ``NULL``, - and the absence of ``typing.Optional[]`` means ``NOT NULL``. If there is no - ``Mapped[]`` annotation present at all, and there is no + ``typing.Optional[]`` (or its equivalent) within the :class:`_orm.Mapped` + type annotation will be used to determine nullability, where + ``typing.Optional[]`` means ``NULL``, and the absence of + ``typing.Optional[]`` means ``NOT NULL``. If there is no ``Mapped[]`` + annotation present at all, and there is no :paramref:`_orm.mapped_column.nullable` or :paramref:`_orm.mapped_column.primary_key` parameter, then SQLAlchemy's usual default for :class:`_schema.Column` of ``NULL`` is used. @@ -297,7 +621,8 @@ The two qualities that :func:`_orm.mapped_column` derives from the .. _orm_declarative_mapped_column_type_map: Customizing the Type Map -~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ + The mapping of Python types to SQLAlchemy :class:`_types.TypeEngine` types described in the previous section defaults to a hardcoded dictionary @@ -308,11 +633,12 @@ as the :paramref:`_orm.registry.type_annotation_map` parameter when constructing the :class:`_orm.registry`, which may be associated with the :class:`_orm.DeclarativeBase` superclass when first used. -As an example, if we wish to make use of the :class:`_sqltypes.BIGINT` datatype for -``int``, the :class:`_sqltypes.TIMESTAMP` datatype with ``timezone=True`` for -``datetime.datetime``, and then only on Microsoft SQL Server we'd like to use -:class:`_sqltypes.NVARCHAR` datatype when Python ``str`` is used, -the registry and Declarative base could be configured as:: +As an example, if we wish to make use of the :class:`_sqltypes.BIGINT` datatype +for ``int``, the :class:`_sqltypes.TIMESTAMP` datatype with ``timezone=True`` +for ``datetime.datetime``, and then for ``str`` types we'd like to see +:class:`_sqltypes.NVARCHAR` when Microsoft SQL Server is used and +``VARCHAR(255)`` when MySQL is used, the registry and Declarative base could be +configured as:: import datetime @@ -324,7 +650,12 @@ the registry and Declarative base could be configured as:: type_annotation_map = { int: BIGINT, datetime.datetime: TIMESTAMP(timezone=True), - str: String().with_variant(NVARCHAR, "mssql"), + # set up variants for str/String() + str: String() + # use NVARCHAR for MSSQL + .with_variant(NVARCHAR, "mssql") + # add a default VARCHAR length for MySQL + .with_variant(VARCHAR(255), "mysql"), } @@ -341,7 +672,7 @@ first on the Microsoft SQL Server backend, illustrating the ``NVARCHAR`` datatyp .. sourcecode:: pycon+sql >>> from sqlalchemy.schema import CreateTable - >>> from sqlalchemy.dialects import mssql, postgresql + >>> from sqlalchemy.dialects import mssql, mysql, postgresql >>> print(CreateTable(SomeClass.__table__).compile(dialect=mssql.dialect())) {printsql}CREATE TABLE some_table ( id BIGINT NOT NULL IDENTITY, @@ -350,6 +681,20 @@ first on the Microsoft SQL Server backend, illustrating the ``NVARCHAR`` datatyp PRIMARY KEY (id) ) +On MySQL, we get a VARCHAR column with an explicit length (required by +MySQL): + +.. sourcecode:: pycon+sql + + >>> print(CreateTable(SomeClass.__table__).compile(dialect=mysql.dialect())) + {printsql}CREATE TABLE some_table ( + id BIGINT NOT NULL AUTO_INCREMENT, + date TIMESTAMP NOT NULL, + status VARCHAR(255) NOT NULL, + PRIMARY KEY (id) + ) + + Then on the PostgreSQL backend, illustrating ``TIMESTAMP WITH TIME ZONE``: .. sourcecode:: pycon+sql @@ -371,7 +716,8 @@ available beyond this, described in the next two sections. .. _orm_declarative_type_map_union_types: Union types inside the Type Map -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + .. versionchanged:: 2.0.37 The features described in this section have been repaired and enhanced to work consistently. Prior to this change, union @@ -384,7 +730,7 @@ SQLAlchemy supports mapping union types inside the ``type_annotation_map`` to allow mapping database types that can support multiple Python types, such as :class:`_types.JSON` or :class:`_postgresql.JSONB`:: - from typing import Union + from typing import Union, Optional from sqlalchemy import JSON from sqlalchemy.dialects import postgresql from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -440,7 +786,7 @@ from ``type_annotation_map`` to :class:`_orm.Mapped`, however is significant as an indicator for nullability of the :class:`_schema.Column`. When ``None`` is present in the union either as it is placed in the :class:`_orm.Mapped` construct. When present in :class:`_orm.Mapped`, it indicates the :class:`_schema.Column` -would be nullable, in the absense of more specific indicators. This logic works +would be nullable, in the absence of more specific indicators. This logic works in the same way as indicating an ``Optional`` type as described at :ref:`orm_declarative_mapped_column_nullability`. @@ -466,7 +812,7 @@ is described in the next section, :ref:`orm_declarative_type_map_pep695_types`. .. _orm_declarative_type_map_pep695_types: Support for Type Alias Types (defined by PEP 695) and NewType -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In contrast to the typing lookup described in :ref:`orm_declarative_type_map_union_types`, Python typing also includes two @@ -476,13 +822,20 @@ differently from ordinary type aliases (i.e. assigning a type to a variable name), and this difference is honored in how SQLAlchemy resolves these types from the type map. -.. versionchanged:: 2.0.37 The behaviors described in this section for ``typing.NewType`` - as well as :pep:`695` ``type`` have been formalized and corrected. - Deprecation warnings are now emitted for "loose matching" patterns that have - worked in some 2.0 releases, but are to be removed in SQLAlchemy 2.1. +.. versionchanged:: 2.0.44 Support for resolving pep-695 types without a + corresponding entry in :paramref:`_orm.registry.type_annotation_map` + has been expanded, reversing part of the restrictions introduced in 2.0.37. Please ensure SQLAlchemy is up to date before attempting to use the features described in this section. +.. versionchanged:: 2.0.37 The behaviors described in this section for ``typing.NewType`` + as well as :pep:`695` ``type`` were formalized to disallow these types + from being implicitly resolvable without entries in + :paramref:`_orm.registry.type_annotation_map`, with deprecation warnings + emitted when these patterns were detected. As of 2.0.44, a pep-695 type + is implicitly resolvable as long as the type it resolves to is present + in the type map. + The typing module allows the creation of "new types" using ``typing.NewType``:: from typing import NewType @@ -490,116 +843,122 @@ The typing module allows the creation of "new types" using ``typing.NewType``:: nstr30 = NewType("nstr30", str) nstr50 = NewType("nstr50", str) -Additionally, in Python 3.12, a new feature defined by :pep:`695` was introduced which -provides the ``type`` keyword to accomplish a similar task; using -``type`` produces an object that is similar in many ways to ``typing.NewType`` -which is internally referred to as ``typing.TypeAliasType``:: +The ``NewType`` construct creates types that are analogous to creating a +subclass of the referenced type. + +Additionally, :pep:`695` introduced in Python 3.12 provides a new ``type`` +keyword for creating type aliases with greater separation of concerns from plain +aliases, as well as succinct support for generics without requiring explicit +use of ``TypeVar`` or ``Generic`` elements. Types created by the ``type`` +keyword are represented at runtime by ``typing.TypeAliasType``:: type SmallInt = int type BigInt = int type JsonScalar = str | float | bool | None -For the purposes of how SQLAlchemy treats these type objects when used -for SQL type lookup inside of :class:`_orm.Mapped`, it's important to note -that Python does not consider two equivalent ``typing.TypeAliasType`` -or ``typing.NewType`` objects to be equal:: - - # two typing.NewType objects are not equal even if they are both str - >>> nstr50 == nstr30 - False - - # two TypeAliasType objects are not equal even if they are both int - >>> SmallInt == BigInt - False - - # an equivalent union is not equal to JsonScalar - >>> JsonScalar == str | float | bool | None - False - -This is the opposite behavior from how ordinary unions are compared, and -informs the correct behavior for SQLAlchemy's ``type_annotation_map``. When -using ``typing.NewType`` or :pep:`695` ``type`` objects, the type object is -expected to be explicit within the ``type_annotation_map`` for it to be matched -from a :class:`_orm.Mapped` type, where the same object must be stated in order -for a match to be made (excluding whether or not the type inside of -:class:`_orm.Mapped` also unions on ``None``). This is distinct from the -behavior described at :ref:`orm_declarative_type_map_union_types`, where a -plain ``Union`` that is referenced directly will match to other ``Unions`` -based on the composition, rather than the object identity, of a particular type -in ``type_annotation_map``. - -In the example below, the composed types for ``nstr30``, ``nstr50``, -``SmallInt``, ``BigInt``, and ``JsonScalar`` have no overlap with each other -and can be named distinctly within each :class:`_orm.Mapped` construct, and -are also all explicit in ``type_annotation_map``. Any of these types may -also be unioned with ``None`` or declared as ``Optional[]`` without affecting -the lookup, only deriving column nullability:: +Both ``NewType`` and pep-695 ``type`` constructs may be used as arguments +within :class:`_orm.Mapped` annotations, where they will be resolved to Python +types using the following rules: - from typing import NewType +* When a ``TypeAliasType`` or ``NewType`` object is present in the + :paramref:`_orm.registry.type_annotation_map`, it will resolve directly:: - from sqlalchemy import SmallInteger, BigInteger, JSON, String - from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column - from sqlalchemy.schema import CreateTable + from typing import NewType + from sqlalchemy import String, BigInteger nstr30 = NewType("nstr30", str) - nstr50 = NewType("nstr50", str) - type SmallInt = int type BigInt = int - type JsonScalar = str | float | bool | None - class TABase(DeclarativeBase): - type_annotation_map = { - nstr30: String(30), - nstr50: String(50), - SmallInt: SmallInteger, - BigInteger: BigInteger, - JsonScalar: JSON, - } + class Base(DeclarativeBase): + type_annotation_map = {nstr30: String(30), BigInt: BigInteger} - class SomeClass(TABase): + class SomeClass(Base): __tablename__ = "some_table" - id: Mapped[int] = mapped_column(primary_key=True) - normal_str: Mapped[str] + # BigInt is in the type_annotation_map. So this + # will resolve to sqlalchemy.BigInteger + id: Mapped[BigInt] = mapped_column(primary_key=True) - short_str: Mapped[nstr30] - long_str_nullable: Mapped[nstr50 | None] + # nstr30 is in the type_annotation_map. So this + # will resolve to sqlalchemy.String(30) + data: Mapped[nstr30] - small_int: Mapped[SmallInt] - big_int: Mapped[BigInteger] - scalar_col: Mapped[JsonScalar] +* A ``TypeAliasType`` that refers **directly** to another type present + in the type map will resolve against that type:: -a CREATE TABLE for the above mapping will illustrate the different variants -of integer and string we've configured, and looks like: + type PlainInt = int -.. sourcecode:: pycon+sql - >>> print(CreateTable(SomeClass.__table__)) - {printsql}CREATE TABLE some_table ( - id INTEGER NOT NULL, - normal_str VARCHAR NOT NULL, - short_str VARCHAR(30) NOT NULL, - long_str_nullable VARCHAR(50), - small_int SMALLINT NOT NULL, - big_int BIGINT NOT NULL, - scalar_col JSON, - PRIMARY KEY (id) - ) + class Base(DeclarativeBase): + pass -Regarding nullability, the ``JsonScalar`` type includes ``None`` in its -definition, which indicates a nullable column. Similarly the -``long_str_nullable`` column applies a union of ``None`` to ``nstr50``, -which matches to the ``nstr50`` type in the ``type_annotation_map`` while -also applying nullability to the mapped column. The other columns all remain -NOT NULL as they are not indicated as optional. + + class SomeClass(Base): + __tablename__ = "some_table" + + # PlainInt refers to int, which is one of the default types + # already in the type_annotation_map. So this + # will resolve to sqlalchemy.Integer via the int type + id: Mapped[PlainInt] = mapped_column(primary_key=True) + +* A ``TypeAliasType`` that refers to another pep-695 ``TypeAliasType`` + not present in the type map will not resolve (emits a deprecation + warning in 2.0), as this would involve a recursive lookup:: + + type PlainInt = int + type AlsoAnInt = PlainInt + + + class Base(DeclarativeBase): + pass + + + class SomeClass(Base): + __tablename__ = "some_table" + + # AlsoAnInt refers to PlainInt, which is not in the type_annotation_map. + # This will emit a deprecation warning in 2.0, will fail in 2.1 + id: Mapped[AlsoAnInt] = mapped_column(primary_key=True) + +* A ``NewType`` that is not in the type map will not resolve (emits a + deprecation warning in 2.0). Since ``NewType`` is analogous to creating an + entirely new type with different semantics than the type it extends, these + must be explicitly matched in the type map:: + + + from typing import NewType + + nstr30 = NewType("nstr30", str) + + + class Base(DeclarativeBase): + pass + + + class SomeClass(Base): + __tablename__ = "some_table" + + # a NewType is a new kind of type, so this will emit a deprecation + # warning in 2.0 and fail in 2.1, as nstr30 is not present + # in the type_annotation_map. + id: Mapped[nstr30] = mapped_column(primary_key=True) + +For all of the above examples, any type that is combined with ``Optional[]`` +or ``| None`` will consider this to indicate the column is nullable, if +no other directive for nullability is present. + +.. seealso:: + + :ref:`orm_declarative_mapped_column_generic_pep593` .. _orm_declarative_mapped_column_type_map_pep593: -Mapping Multiple Type Configurations to Python Types -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Mapping Multiple Type Configurations to Python Types with pep-593 ``Annotated`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + As individual Python types may be associated with :class:`_types.TypeEngine` configurations of any variety by using the :paramref:`_orm.registry.type_annotation_map` @@ -699,14 +1058,15 @@ more open ended. col_a: Mapped[str | float | bool | None] col_b: Mapped[str | float | bool] - This raises an error since the union types used by ``col_a`` or ``col_b``, - are not found in ``TABase`` type map and ``JsonScalar`` must be referenced - directly. + This raises an error since the union types used by ``col_a`` or ``col_b``, + are not found in ``TABase`` type map and ``JsonScalar`` must be referenced + directly. .. _orm_declarative_mapped_column_pep593: -Mapping Whole Column Declarations to Python Types -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Mapping Whole Column Declarations to Python Types with pep-593 ``Annotated`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + The previous section illustrated using :pep:`593` ``Annotated`` type instances as keys within the :paramref:`_orm.registry.type_annotation_map` @@ -725,7 +1085,7 @@ key style that is common to all mapped classes. There also may be common column configurations such as timestamps with defaults and other fields of pre-established sizes and configurations. We can compose these configurations into :func:`_orm.mapped_column` instances that we then bundle directly into -instances of ``Annotated``, which are then re-used in any number of class +instances of ``Annotated``, which are then reused in any number of class declarations. Declarative will unpack an ``Annotated`` object when provided in this manner, skipping over any other directives that don't apply to SQLAlchemy and searching only for SQLAlchemy ORM constructs. @@ -860,32 +1220,85 @@ The CREATE TABLE statement illustrates these per-attribute settings, adding a ``FOREIGN KEY`` constraint as well as substituting ``UTC_TIMESTAMP`` for ``CURRENT_TIMESTAMP``: -.. sourcecode:: pycon+sql +.. sourcecode:: pycon+sql + + >>> from sqlalchemy.schema import CreateTable + >>> print(CreateTable(SomeClass.__table__)) + {printsql}CREATE TABLE some_table ( + id INTEGER NOT NULL, + created_at DATETIME DEFAULT UTC_TIMESTAMP() NOT NULL, + PRIMARY KEY (id), + FOREIGN KEY(id) REFERENCES parent (id) + ) + +.. note:: The feature of :func:`_orm.mapped_column` just described, where + a fully constructed set of column arguments may be indicated using + :pep:`593` ``Annotated`` objects that contain a "template" + :func:`_orm.mapped_column` object to be copied into the attribute, is + currently not implemented for other ORM constructs such as + :func:`_orm.relationship` and :func:`_orm.composite`. While this functionality + is in theory possible, for the moment attempting to use ``Annotated`` + to indicate further arguments for :func:`_orm.relationship` and similar + will raise a ``NotImplementedError`` exception at runtime, but + may be implemented in future releases. + + +.. _orm_declarative_mapped_column_generic_pep593: + +Mapping Whole Column Declarations to Generic Python Types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Using the ``Annotated`` approach from the previous section, we may also +create a generic version that will apply particular :func:`_orm.mapped_column` +elements across many different Python/SQL types in one step. Below +illustrates a plain alias against a generic form of ``Annotated`` that +will apply the ``primary_key=True`` option to any column to which it's applied:: + + from typing import Annotated + from typing import TypeVar + + T = TypeVar("T", bound=Any) + + PrimaryKey = Annotated[T, mapped_column(primary_key=True)] + +The above type can now apply ``primary_key=True`` to any Python type:: + + import uuid + + + class Base(DeclarativeBase): + pass + + + class A(Base): + __tablename__ = "a" + + # will create an Integer primary key + id: Mapped[PrimaryKey[int]] + + + class B(Base): + __tablename__ = "b" + + # will create a UUID primary key + id: Mapped[PrimaryKey[uuid.UUID]] + +For a more shorthand approach, we may opt to use the :pep:`695` ``type`` +keyword (Python 3.12 or above) which allows us to skip having to define a +``TypeVar`` variable:: - >>> from sqlalchemy.schema import CreateTable - >>> print(CreateTable(SomeClass.__table__)) - {printsql}CREATE TABLE some_table ( - id INTEGER NOT NULL, - created_at DATETIME DEFAULT UTC_TIMESTAMP() NOT NULL, - PRIMARY KEY (id), - FOREIGN KEY(id) REFERENCES parent (id) - ) + type PrimaryKey[T] = Annotated[T, mapped_column(primary_key=True)] + +.. versionadded:: 2.0.44 Generic :pep:`695` types may be used with :pep:`593` + ``Annotated`` elements to create generic types that automatically + deliver :func:`_orm.mapped_column` arguments. -.. note:: The feature of :func:`_orm.mapped_column` just described, where - a fully constructed set of column arguments may be indicated using - :pep:`593` ``Annotated`` objects that contain a "template" - :func:`_orm.mapped_column` object to be copied into the attribute, is - currently not implemented for other ORM constructs such as - :func:`_orm.relationship` and :func:`_orm.composite`. While this functionality - is in theory possible, for the moment attempting to use ``Annotated`` - to indicate further arguments for :func:`_orm.relationship` and similar - will raise a ``NotImplementedError`` exception at runtime, but - may be implemented in future releases. .. _orm_declarative_mapped_column_enums: Using Python ``Enum`` or pep-586 ``Literal`` types in the type map -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + .. versionadded:: 2.0.0b4 - Added ``Enum`` support @@ -1011,7 +1424,7 @@ objects that are otherwise structurally equivalent individually, these must be present in ``type_annotation_map`` to avoid ambiguity. Native Enums and Naming -+++++++++++++++++++++++ +~~~~~~~~~~~~~~~~~~~~~~~~ The :paramref:`.sqltypes.Enum.native_enum` parameter refers to if the :class:`.sqltypes.Enum` datatype should create a so-called "native" @@ -1077,7 +1490,7 @@ Or alternatively within :func:`_orm.mapped_column`:: ) Altering the Configuration of the Default Enum -+++++++++++++++++++++++++++++++++++++++++++++++ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In order to modify the fixed configuration of the :class:`.enum.Enum` datatype that's generated implicitly, specify new entries in the @@ -1139,7 +1552,7 @@ table they belong to the :paramref:`_sqltypes.Enum.inherit_schema` can be set:: type_annotation_map = {Enum: sa.Enum(Enum, inherit_schema=True)} Linking Specific ``enum.Enum`` or ``typing.Literal`` to other datatypes -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The above examples feature the use of an :class:`_sqltypes.Enum` that is automatically configuring itself to the arguments / attributes present on @@ -1165,283 +1578,171 @@ In the above configuration, the ``my_literal`` datatype will resolve to a :class:`._sqltypes.JSON` instance. Other ``Literal`` variants will continue to resolve to :class:`_sqltypes.Enum` datatypes. +.. _orm_declarative_resolve_type_event: -Dataclass features in ``mapped_column()`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :func:`_orm.mapped_column` construct integrates with SQLAlchemy's -"native dataclasses" feature, discussed at -:ref:`orm_declarative_native_dataclasses`. See that section for current -background on additional directives supported by :func:`_orm.mapped_column`. - - - -.. _orm_declarative_metadata: - -Accessing Table and Metadata -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A declaratively mapped class will always include an attribute called -``__table__``; when the above configuration using ``__tablename__`` is -complete, the declarative process makes the :class:`_schema.Table` -available via the ``__table__`` attribute:: - - - # access the Table - user_table = User.__table__ - -The above table is ultimately the same one that corresponds to the -:attr:`_orm.Mapper.local_table` attribute, which we can see through the -:ref:`runtime inspection system `:: - - from sqlalchemy import inspect - - user_table = inspect(User).local_table - -The :class:`_schema.MetaData` collection associated with both the declarative -:class:`_orm.registry` as well as the base class is frequently necessary in -order to run DDL operations such as CREATE, as well as in use with migration -tools such as Alembic. This object is available via the ``.metadata`` -attribute of :class:`_orm.registry` as well as the declarative base class. -Below, for a small script we may wish to emit a CREATE for all tables against a -SQLite database:: - - engine = create_engine("sqlite://") - - Base.metadata.create_all(engine) - -.. _orm_declarative_table_configuration: - -Declarative Table Configuration -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When using Declarative Table configuration with the ``__tablename__`` -declarative class attribute, additional arguments to be supplied to the -:class:`_schema.Table` constructor should be provided using the -``__table_args__`` declarative class attribute. - -This attribute accommodates both positional as well as keyword -arguments that are normally sent to the -:class:`_schema.Table` constructor. -The attribute can be specified in one of two forms. One is as a -dictionary:: - - class MyClass(Base): - __tablename__ = "sometable" - __table_args__ = {"mysql_engine": "InnoDB"} - -The other, a tuple, where each argument is positional -(usually constraints):: - - class MyClass(Base): - __tablename__ = "sometable" - __table_args__ = ( - ForeignKeyConstraint(["id"], ["remote_table.id"]), - UniqueConstraint("foo"), - ) - -Keyword arguments can be specified with the above form by -specifying the last argument as a dictionary:: - - class MyClass(Base): - __tablename__ = "sometable" - __table_args__ = ( - ForeignKeyConstraint(["id"], ["remote_table.id"]), - UniqueConstraint("foo"), - {"autoload": True}, - ) - -A class may also specify the ``__table_args__`` declarative attribute, -as well as the ``__tablename__`` attribute, in a dynamic style using the -:func:`_orm.declared_attr` method decorator. See -:ref:`orm_mixins_toplevel` for background. - -.. _orm_declarative_table_schema_name: - -Explicit Schema Name with Declarative Table -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The schema name for a :class:`_schema.Table` as documented at -:ref:`schema_table_schema_name` is applied to an individual :class:`_schema.Table` -using the :paramref:`_schema.Table.schema` argument. When using Declarative -tables, this option is passed like any other to the ``__table_args__`` -dictionary:: - - from sqlalchemy.orm import DeclarativeBase - - - class Base(DeclarativeBase): - pass - - - class MyClass(Base): - __tablename__ = "sometable" - __table_args__ = {"schema": "some_schema"} - -The schema name can also be applied to all :class:`_schema.Table` objects -globally by using the :paramref:`_schema.MetaData.schema` parameter documented -at :ref:`schema_metadata_schema_name`. The :class:`_schema.MetaData` object -may be constructed separately and associated with a :class:`_orm.DeclarativeBase` -subclass by assigning to the ``metadata`` attribute directly:: - - from sqlalchemy import MetaData - from sqlalchemy.orm import DeclarativeBase - - metadata_obj = MetaData(schema="some_schema") +Resolving Types Programmatically with Events +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. versionadded:: 2.1 - class Base(DeclarativeBase): - metadata = metadata_obj +The :paramref:`_orm.registry.type_annotation_map` is the usual +way to customize how :func:`_orm.mapped_column` types are assigned to Python +types. But for automation of whole classes of types or other custom rules, +the type map resolution can be augmented and/or replaced using the +:meth:`.RegistryEvents.resolve_type_annotation` hook. +This event hook allows for dynamic type resolution that goes beyond the static +mappings possible with :paramref:`_orm.registry.type_annotation_map`. It's +particularly useful when working with generic types, complex type hierarchies, +or when you need to implement custom logic for determining SQL types based +on Python type annotations. - class MyClass(Base): - # will use "some_schema" by default - __tablename__ = "sometable" +Basic Type Resolution with :meth:`.RegistryEvents.resolve_type_annotation` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. seealso:: +Basic type resolution can be set up by registering the event against +a :class:`_orm.registry` or :class:`_orm.DeclarativeBase` class. The event +receives a single parameter that allows inspection of the type annotation +and provides hooks for custom resolution logic. - :ref:`schema_table_schema_name` - in the :ref:`metadata_toplevel` documentation. +The following example shows how to use the hook to resolve custom type aliases +to appropriate SQL types:: -.. _orm_declarative_column_options: + from __future__ import annotations -Setting Load and Persistence Options for Declarative Mapped Columns -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + from typing import Annotated + from typing import Any + from typing import get_args -The :func:`_orm.mapped_column` construct accepts additional ORM-specific -arguments that affect how the generated :class:`_schema.Column` is -mapped, affecting its load and persistence-time behavior. Options -that are commonly used include: + from sqlalchemy import create_engine + from sqlalchemy import event + from sqlalchemy import Integer + from sqlalchemy import String + from sqlalchemy.orm import DeclarativeBase + from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_column + from sqlalchemy.orm import TypeResolve + from sqlalchemy.types import TypeEngine -* **deferred column loading** - The :paramref:`_orm.mapped_column.deferred` - boolean establishes the :class:`_schema.Column` using - :ref:`deferred column loading ` by default. In the example - below, the ``User.bio`` column will not be loaded by default, but only - when accessed:: + # Define some custom type aliases + type UserId = int + type Username = str + LongText = Annotated[str, "long"] - class User(Base): - __tablename__ = "user" - id: Mapped[int] = mapped_column(primary_key=True) - name: Mapped[str] - bio: Mapped[str] = mapped_column(Text, deferred=True) + class Base(DeclarativeBase): + pass - .. seealso:: - :ref:`orm_queryguide_column_deferral` - full description of deferred column loading + @event.listens_for(Base.registry, "resolve_type_annotation") + def resolve_custom_types(resolve_type: TypeResolve) -> TypeEngine[Any] | None: + # Handle our custom type aliases + if resolve_type.raw_pep_695_type is UserId: + return Integer() + elif resolve_type.raw_pep_695_type is Username: + return String(50) + elif resolve_type.raw_pep_593_type: + inner_type, *metadata = get_args(resolve_type.raw_pep_593_type) + if inner_type is str and "long" in metadata: + return String(1000) -* **active history** - The :paramref:`_orm.mapped_column.active_history` - ensures that upon change of value for the attribute, the previous value - will have been loaded and made part of the :attr:`.AttributeState.history` - collection when inspecting the history of the attribute. This may incur - additional SQL statements:: + # Fall back to default resolution + return None - class User(Base): - __tablename__ = "user" - id: Mapped[int] = mapped_column(primary_key=True) - important_identifier: Mapped[str] = mapped_column(active_history=True) + class User(Base): + __tablename__ = "user" -See the docstring for :func:`_orm.mapped_column` for a list of supported -parameters. + id: Mapped[UserId] = mapped_column(primary_key=True) + name: Mapped[Username] + description: Mapped[LongText] -.. seealso:: - :ref:`orm_imperative_table_column_options` - describes using - :func:`_orm.column_property` and :func:`_orm.deferred` for use with - Imperative Table configuration + e = create_engine("sqlite://", echo=True) + Base.metadata.create_all(e) -.. _mapper_column_distinct_names: +In this example, the event handler checks for specific type aliases and +returns appropriate SQL types. When the handler returns ``None``, the +default type resolution logic is used. -.. _orm_declarative_table_column_naming: +Programmatic Resolution of pep-695 and NewType types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Naming Declarative Mapped Columns Explicitly -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +As detailed in :ref:`orm_declarative_type_map_pep695_types`, SQLAlchemy now +automatically resolves simple :pep:`695` ``type`` aliases, but does not +automatically resolve types made using ``typing.NewType`` without +these types being explicitly present in :paramref:`_orm.registry.type_annotation_map`. -All of the examples thus far feature the :func:`_orm.mapped_column` construct -linked to an ORM mapped attribute, where the Python attribute name given -to the :func:`_orm.mapped_column` is also that of the column as we see in -CREATE TABLE statements as well as queries. The name for a column as -expressed in SQL may be indicated by passing the string positional argument -:paramref:`_orm.mapped_column.__name` as the first positional argument. -In the example below, the ``User`` class is mapped with alternate names -given to the columns themselves:: +The :meth:`.RegistryEvents.resolve_type_annotation` event provides a way +to programmatically handle these types. This is particularly useful when you have +many ``NewType`` instances that would be cumbersome +to list individually in the type annotation map:: - class User(Base): - __tablename__ = "user" + from __future__ import annotations - id: Mapped[int] = mapped_column("user_id", primary_key=True) - name: Mapped[str] = mapped_column("user_name") + from typing import Annotated + from typing import Any + from typing import NewType -Where above ``User.id`` resolves to a column named ``user_id`` -and ``User.name`` resolves to a column named ``user_name``. We -may write a :func:`_sql.select` statement using our Python attribute names -and will see the SQL names generated: + from sqlalchemy import event + from sqlalchemy import String + from sqlalchemy.orm import DeclarativeBase + from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_column + from sqlalchemy.orm import TypeResolve + from sqlalchemy.types import TypeEngine -.. sourcecode:: pycon+sql + # Multiple NewType instances + IntPK = NewType("IntPK", int) + UserId = NewType("UserId", int) + ProductId = NewType("ProductId", int) + CategoryName = NewType("CategoryName", str) - >>> from sqlalchemy import select - >>> print(select(User.id, User.name).where(User.name == "x")) - {printsql}SELECT "user".user_id, "user".user_name - FROM "user" - WHERE "user".user_name = :user_name_1 + # PEP 695 type alias that recursively refers to a NewType + type OrderId = Annotated[IntPK, mapped_column(primary_key=True)] -.. seealso:: + class Base(DeclarativeBase): + pass - :ref:`orm_imperative_table_column_naming` - applies to Imperative Table -.. _orm_declarative_table_adding_columns: + @event.listens_for(Base.registry, "resolve_type_annotation") + def resolve_newtype_and_pep695(resolve_type: TypeResolve) -> TypeEngine[Any] | None: -Appending additional columns to an existing Declarative mapped class -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + # Handle NewType instances by checking their supertype + if hasattr(resolve_type.resolved_type, "__supertype__"): + supertype = resolve_type.resolved_type.__supertype__ + if supertype is int: + # return default resolution for int + return resolve_type.resolve(int) + elif supertype is str: + return String(100) -A declarative table configuration allows the addition of new -:class:`_schema.Column` objects to an existing mapping after the :class:`.Table` -metadata has already been generated. + # detect nested pep-695 IntPK type + if ( + resolve_type.resolved_type is IntPK + or resolve_type.pep_593_resolved_argument is IntPK + ): + return resolve_type.resolve(int) -For a declarative class that is declared using a declarative base class, -the underlying metaclass :class:`.DeclarativeMeta` includes a ``__setattr__()`` -method that will intercept additional :func:`_orm.mapped_column` or Core -:class:`.Column` objects and -add them to both the :class:`.Table` using :meth:`.Table.append_column` -as well as to the existing :class:`.Mapper` using :meth:`.Mapper.add_property`:: + return None - MyClass.some_new_column = mapped_column(String) -Using core :class:`_schema.Column`:: + class Order(Base): + __tablename__ = "order" - MyClass.some_new_column = Column(String) + id: Mapped[OrderId] + user_id: Mapped[UserId] + product_id: Mapped[ProductId] + category_name: Mapped[CategoryName] -All arguments are supported including an alternate name, such as -``MyClass.some_new_column = mapped_column("some_name", String)``. However, -the SQL type must be passed to the :func:`_orm.mapped_column` or -:class:`_schema.Column` object explicitly, as in the above examples where -the :class:`_sqltypes.String` type is passed. There's no capability for -the :class:`_orm.Mapped` annotation type to take part in the operation. +This approach allows you to handle entire categories of types programmatically +rather than having to enumerate each one in the type annotation map. -Additional :class:`_schema.Column` objects may also be added to a mapping -in the specific circumstance of using single table inheritance, where -additional columns are present on mapped subclasses that have -no :class:`.Table` of their own. This is illustrated in the section -:ref:`single_inheritance`. .. seealso:: - :ref:`orm_declarative_table_adding_relationship` - similar examples for :func:`_orm.relationship` - -.. note:: Assignment of mapped - properties to an already mapped class will only - function correctly if the "declarative base" class is used, meaning - the user-defined subclass of :class:`_orm.DeclarativeBase` or the - dynamically generated class returned by :func:`_orm.declarative_base` - or :meth:`_orm.registry.generate_base`. This "base" class includes - a Python metaclass which implements a special ``__setattr__()`` method - that intercepts these operations. - - Runtime assignment of class-mapped attributes to a mapped class will **not** work - if the class is mapped using decorators like :meth:`_orm.registry.mapped` - or imperative functions like :meth:`_orm.registry.map_imperatively`. - + :meth:`.RegistryEvents.resolve_type_annotation` .. _orm_imperative_table_configuration: @@ -1667,7 +1968,7 @@ associate additional parameters with the column. Options include: collection when inspecting the history of the attribute. This may incur additional SQL statements:: - from sqlalchemy.orm import deferred + from sqlalchemy.orm import column_property user_table = Table( "user", @@ -1905,7 +2206,7 @@ that selectable. This is so that when an ORM object is loaded or persisted, it can be placed in the :term:`identity map` with an appropriate :term:`identity key`. -In those cases where the a reflected table to be mapped does not include +In those cases where a reflected table to be mapped does not include a primary key constraint, as well as in the general case for :ref:`mapping against arbitrary selectables ` where primary key columns might not be present, the diff --git a/doc/build/orm/events.rst b/doc/build/orm/events.rst index 1db1137e085..37e278df322 100644 --- a/doc/build/orm/events.rst +++ b/doc/build/orm/events.rst @@ -70,6 +70,22 @@ Types of things which occur at the :class:`_orm.Mapper` level include: .. autoclass:: sqlalchemy.orm.MapperEvents :members: +Registry Events +--------------- + +Registry event hooks indicate things happening in reference to a particular +:class:`_orm.registry`. These include configurational events +:meth:`_orm.RegistryEvents.before_configured` and +:meth:`_orm.RegistryEvents.after_configured`, as well as a hook to customize +type resolution :meth:`_orm.RegistryEvents.resolve_type_annotation`. + +.. autoclass:: sqlalchemy.orm.RegistryEvents + :members: + +.. autoclass:: sqlalchemy.orm.TypeResolve + :members: + + Instance Events --------------- diff --git a/doc/build/orm/extensions/associationproxy.rst b/doc/build/orm/extensions/associationproxy.rst index 36c8ef22777..d7c715c0b29 100644 --- a/doc/build/orm/extensions/associationproxy.rst +++ b/doc/build/orm/extensions/associationproxy.rst @@ -619,19 +619,11 @@ convenient for generating WHERE criteria quickly, SQL results should be inspected and "unrolled" into explicit JOIN criteria for best use, especially when chaining association proxies together. - -.. versionchanged:: 1.3 Association proxy features distinct querying modes - based on the type of target. See :ref:`change_4351`. - - - .. _cascade_scalar_deletes: Cascading Scalar Deletes ------------------------ -.. versionadded:: 1.3 - Given a mapping as:: from __future__ import annotations diff --git a/doc/build/orm/extensions/asyncio.rst b/doc/build/orm/extensions/asyncio.rst index 784265f625d..6985c4c34cb 100644 --- a/doc/build/orm/extensions/asyncio.rst +++ b/doc/build/orm/extensions/asyncio.rst @@ -273,7 +273,7 @@ configuration: CREATE TABLE a ( id INTEGER NOT NULL, data VARCHAR NOT NULL, - create_date DATETIME DEFAULT (CURRENT_TIMESTAMP) NOT NULL, + create_date DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, PRIMARY KEY (id) ) ... @@ -298,7 +298,7 @@ configuration: SELECT a.id, a.data, a.create_date FROM a ORDER BY a.id [...] () - SELECT b.a_id AS b_a_id, b.id AS b_id, b.data AS b_data + SELECT b.a_id, b.id, b.data FROM b WHERE b.a_id IN (?, ?, ?) [...] (1, 2, 3) @@ -980,7 +980,7 @@ default pool implementation. If an :class:`_asyncio.AsyncEngine` is be passed from one event loop to another, the method :meth:`_asyncio.AsyncEngine.dispose()` should be called before it's -re-used on a new event loop. Failing to do so may lead to a ``RuntimeError`` +reused on a new event loop. Failing to do so may lead to a ``RuntimeError`` along the lines of ``Task got Future attached to a different loop`` @@ -1172,9 +1172,6 @@ ORM Session API Documentation .. autoclass:: AsyncSession :members: - :exclude-members: sync_session_class - - .. autoattribute:: sync_session_class .. autoclass:: AsyncSessionTransaction :members: diff --git a/doc/build/orm/extensions/baked.rst b/doc/build/orm/extensions/baked.rst index b495f42a422..107424e2576 100644 --- a/doc/build/orm/extensions/baked.rst +++ b/doc/build/orm/extensions/baked.rst @@ -176,7 +176,7 @@ Rationale The "lambda" approach above is a superset of what would be a more traditional "parameterized" approach. Suppose we wished to build a simple system where we build a :class:`~.query.Query` just once, then -store it in a dictionary for re-use. This is possible right now by +store it in a dictionary for reuse. This is possible right now by just building up the query, and removing its :class:`.Session` by calling ``my_cached_query = query.with_session(None)``:: @@ -193,7 +193,7 @@ just building up the query, and removing its :class:`.Session` by calling return query.params(id=id_argument).all() The above approach gets us a very minimal performance benefit. -By re-using a :class:`~.query.Query`, we save on the Python work within +By reusing a :class:`~.query.Query`, we save on the Python work within the ``session.query(Model)`` constructor as well as calling upon ``filter(Model.id == bindparam('id'))``, which will skip for us the building up of the Core expression as well as sending it to :meth:`_query.Query.filter`. @@ -403,8 +403,6 @@ of the baked query:: # the "query" argument, pass that. my_q += lambda q: q.filter(my_subq.to_query(q).exists()) -.. versionadded:: 1.3 - .. _baked_with_before_compile: Using the before_compile event @@ -433,12 +431,6 @@ The above strategy is appropriate for an event that will modify a given :class:`_query.Query` in exactly the same way every time, not dependent on specific parameters or external state that changes. -.. versionadded:: 1.3.11 - added the "bake_ok" flag to the - :meth:`.QueryEvents.before_compile` event and disallowed caching via - the "baked" extension from occurring for event handlers that - return a new :class:`_query.Query` object if this flag is not set. - - Disabling Baked Queries Session-wide ------------------------------------ @@ -456,8 +448,6 @@ which is seeing issues potentially due to cache key conflicts from user-defined baked queries or other baked query issues can turn the behavior off, in order to identify or eliminate baked queries as the cause of an issue. -.. versionadded:: 1.2 - Lazy Loading Integration ------------------------ diff --git a/doc/build/orm/join_conditions.rst b/doc/build/orm/join_conditions.rst index 1a26d94a8b7..06223013897 100644 --- a/doc/build/orm/join_conditions.rst +++ b/doc/build/orm/join_conditions.rst @@ -360,8 +360,6 @@ Above, the :meth:`.FunctionElement.as_comparison` indicates that the ``Point.geom`` expressions. The :func:`.foreign` annotation additionally notes which column takes on the "foreign key" role in this particular relationship. -.. versionadded:: 1.3 Added :meth:`.FunctionElement.as_comparison`. - .. _relationship_overlapping_foreignkeys: Overlapping Foreign Keys @@ -389,7 +387,7 @@ for both; then to make ``Article`` refer to ``Writer`` as well, article_id = mapped_column(Integer) magazine_id = mapped_column(ForeignKey("magazine.id")) - writer_id = mapped_column() + writer_id = mapped_column(Integer) magazine = relationship("Magazine") writer = relationship("Writer") @@ -424,13 +422,19 @@ What this refers to originates from the fact that ``Article.magazine_id`` is the subject of two different foreign key constraints; it refers to ``Magazine.id`` directly as a source column, but also refers to ``Writer.magazine_id`` as a source column in the context of the -composite key to ``Writer``. If we associate an ``Article`` with a -particular ``Magazine``, but then associate the ``Article`` with a -``Writer`` that's associated with a *different* ``Magazine``, the ORM -will overwrite ``Article.magazine_id`` non-deterministically, silently -changing which magazine to which we refer; it may -also attempt to place NULL into this column if we de-associate a -``Writer`` from an ``Article``. The warning lets us know this is the case. +composite key to ``Writer``. + +When objects are added to an ORM :class:`.Session` using :meth:`.Session.add`, +the ORM :term:`flush` process takes on the task of reconciling object +references that correspond to :func:`_orm.relationship` configurations and +delivering this state to the database using INSERT/UPDATE/DELETE statements. In +this specific example, if we associate an ``Article`` with a particular +``Magazine``, but then associate the ``Article`` with a ``Writer`` that's +associated with a *different* ``Magazine``, this flush process will overwrite +``Article.magazine_id`` non-deterministically, silently changing which magazine +to which we refer; it may also attempt to place NULL into this column if we +de-associate a ``Writer`` from an ``Article``. The warning lets us know that +this scenario may occur during ORM flush sequences. To solve this, we need to break out the behavior of ``Article`` to include all three of the following features: @@ -1051,7 +1055,14 @@ conjunction with :class:`_query.Query` as follows: @property def addresses(self): - return object_session(self).query(Address).with_parent(self).filter(...).all() + # query using any kind of filter() criteria + return ( + object_session(self) + .query(Address) + .filter(Address.user_id == self.id) + .filter(...) + .all() + ) In other cases, the descriptor can be built to make use of existing in-Python data. See the section on :ref:`mapper_hybrids` for more general discussion diff --git a/doc/build/orm/mapped_attributes.rst b/doc/build/orm/mapped_attributes.rst index d0610f4e0fa..b114680132e 100644 --- a/doc/build/orm/mapped_attributes.rst +++ b/doc/build/orm/mapped_attributes.rst @@ -234,7 +234,7 @@ logic:: """Produce a SQL expression that represents the value of the _email column, minus the last twelve characters.""" - return func.substr(cls._email, 0, func.length(cls._email) - 12) + return func.substr(cls._email, 1, func.length(cls._email) - 12) Above, accessing the ``email`` property of an instance of ``EmailAddress`` will return the value of the ``_email`` attribute, removing or adding the @@ -249,7 +249,7 @@ attribute, a SQL function is rendered which produces the same effect: {execsql}SELECT address.email AS address_email, address.id AS address_id FROM address WHERE substr(address.email, ?, length(address.email) - ?) = ? - (0, 12, 'address') + (1, 12, 'address') {stop} Read more about Hybrids at :ref:`hybrids_toplevel`. diff --git a/doc/build/orm/mapping_api.rst b/doc/build/orm/mapping_api.rst index 399111d6058..286839b65c4 100644 --- a/doc/build/orm/mapping_api.rst +++ b/doc/build/orm/mapping_api.rst @@ -4,20 +4,29 @@ Class Mapping API ================= -.. autoclass:: registry - :members: - .. autofunction:: add_mapped_attribute +.. autofunction:: as_declarative + +.. autofunction:: class_mapper + +.. autofunction:: clear_mappers + .. autofunction:: column_property -.. autofunction:: declarative_base +.. autofunction:: configure_mappers -.. autofunction:: declarative_mixin +.. autofunction:: declarative_base -.. autofunction:: as_declarative +.. autoclass:: DeclarativeBase + :members: + :inherited-members: + :special-members: __table__, __mapper__, __mapper_args__, __tablename__, __table_args__ -.. autofunction:: mapped_column +.. autoclass:: DeclarativeBaseNoMeta + :members: + :inherited-members: + :special-members: __table__, __mapper__, __mapper_args__, __tablename__, __table_args__ .. autoclass:: declared_attr @@ -109,39 +118,38 @@ Class Mapping API :class:`_orm.declared_attr` -.. autoclass:: DeclarativeBase - :members: - :special-members: __table__, __mapper__, __mapper_args__, __tablename__, __table_args__ - -.. autoclass:: DeclarativeBaseNoMeta - :members: - :special-members: __table__, __mapper__, __mapper_args__, __tablename__, __table_args__ - .. autofunction:: has_inherited_table -.. autofunction:: synonym_for +.. autofunction:: sqlalchemy.orm.util.identity_key -.. autofunction:: object_mapper +.. autofunction:: mapped_as_dataclass -.. autofunction:: class_mapper +.. autofunction:: mapped_column -.. autofunction:: configure_mappers +.. autoclass:: MappedAsDataclass + :members: -.. autofunction:: clear_mappers +.. autoclass:: MappedClassProtocol + :no-members: -.. autofunction:: sqlalchemy.orm.util.identity_key +.. autoclass:: Mapper + :members: -.. autofunction:: polymorphic_union +.. autofunction:: object_mapper .. autofunction:: orm_insert_sentinel -.. autofunction:: reconstructor +.. autofunction:: polymorphic_union -.. autoclass:: Mapper - :members: +.. autofunction:: reconstructor -.. autoclass:: MappedAsDataclass +.. autoclass:: registry :members: -.. autoclass:: MappedClassProtocol - :no-members: +.. autofunction:: synonym_for + +.. autofunction:: unmapped_dataclass + +.. autofunction:: as_typed_table + + diff --git a/doc/build/orm/nonstandard_mappings.rst b/doc/build/orm/nonstandard_mappings.rst index d71343e99fd..10142cfcfbf 100644 --- a/doc/build/orm/nonstandard_mappings.rst +++ b/doc/build/orm/nonstandard_mappings.rst @@ -86,10 +86,6 @@ may be used:: stmt = select(AddressUser).group_by(*AddressUser.id.expressions) -.. versionadded:: 1.3.17 Added the - :attr:`.ColumnProperty.Comparator.expressions` accessor. - - .. note:: A mapping against multiple tables as illustrated above supports diff --git a/doc/build/orm/persistence_techniques.rst b/doc/build/orm/persistence_techniques.rst index a877fcd0e0e..793e088f2a2 100644 --- a/doc/build/orm/persistence_techniques.rst +++ b/doc/build/orm/persistence_techniques.rst @@ -67,12 +67,6 @@ On PostgreSQL, the above :class:`.Session` will emit the following INSERT: ((SELECT coalesce(max(foo.foopk) + %(max_1)s, %(coalesce_2)s) AS coalesce_1 FROM foo), %(bar)s) RETURNING foo.foopk -.. versionadded:: 1.3 - SQL expressions can now be passed to a primary key column during an ORM - flush; if the database supports RETURNING, or if pysqlite is in use, the - ORM will be able to retrieve the server-generated value as the value - of the primary key attribute. - .. _session_sql_expressions: Using SQL Expressions with Sessions @@ -421,7 +415,7 @@ against MySQL (not MariaDB) results in SQL like this upon flush: FROM my_table WHERE my_table.id = %s A future release of SQLAlchemy may seek to improve the efficiency of -eager defaults in the abcense of RETURNING to batch many rows within a +eager defaults in the absence of RETURNING to batch many rows within a single SELECT statement. Case 4: primary key, RETURNING or equivalent is supported diff --git a/doc/build/orm/queryguide/api.rst b/doc/build/orm/queryguide/api.rst index fe4d6b02a49..25b4f861c2f 100644 --- a/doc/build/orm/queryguide/api.rst +++ b/doc/build/orm/queryguide/api.rst @@ -513,6 +513,8 @@ in a manner roughly similar to that of :attr:`.Select.column_descriptions`:: and :attr:`.UpdateBase.returning_column_descriptions` attributes. +.. highlight:: python3 + .. _queryguide_additional: Additional ORM API Constructs @@ -528,6 +530,8 @@ Additional ORM API Constructs .. autoclass:: sqlalchemy.orm.Bundle :members: +.. autoclass:: sqlalchemy.orm.DictBundle + .. autofunction:: sqlalchemy.orm.with_loader_criteria .. autofunction:: sqlalchemy.orm.join diff --git a/doc/build/orm/queryguide/columns.rst b/doc/build/orm/queryguide/columns.rst index ace6a63f4ce..f461d524f8a 100644 --- a/doc/build/orm/queryguide/columns.rst +++ b/doc/build/orm/queryguide/columns.rst @@ -87,7 +87,7 @@ order to load the value. Below, accessing ``.cover_photo`` emits a SELECT statement to load its value:: >>> img_data = books[0].cover_photo - {execsql}SELECT book.cover_photo AS book_cover_photo + {execsql}SELECT book.cover_photo FROM book WHERE book.id = ? [...] (1,) @@ -157,7 +157,7 @@ in addition to primary key column:: {execsql}SELECT user_account.id, user_account.name, user_account.fullname FROM user_account [...] () - SELECT book.owner_id AS book_owner_id, book.id AS book_id, book.title AS book_title + SELECT book.owner_id, book.id, book.title FROM book WHERE book.owner_id IN (?, ?) [...] (1, 2) @@ -184,12 +184,12 @@ the SELECT statement emitted for each ``User.books`` collection:: {execsql}SELECT user_account.id, user_account.name, user_account.fullname FROM user_account [...] () - SELECT book.id AS book_id, book.title AS book_title + SELECT book.id, book.title FROM book WHERE ? = book.owner_id [...] (1,) {stop}Spongebob Squarepants ['100 Years of Krabby Patties', 'Sea Catch 22', 'The Sea Grapes of Wrath'] - {execsql}SELECT book.id AS book_id, book.title AS book_title + {execsql}SELECT book.id, book.title FROM book WHERE ? = book.owner_id [...] (2,) @@ -223,7 +223,7 @@ As is the case with :func:`_orm.load_only`, unloaded columns by default will load themselves when accessed using :term:`lazy loading`:: >>> img_data = books[0].cover_photo - {execsql}SELECT book.cover_photo AS book_cover_photo + {execsql}SELECT book.cover_photo FROM book WHERE book.id = ? [...] (4,) @@ -354,7 +354,7 @@ on the loaded object are first accessed is that they will :term:`lazy load` their value:: >>> img_data = book.cover_photo - {execsql}SELECT book.cover_photo AS book_cover_photo + {execsql}SELECT book.cover_photo FROM book WHERE book.id = ? [...] (2,) @@ -510,7 +510,7 @@ will load both columns at once using just one SELECT statement:: WHERE book.id = ? [...] (2,) {stop}>>> img_data, summary = book.cover_photo, book.summary - {execsql}SELECT book.summary AS book_summary, book.cover_photo AS book_cover_photo + {execsql}SELECT book.summary, book.cover_photo FROM book WHERE book.id = ? [...] (2,) diff --git a/doc/build/orm/queryguide/dml.rst b/doc/build/orm/queryguide/dml.rst index 91fe9e7741d..80c7d3b14fc 100644 --- a/doc/build/orm/queryguide/dml.rst +++ b/doc/build/orm/queryguide/dml.rst @@ -79,6 +79,15 @@ or :func:`_orm.mapped_column` declarations, as well as with the **ORM mapped attribute name** and **not** the actual database column name, if these two names happen to be different. +.. tip:: ORM bulk INSERT **allows each dictionary to have different keys**. + The operation will emit multiple INSERT statements with different VALUES + clauses for each set of keys. This is distinctly different from a Core + :class:`_sql.Insert` operation, which as introduced at + :ref:`tutorial_core_insert_values_clause` only uses the **first** dictionary + in the list to determine a single VALUES clause for all parameter sets. + + + .. versionchanged:: 2.0 Passing an :class:`_dml.Insert` construct to the :meth:`_orm.Session.execute` method now invokes a "bulk insert", which makes use of the same functionality as the legacy diff --git a/doc/build/orm/queryguide/inheritance.rst b/doc/build/orm/queryguide/inheritance.rst index 537d51ae59e..d76edde3f31 100644 --- a/doc/build/orm/queryguide/inheritance.rst +++ b/doc/build/orm/queryguide/inheritance.rst @@ -151,12 +151,12 @@ load columns local to both the ``Manager`` and ``Engineer`` subclasses:: SELECT manager.id AS manager_id, employee.id AS employee_id, employee.type AS employee_type, manager.manager_name AS manager_manager_name FROM employee JOIN manager ON employee.id = manager.id - WHERE employee.id IN (?) ORDER BY employee.id + WHERE employee.id IN (?) [...] (1,) SELECT engineer.id AS engineer_id, employee.id AS employee_id, employee.type AS employee_type, engineer.engineer_info AS engineer_engineer_info FROM employee JOIN engineer ON employee.id = engineer.id - WHERE employee.id IN (?, ?) ORDER BY employee.id + WHERE employee.id IN (?, ?) [...] (2, 3) {stop}>>> print(objects) [Manager('Mr. Krabs'), Engineer('SpongeBob'), Engineer('Squidward')] @@ -211,8 +211,7 @@ we only indicate the additional target subclasses we wish to load:: SELECT company.id, company.name FROM company [...] () - SELECT employee.company_id AS employee_company_id, employee.id AS employee_id, - employee.name AS employee_name, employee.type AS employee_type + SELECT employee.company_id, employee.id, employee.name, employee.type FROM employee WHERE employee.company_id IN (?) [...] (1,) @@ -220,13 +219,13 @@ we only indicate the additional target subclasses we wish to load:: employee.type AS employee_type, manager.manager_name AS manager_manager_name FROM employee JOIN manager ON employee.id = manager.id - WHERE employee.id IN (?) ORDER BY employee.id + WHERE employee.id IN (?) [...] (1,) SELECT engineer.id AS engineer_id, employee.id AS employee_id, employee.type AS employee_type, engineer.engineer_info AS engineer_engineer_info FROM employee JOIN engineer ON employee.id = engineer.id - WHERE employee.id IN (?, ?) ORDER BY employee.id + WHERE employee.id IN (?, ?) [...] (2, 3) {stop}company: Krusty Krab employees: [Manager('Mr. Krabs'), Engineer('SpongeBob'), Engineer('Squidward')] @@ -271,15 +270,15 @@ this collection on all ``Manager`` objects, where the sub-attributes of [...] () SELECT manager.id AS manager_id, employee.id AS employee_id, employee.type AS employee_type, manager.manager_name AS manager_manager_name FROM employee JOIN manager ON employee.id = manager.id - WHERE employee.id IN (?) ORDER BY employee.id + WHERE employee.id IN (?) [...] (1,) - SELECT paperwork.manager_id AS paperwork_manager_id, paperwork.id AS paperwork_id, paperwork.document_name AS paperwork_document_name + SELECT paperwork.manager_id, paperwork.id, paperwork.document_name FROM paperwork WHERE paperwork.manager_id IN (?) [...] (1,) SELECT engineer.id AS engineer_id, employee.id AS employee_id, employee.type AS employee_type, engineer.engineer_info AS engineer_engineer_info FROM employee JOIN engineer ON employee.id = engineer.id - WHERE employee.id IN (?, ?) ORDER BY employee.id + WHERE employee.id IN (?, ?) [...] (2, 3) {stop}>>> print(objects[0]) Manager('Mr. Krabs') @@ -327,21 +326,21 @@ examples to load ``Company.employees``, also loading the attributes for the SELECT company.id, company.name FROM company [...] () - SELECT employee.company_id AS employee_company_id, employee.id AS employee_id, employee.name AS employee_name, employee.type AS employee_type + SELECT employee.company_id, employee.id, employee.name, employee.type FROM employee WHERE employee.company_id IN (?) [...] (1,) SELECT manager.id AS manager_id, employee.id AS employee_id, employee.type AS employee_type, manager.manager_name AS manager_manager_name FROM employee JOIN manager ON employee.id = manager.id - WHERE employee.id IN (?) ORDER BY employee.id + WHERE employee.id IN (?) [...] (1,) - SELECT paperwork.manager_id AS paperwork_manager_id, paperwork.id AS paperwork_id, paperwork.document_name AS paperwork_document_name + SELECT paperwork.manager_id, paperwork.id, paperwork.document_name FROM paperwork WHERE paperwork.manager_id IN (?) [...] (1,) SELECT engineer.id AS engineer_id, employee.id AS employee_id, employee.type AS employee_type, engineer.engineer_info AS engineer_engineer_info FROM employee JOIN engineer ON employee.id = engineer.id - WHERE employee.id IN (?, ?) ORDER BY employee.id + WHERE employee.id IN (?, ?) [...] (2, 3) {stop}company: Krusty Krab manager: Mr. Krabs paperwork: [Paperwork('Secret Recipes'), Paperwork('Krabby Patty Orders')] @@ -847,10 +846,8 @@ eagerly load all elements of ``Company.employees`` using the {execsql}SELECT company.id, company.name FROM company [...] () - SELECT employee.company_id AS employee_company_id, employee.id AS employee_id, - employee.name AS employee_name, employee.type AS employee_type, manager.id AS manager_id, - manager.manager_name AS manager_manager_name, engineer.id AS engineer_id, - engineer.engineer_info AS engineer_engineer_info + SELECT employee.company_id, employee.id, employee.name, employee.type, + manager.id, manager.manager_name, engineer.id, engineer.engineer_info FROM employee LEFT OUTER JOIN manager ON employee.id = manager.id LEFT OUTER JOIN engineer ON employee.id = engineer.id @@ -954,7 +951,7 @@ when it's accessed:: WHERE employee.name = ? [...] ('Mr. Krabs',) {stop}>>> mr_krabs.manager_name - {execsql}SELECT employee.manager_name AS employee_manager_name + {execsql}SELECT employee.manager_name FROM employee WHERE employee.id = ? AND employee.type IN (?) [...] (1, 'manager') diff --git a/doc/build/orm/queryguide/relationships.rst b/doc/build/orm/queryguide/relationships.rst index d63ae67ac74..166c2af8688 100644 --- a/doc/build/orm/queryguide/relationships.rst +++ b/doc/build/orm/queryguide/relationships.rst @@ -754,7 +754,7 @@ IN, which currently includes SQL Server. :paramref:`_orm.relationship.lazy` or by using the :func:`.selectinload` loader option. This style of loading emits a SELECT that refers to the primary key values of the parent object, or in the case of a many-to-one -relationship to the those of the child objects, inside of an IN clause, in +relationship to those of the child objects, inside of an IN clause, in order to load related associations: .. sourcecode:: pycon+sql diff --git a/doc/build/orm/queryguide/select.rst b/doc/build/orm/queryguide/select.rst index a8b273a62dc..5b78ada4625 100644 --- a/doc/build/orm/queryguide/select.rst +++ b/doc/build/orm/queryguide/select.rst @@ -231,11 +231,14 @@ The :class:`_orm.Bundle` is potentially useful for creating lightweight views and custom column groupings. :class:`_orm.Bundle` may also be subclassed in order to return alternate data structures; see :meth:`_orm.Bundle.create_row_processor` for an example. +A dict-returning subclass :class:`_orm.DictBundle` is provided for convenience. .. seealso:: :class:`_orm.Bundle` + :class:`_orm.DictBundle` + :meth:`_orm.Bundle.create_row_processor` @@ -362,7 +365,7 @@ Selecting Entities from Subqueries The :func:`_orm.aliased` construct discussed in the previous section can be used with any :class:`_sql.Subquery` construct that comes from a method such as :meth:`_sql.Select.subquery` to link ORM entities to the -columns returned by that subquery; there must be a **column correspondence** +columns returned by that subquery; by default, there must be a **column correspondence** relationship between the columns delivered by the subquery and the columns to which the entity is mapped, meaning, the subquery needs to be ultimately derived from those entities, such as in the example below:: @@ -384,6 +387,25 @@ derived from those entities, such as in the example below:: User(id=4, name='squidward', fullname='Squidward Tentacles') User(id=5, name='ehkrabs', fullname='Eugene H. Krabs') +Alternatively, an aliased subquery can be matched to the entity based on name +by applying the :paramref:`_orm.aliased.adapt_on_names` parameter:: + + >>> from sqlalchemy import literal + >>> inner_stmt = select( + ... literal(14).label("id"), + ... literal("made up name").label("name"), + ... literal("made up fullname").label("fullname"), + ... ) + >>> subq = inner_stmt.subquery() + >>> aliased_user = aliased(User, subq, adapt_on_names=True) + >>> stmt = select(aliased_user) + >>> for user_obj in session.execute(stmt).scalars(): + ... print(user_obj) + {execsql}SELECT anon_1.id, anon_1.name, anon_1.fullname + FROM (SELECT ? AS id, ? AS name, ? AS fullname) AS anon_1 + [generated in ...] (14, 'made up name', 'made up fullname') + {stop}User(id=14, name='made up name', fullname='made up fullname') + .. seealso:: :ref:`tutorial_subqueries_orm_aliased` - in the :ref:`unified_tutorial` diff --git a/doc/build/orm/quickstart.rst b/doc/build/orm/quickstart.rst index 48f3673699f..0b9bc2a78aa 100644 --- a/doc/build/orm/quickstart.rst +++ b/doc/build/orm/quickstart.rst @@ -80,11 +80,11 @@ of each attribute corresponds to the column that is to be part of the database table. The datatype of each column is taken first from the Python datatype that's associated with each :class:`_orm.Mapped` annotation; ``int`` for ``INTEGER``, ``str`` for ``VARCHAR``, etc. Nullability derives from whether or -not the ``Optional[]`` type modifier is used. More specific typing information -may be indicated using SQLAlchemy type objects in the right side -:func:`_orm.mapped_column` directive, such as the :class:`.String` datatype -used above in the ``User.name`` column. The association between Python types -and SQL types can be customized using the +not the ``Optional[]`` (or its equivalent) type modifier is used. More specific +typing information may be indicated using SQLAlchemy type objects in the right +side :func:`_orm.mapped_column` directive, such as the :class:`.String` +datatype used above in the ``User.name`` column. The association between Python +types and SQL types can be customized using the :ref:`type annotation map `. The :func:`_orm.mapped_column` directive is used for all column-based @@ -341,7 +341,7 @@ address associated with "sandy", and also add a new email address to {stop} >>> patrick.addresses.append(Address(email_address="patrickstar@sqlalchemy.org")) - {execsql}SELECT address.id AS address_id, address.email_address AS address_email_address, address.user_id AS address_user_id + {execsql}SELECT address.id, address.email_address, address.user_id FROM address WHERE ? = address.user_id [...] (3,){stop} @@ -380,13 +380,13 @@ object by primary key using :meth:`_orm.Session.get`, then work with the object: >>> sandy = session.get(User, 2) {execsql}BEGIN (implicit) - SELECT user_account.id AS user_account_id, user_account.name AS user_account_name, user_account.fullname AS user_account_fullname + SELECT user_account.id, user_account.name, user_account.fullname FROM user_account WHERE user_account.id = ? [...] (2,){stop} >>> sandy.addresses.remove(sandy_address) - {execsql}SELECT address.id AS address_id, address.email_address AS address_email_address, address.user_id AS address_user_id + {execsql}SELECT address.id, address.email_address, address.user_id FROM address WHERE ? = address.user_id [...] (2,) @@ -416,11 +416,11 @@ options that we configured, in this case, onto the related ``Address`` objects: .. sourcecode:: pycon+sql >>> session.delete(patrick) - {execsql}SELECT user_account.id AS user_account_id, user_account.name AS user_account_name, user_account.fullname AS user_account_fullname + {execsql}SELECT user_account.id, user_account.name, user_account.fullname FROM user_account WHERE user_account.id = ? [...] (3,) - SELECT address.id AS address_id, address.email_address AS address_email_address, address.user_id AS address_user_id + SELECT address.id, address.email_address, address.user_id FROM address WHERE ? = address.user_id [...] (3,) diff --git a/doc/build/orm/session_events.rst b/doc/build/orm/session_events.rst index 8ab2842bae9..4c192b9b7bd 100644 --- a/doc/build/orm/session_events.rst +++ b/doc/build/orm/session_events.rst @@ -235,7 +235,7 @@ Above, a custom execution option is passed to :meth:`_sql.Select.execution_options` in order to establish a "cache key" that will then be intercepted by the :meth:`_orm.SessionEvents.do_orm_execute` hook. This cache key is then matched to a :class:`_engine.FrozenResult` object that may be -present in the cache, and if present, the object is re-used. The recipe makes +present in the cache, and if present, the object is reused. The recipe makes use of the :meth:`_engine.Result.freeze` method to "freeze" a :class:`_engine.Result` object, which above will contain ORM results, such that it can be stored in a cache and used multiple times. In order to return a live diff --git a/doc/build/orm/versioning.rst b/doc/build/orm/versioning.rst index 7f209e24b26..941a2fcca48 100644 --- a/doc/build/orm/versioning.rst +++ b/doc/build/orm/versioning.rst @@ -21,9 +21,9 @@ the value held in memory matches the database value. The purpose of this feature is to detect when two concurrent transactions are modifying the same row at roughly the same time, or alternatively to provide -a guard against the usage of a "stale" row in a system that might be re-using +a guard against the usage of a "stale" row in a system that might be reusing data from a previous transaction without refreshing (e.g. if one sets ``expire_on_commit=False`` -with a :class:`.Session`, it is possible to re-use the data from a previous +with a :class:`.Session`, it is possible to reuse the data from a previous transaction). .. topic:: Concurrent transaction updates @@ -233,14 +233,14 @@ at our choosing:: __mapper_args__ = {"version_id_col": version_uuid, "version_id_generator": False} - u1 = User(name="u1", version_uuid=uuid.uuid4()) + u1 = User(name="u1", version_uuid=uuid.uuid4().hex) session.add(u1) session.commit() u1.name = "u2" - u1.version_uuid = uuid.uuid4() + u1.version_uuid = uuid.uuid4().hex session.commit() diff --git a/doc/build/requirements.txt b/doc/build/requirements.txt index 9b9bffd36e5..7ad5825770e 100644 --- a/doc/build/requirements.txt +++ b/doc/build/requirements.txt @@ -3,4 +3,5 @@ git+https://github.com/sqlalchemyorg/sphinx-paramlinks.git#egg=sphinx-paramlinks git+https://github.com/sqlalchemyorg/zzzeeksphinx.git#egg=zzzeeksphinx sphinx-copybutton==0.5.1 sphinx-autobuild -typing-extensions +typing-extensions # for autodoc to be able to import source files +greenlet # for autodoc to be able to import sqlalchemy source files diff --git a/doc/build/texinputs/sphinx.sty b/doc/build/texinputs/sphinx.sty index 3782b69fabf..5d1b8f4a0b4 100644 --- a/doc/build/texinputs/sphinx.sty +++ b/doc/build/texinputs/sphinx.sty @@ -42,7 +42,7 @@ % size more like a typical published manual. %\renewcommand{\paperheight}{9in} %\renewcommand{\paperwidth}{8.5in} % typical squarish manual -%\renewcommand{\paperwidth}{7in} % O'Reilly ``Programmming Python'' +%\renewcommand{\paperwidth}{7in} % O'Reilly ``Programming Python'' % For graphicx, check if we are compiling under latex or pdflatex. \ifx\pdftexversion\undefined diff --git a/doc/build/tutorial/data_insert.rst b/doc/build/tutorial/data_insert.rst index d0f6b236d5d..843bd761e6b 100644 --- a/doc/build/tutorial/data_insert.rst +++ b/doc/build/tutorial/data_insert.rst @@ -157,6 +157,22 @@ method in conjunction with the :class:`_sql.Insert` construct, the will be expressed in the VALUES clause of the :class:`_sql.Insert` construct automatically. +.. tip:: + + When passing a list of dictionaries to :meth:`_engine.Connection.execute` + along with a Core :class:`_sql.Insert`, **only the first dictionary in the + list determines what columns will be in the VALUES clause**. The rest of + the dictionaries are not scanned. This is both because within traditional + ``executemany()``, the INSERT statement can only have one VALUES clause for + all parameters, and additionally SQLAlchemy does not want to add overhead + by scanning every parameter dictionary to verify each contains the identical + keys as the first one. + + Note this behavior is distinctly different from that of an :ref:`ORM + enabled INSERT `, introduced later in this tutorial, + which performs a full scan of parameter sets in terms of an ORM entity. + + .. deepalchemy:: Hi, welcome to the first edition of **Deep Alchemy**. The person on the diff --git a/doc/build/tutorial/data_select.rst b/doc/build/tutorial/data_select.rst index 5052a5bae32..7a976c0873d 100644 --- a/doc/build/tutorial/data_select.rst +++ b/doc/build/tutorial/data_select.rst @@ -392,10 +392,31 @@ of ORM entities:: WHERE (user_account.name = :name_1 OR user_account.name = :name_2) AND address.user_id = user_account.id +.. tip:: + + The rendering of parentheses is based on operator precedence rules (there's no + way to detect parentheses from a Python expression at runtime), so if we combine + AND and OR in a way that matches the natural precedence of AND, the rendered + expression might not have similar looking parentheses as our Python code:: + + >>> print( + ... select(Address.email_address).where( + ... or_( + ... User.name == "squidward", + ... and_(Address.user_id == User.id, User.name == "sandy"), + ... ) + ... ) + ... ) + {printsql}SELECT address.email_address + FROM address, user_account + WHERE user_account.name = :name_1 OR address.user_id = user_account.id AND user_account.name = :name_2 + + More background on parenthesization is in the :ref:`operators_parentheses` in the Operator Reference. + For simple "equality" comparisons against a single entity, there's also a popular method known as :meth:`_sql.Select.filter_by` which accepts keyword -arguments that match to column keys or ORM attribute names. It will filter -against the leftmost FROM clause or the last entity joined:: +arguments that match to column keys or ORM attribute names. It searches +across all entities in the FROM clause for the given attribute names:: >>> print(select(User).filter_by(name="spongebob", fullname="Spongebob Squarepants")) {printsql}SELECT user_account.id, user_account.name, user_account.fullname @@ -1631,17 +1652,48 @@ Further options for window functions include usage of ranges; see .. _tutorial_functions_within_group: -Special Modifiers WITHIN GROUP, FILTER -###################################### +Special Modifiers ORDER BY, WITHIN GROUP, FILTER +################################################ -The "WITHIN GROUP" SQL syntax is used in conjunction with an "ordered set" -or a "hypothetical set" aggregate -function. Common "ordered set" functions include ``percentile_cont()`` -and ``rank()``. SQLAlchemy includes built in implementations +Some forms of SQL aggregate functions support ordering of the aggregated elements +within the scope of the function. This typically applies to aggregate +functions that produce a value which continues to enumerate the contents of the +collection, such as the ``array_agg()`` function that generates an array of +elements, or the ``string_agg()`` PostgreSQL function which generates a +delimited string (other backends like MySQL and SQLite use the +``group_concat()`` function in a similar way), or the MySQL ``json_arrayagg()`` +function which produces a JSON array. Ordering of the elements passed +to these functions is supported using the :meth:`_functions.FunctionElement.aggregate_order_by` +method, which will render ORDER BY in the appropriate part of the function:: + + >>> stmt = select( + ... func.group_concat(user_table.c.name).aggregate_order_by(user_table.c.name.desc()) + ... ) + >>> print(stmt) + {printsql}SELECT group_concat(user_account.name ORDER BY user_account.name DESC) AS group_concat_1 + FROM user_account + +.. tip:: The above demonstration shows use of the ``group_concat()`` function + available on SQLite which concatenates strings; the ORDER BY feature + for SQLite requires SQLite 3.44.0 or greater. As the availability, name + and specific syntax of the string aggregation functions varies + widely by backend, SQLAlchemy also provides a backend-agnostic + version specifically for concatenating strings called + :func:`_functions.aggregate_strings`. + +A more specific form of ORDER BY for aggregate functions is the "WITHIN GROUP" +SQL syntax. In some cases, the :meth:`_functions.FunctionElement.aggregate_order_by` +will render this syntax directly, when compiling on a backend such as Oracle +Database or Microsoft SQL Server which requires it for all aggregate ordering. +Beyond that, the "WITHIN GROUP" SQL syntax must sometimes be called upon explicitly, +when used in conjunction with an "ordered set" or a "hypothetical set" +aggregate function, supported by PostgreSQL, Oracle Database, and Microsoft SQL +Server. Common "ordered set" functions include ``percentile_cont()`` and +``rank()``. SQLAlchemy includes built in implementations :class:`_functions.rank`, :class:`_functions.dense_rank`, :class:`_functions.mode`, :class:`_functions.percentile_cont` and -:class:`_functions.percentile_disc` which include a :meth:`_functions.FunctionElement.within_group` -method:: +:class:`_functions.percentile_disc` which include a +:meth:`_functions.FunctionElement.within_group` method:: >>> print( ... func.unnest( @@ -1766,6 +1818,8 @@ where it is usable for custom SQL functions:: :ref:`postgresql_column_valued` - in the :ref:`postgresql_toplevel` documentation. + + .. _tutorial_casts: Data Casts and Type Coercion diff --git a/doc/build/tutorial/data_update.rst b/doc/build/tutorial/data_update.rst index e32b6676c76..29f9a216a78 100644 --- a/doc/build/tutorial/data_update.rst +++ b/doc/build/tutorial/data_update.rst @@ -135,7 +135,7 @@ anywhere a column expression might be placed:: UPDATE..FROM ~~~~~~~~~~~~~ -Some databases such as PostgreSQL and MySQL support a syntax "UPDATE FROM" +Some databases such as PostgreSQL, MSSQL and MySQL support a syntax ``UPDATE...FROM`` where additional tables may be stated directly in a special FROM clause. This syntax will be generated implicitly when additional tables are located in the WHERE clause of the statement:: @@ -172,6 +172,30 @@ order to refer to additional tables:: SET address.email_address=%s, user_account.fullname=%s WHERE user_account.id = address.user_id AND address.email_address = %s +``UPDATE...FROM`` can also be +combined with the :class:`_sql.Values` construct +on backends such as PostgreSQL, to create a single UPDATE statement that updates +multiple rows at once against the named form of VALUES:: + + >>> from sqlalchemy import Values + >>> values = Values( + ... user_table.c.id, + ... user_table.c.name, + ... name="my_values", + ... ).data([(1, "new_name"), (2, "another_name"), ("3", "name_name")]) + >>> update_stmt = ( + ... user_table.update().values(name=values.c.name).where(user_table.c.id == values.c.id) + ... ) + >>> from sqlalchemy.dialects import postgresql + >>> print(update_stmt.compile(dialect=postgresql.dialect())) + {printsql}UPDATE user_account + SET name=my_values.name + FROM (VALUES + (%(param_1)s::INTEGER, %(param_2)s::VARCHAR), + (%(param_3)s::INTEGER, %(param_4)s::VARCHAR), + (%(param_5)s::INTEGER, %(param_6)s::VARCHAR)) AS my_values (id, name) + WHERE user_account.id = my_values.id + .. _tutorial_parameter_ordered_updates: Parameter Ordered Updates @@ -270,8 +294,9 @@ is available from the :attr:`_engine.CursorResult.rowcount` attribute: specific to the DBAPI ``cursor`` object. An instance of this subclass is returned when a statement is invoked via the :meth:`_engine.Connection.execute` method. When using the ORM, the - :meth:`_orm.Session.execute` method returns an object of this type for - all INSERT, UPDATE, and DELETE statements. + :meth:`_orm.Session.execute` method will normally **not** return this type + of object, unless the given query uses only Core :class:`.Table` objects + directly. Facts about :attr:`_engine.CursorResult.rowcount`: diff --git a/doc/build/tutorial/metadata.rst b/doc/build/tutorial/metadata.rst index 7d6f5b31377..5b3730851b2 100644 --- a/doc/build/tutorial/metadata.rst +++ b/doc/build/tutorial/metadata.rst @@ -197,7 +197,39 @@ parameter. related column, in the above example the :class:`_types.Integer` datatype of the ``user_account.id`` column. -In the next section we will emit the completed DDL for the ``user`` and +Using :class:`.TypedColumns` to get a better typing experience +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A SQLAlchemy :class:`_schema.Table` can also be defined using a +:class:`_schema.TypedColumns` to offers better integration with type checker and IDEs. +The tables defined above could be declared as follows:: + + >>> from sqlalchemy import Named, TypedColumns, Table + >>> other_meta = MetaData() + >>> class user_cols(TypedColumns): + ... id: Named[int] = Column(primary_key=True) + ... name: Named[str | None] = Column(String(30)) + ... fullname: Named[str | None] + + >>> typed_user_table = Table("user_account", other_meta, user_cols) + + >>> class address_cols(TypedColumns): + ... id: Named[int] = Column(primary_key=True) + ... user_id: Named[int] = Column(ForeignKey("user_account.id")) + ... email_address: Named[str] + ... __row_pos__: tuple[int, int, str] + + >>> typed_address_table = Table("address", other_meta, address_cols) + +The columns are defined by subclassing :class:`.TypedColumns`, so that +static type checkers can understand what columns are present in the +:attr:`_schema.Table.c` collection. Functionally the two methods of defining +the metadata objects are equivalent. +The optional ``__row_pos__`` annotation is an aid to type checker so that +they can correctly suggest the type to apply when selecting from the complete +table, without specifying the single columns. + +In the next section we will emit the completed DDL for the ``user_account`` and ``address`` table to see the completed result. .. _tutorial_emitting_ddl: @@ -576,7 +608,7 @@ are found to be present already: .. _tutorial_table_reflection: Table Reflection -------------------------------- +---------------- .. topic:: Optional Section @@ -599,7 +631,7 @@ that database. .. tip:: There is no requirement that reflection must be used in order to use SQLAlchemy with a pre-existing database. It is entirely typical that the SQLAlchemy application declares all metadata explicitly in Python, - such that its structure corresponds to that the existing database. + such that its structure corresponds to the existing database. The metadata structure also need not include tables, columns, or other constraints and constructs in the pre-existing database that are not needed for the local application to function. diff --git a/doc/build/tutorial/orm_data_manipulation.rst b/doc/build/tutorial/orm_data_manipulation.rst index 9329d205245..f576b310712 100644 --- a/doc/build/tutorial/orm_data_manipulation.rst +++ b/doc/build/tutorial/orm_data_manipulation.rst @@ -337,8 +337,7 @@ Let's load up ``patrick`` from the database: .. sourcecode:: pycon+sql >>> patrick = session.get(User, 3) - {execsql}SELECT user_account.id AS user_account_id, user_account.name AS user_account_name, - user_account.fullname AS user_account_fullname + {execsql}SELECT user_account.id, user_account.name, user_account.fullname FROM user_account WHERE user_account.id = ? [...] (3,) @@ -354,8 +353,7 @@ until the flush proceeds, which as mentioned before occurs if we emit a query: .. sourcecode:: pycon+sql >>> session.execute(select(User).where(User.name == "patrick")).first() - {execsql}SELECT address.id AS address_id, address.email_address AS address_email_address, - address.user_id AS address_user_id + {execsql}SELECT address.id, address.email_address, address.user_id FROM address WHERE ? = address.user_id [...] (3,) @@ -465,8 +463,7 @@ a new transaction and refresh ``sandy`` with the current database row: >>> sandy.fullname {execsql}BEGIN (implicit) - SELECT user_account.id AS user_account_id, user_account.name AS user_account_name, - user_account.fullname AS user_account_fullname + SELECT user_account.id, user_account.name, user_account.fullname FROM user_account WHERE user_account.id = ? [...] (2,){stop} @@ -548,7 +545,7 @@ a context manager as well, accomplishes the following things: >>> session.add(squidward) >>> squidward.name {execsql}BEGIN (implicit) - SELECT user_account.id AS user_account_id, user_account.name AS user_account_name, user_account.fullname AS user_account_fullname + SELECT user_account.id, user_account.name, user_account.fullname FROM user_account WHERE user_account.id = ? [...] (4,){stop} diff --git a/doc/build/tutorial/orm_related_objects.rst b/doc/build/tutorial/orm_related_objects.rst index 48e049dd9e8..e0dcbac871c 100644 --- a/doc/build/tutorial/orm_related_objects.rst +++ b/doc/build/tutorial/orm_related_objects.rst @@ -227,8 +227,7 @@ newly generated primary key for the ``u1`` object: >>> u1.id {execsql}BEGIN (implicit) - SELECT user_account.id AS user_account_id, user_account.name AS user_account_name, - user_account.fullname AS user_account_fullname + SELECT user_account.id, user_account.name, user_account.fullname FROM user_account WHERE user_account.id = ? [...] (6,){stop} @@ -242,8 +241,7 @@ we again see a :term:`lazy load` emitted in order to retrieve the objects: .. sourcecode:: pycon+sql >>> u1.addresses - {execsql}SELECT address.id AS address_id, address.email_address AS address_email_address, - address.user_id AS address_user_id + {execsql}SELECT address.id, address.email_address, address.user_id FROM address WHERE ? = address.user_id [...] (6,){stop} @@ -456,8 +454,7 @@ related ``Address`` objects: {execsql}SELECT user_account.id, user_account.name, user_account.fullname FROM user_account ORDER BY user_account.id [...] () - SELECT address.user_id AS address_user_id, address.id AS address_id, - address.email_address AS address_email_address + SELECT address.user_id, address.id, address.email_address FROM address WHERE address.user_id IN (?, ?, ?, ?, ?, ?) [...] (1, 2, 3, 4, 5, 6){stop} @@ -669,7 +666,7 @@ instead:: {execsql}SELECT user_account.id FROM user_account [...] () - SELECT address.user_id AS address_user_id, address.id AS address_id + SELECT address.user_id, address.id FROM address WHERE address.user_id IN (?, ?, ?, ?, ?, ?) [...] (1, 2, 3, 4, 5, 6) diff --git a/examples/association/basic_association.py b/examples/association/basic_association.py index 7a5b46097e3..1ef1f698d33 100644 --- a/examples/association/basic_association.py +++ b/examples/association/basic_association.py @@ -10,104 +10,116 @@ """ +from __future__ import annotations + from datetime import datetime -from sqlalchemy import and_ -from sqlalchemy import Column from sqlalchemy import create_engine -from sqlalchemy import DateTime -from sqlalchemy import Float from sqlalchemy import ForeignKey -from sqlalchemy import Integer +from sqlalchemy import select from sqlalchemy import String -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column from sqlalchemy.orm import relationship from sqlalchemy.orm import Session -Base = declarative_base() +class Base(DeclarativeBase): + pass class Order(Base): __tablename__ = "order" - order_id = Column(Integer, primary_key=True) - customer_name = Column(String(30), nullable=False) - order_date = Column(DateTime, nullable=False, default=datetime.now()) - order_items = relationship( - "OrderItem", cascade="all, delete-orphan", backref="order" + order_id: Mapped[int] = mapped_column(primary_key=True) + customer_name: Mapped[str] = mapped_column(String(30)) + order_date: Mapped[datetime] = mapped_column(default=datetime.now()) + order_items: Mapped[list[OrderItem]] = relationship( + cascade="all, delete-orphan", backref="order" ) - def __init__(self, customer_name): + def __init__(self, customer_name: str) -> None: self.customer_name = customer_name class Item(Base): __tablename__ = "item" - item_id = Column(Integer, primary_key=True) - description = Column(String(30), nullable=False) - price = Column(Float, nullable=False) + item_id: Mapped[int] = mapped_column(primary_key=True) + description: Mapped[str] = mapped_column(String(30)) + price: Mapped[float] - def __init__(self, description, price): + def __init__(self, description: str, price: float) -> None: self.description = description self.price = price - def __repr__(self): - return "Item(%r, %r)" % (self.description, self.price) + def __repr__(self) -> str: + return "Item({!r}, {!r})".format(self.description, self.price) class OrderItem(Base): __tablename__ = "orderitem" - order_id = Column(Integer, ForeignKey("order.order_id"), primary_key=True) - item_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True) - price = Column(Float, nullable=False) + order_id: Mapped[int] = mapped_column( + ForeignKey("order.order_id"), primary_key=True + ) + item_id: Mapped[int] = mapped_column( + ForeignKey("item.item_id"), primary_key=True + ) + price: Mapped[float] - def __init__(self, item, price=None): + def __init__(self, item: Item, price: float | None = None) -> None: self.item = item self.price = price or item.price - item = relationship(Item, lazy="joined") + item: Mapped[Item] = relationship(lazy="joined") if __name__ == "__main__": engine = create_engine("sqlite://") Base.metadata.create_all(engine) - session = Session(engine) - - # create catalog - tshirt, mug, hat, crowbar = ( - Item("SA T-Shirt", 10.99), - Item("SA Mug", 6.50), - Item("SA Hat", 8.99), - Item("MySQL Crowbar", 16.99), - ) - session.add_all([tshirt, mug, hat, crowbar]) - session.commit() - - # create an order - order = Order("john smith") - - # add three OrderItem associations to the Order and save - order.order_items.append(OrderItem(mug)) - order.order_items.append(OrderItem(crowbar, 10.99)) - order.order_items.append(OrderItem(hat)) - session.add(order) - session.commit() - - # query the order, print items - order = session.query(Order).filter_by(customer_name="john smith").one() - print( - [ - (order_item.item.description, order_item.price) - for order_item in order.order_items - ] - ) - - # print customers who bought 'MySQL Crowbar' on sale - q = session.query(Order).join(OrderItem).join(Item) - q = q.filter( - and_(Item.description == "MySQL Crowbar", Item.price > OrderItem.price) - ) - - print([order.customer_name for order in q]) + with Session(engine) as session: + + # create catalog + tshirt, mug, hat, crowbar = ( + Item("SA T-Shirt", 10.99), + Item("SA Mug", 6.50), + Item("SA Hat", 8.99), + Item("MySQL Crowbar", 16.99), + ) + session.add_all([tshirt, mug, hat, crowbar]) + session.commit() + + # create an order + order = Order("john smith") + + # add three OrderItem associations to the Order and save + order.order_items.append(OrderItem(mug)) + order.order_items.append(OrderItem(crowbar, 10.99)) + order.order_items.append(OrderItem(hat)) + session.add(order) + session.commit() + + # query the order, print items + order = session.scalars( + select(Order).filter_by(customer_name="john smith") + ).one() + print( + [ + (order_item.item.description, order_item.price) + for order_item in order.order_items + ] + ) + + # print customers who bought 'MySQL Crowbar' on sale + q = ( + select(Order) + .join(OrderItem) + .join(Item) + .where( + Item.description == "MySQL Crowbar", + Item.price > OrderItem.price, + ) + ) + + print([order.customer_name for order in session.scalars(q)]) diff --git a/examples/association/dict_of_sets_with_default.py b/examples/association/dict_of_sets_with_default.py index f515ab975b5..fef3c1d57a2 100644 --- a/examples/association/dict_of_sets_with_default.py +++ b/examples/association/dict_of_sets_with_default.py @@ -12,43 +12,46 @@ """ +from __future__ import annotations + import operator +from typing import Mapping -from sqlalchemy import Column from sqlalchemy import create_engine from sqlalchemy import ForeignKey -from sqlalchemy import Integer -from sqlalchemy import String +from sqlalchemy import select from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.associationproxy import AssociationProxy +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column from sqlalchemy.orm import relationship from sqlalchemy.orm import Session from sqlalchemy.orm.collections import KeyFuncDict -class Base: - id = Column(Integer, primary_key=True) - +class Base(DeclarativeBase): + id: Mapped[int] = mapped_column(primary_key=True) -Base = declarative_base(cls=Base) - -class GenDefaultCollection(KeyFuncDict): - def __missing__(self, key): +class GenDefaultCollection(KeyFuncDict[str, "B"]): + def __missing__(self, key: str) -> B: self[key] = b = B(key) return b class A(Base): __tablename__ = "a" - associations = relationship( + associations: Mapped[Mapping[str, B]] = relationship( "B", collection_class=lambda: GenDefaultCollection( operator.attrgetter("key") ), ) - collections = association_proxy("associations", "values") + collections: AssociationProxy[dict[str, set[int]]] = association_proxy( + "associations", "values" + ) """Bridge the association from 'associations' over to the 'values' association proxy of B. """ @@ -56,15 +59,15 @@ class A(Base): class B(Base): __tablename__ = "b" - a_id = Column(Integer, ForeignKey("a.id"), nullable=False) - elements = relationship("C", collection_class=set) - key = Column(String) + a_id: Mapped[int] = mapped_column(ForeignKey("a.id")) + elements: Mapped[set[C]] = relationship("C", collection_class=set) + key: Mapped[str] - values = association_proxy("elements", "value") + values: AssociationProxy[set[int]] = association_proxy("elements", "value") """Bridge the association from 'elements' over to the 'value' element of C.""" - def __init__(self, key, values=None): + def __init__(self, key: str, values: set[int] | None = None) -> None: self.key = key if values: self.values = values @@ -72,10 +75,10 @@ def __init__(self, key, values=None): class C(Base): __tablename__ = "c" - b_id = Column(Integer, ForeignKey("b.id"), nullable=False) - value = Column(Integer) + b_id: Mapped[int] = mapped_column(ForeignKey("b.id")) + value: Mapped[int] - def __init__(self, value): + def __init__(self, value: int) -> None: self.value = value @@ -90,7 +93,7 @@ def __init__(self, value): session.add_all([A(collections={"1": {1, 2, 3}})]) session.commit() - a1 = session.query(A).first() + a1 = session.scalars(select(A)).one() print(a1.collections["1"]) a1.collections["1"].add(4) session.commit() diff --git a/examples/association/proxied_association.py b/examples/association/proxied_association.py index 65dcd6c0b66..0f18e167eba 100644 --- a/examples/association/proxied_association.py +++ b/examples/association/proxied_association.py @@ -5,116 +5,127 @@ """ +from __future__ import annotations + from datetime import datetime -from sqlalchemy import Column from sqlalchemy import create_engine -from sqlalchemy import DateTime -from sqlalchemy import Float from sqlalchemy import ForeignKey -from sqlalchemy import Integer +from sqlalchemy import select from sqlalchemy import String from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.associationproxy import AssociationProxy +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column from sqlalchemy.orm import relationship from sqlalchemy.orm import Session -Base = declarative_base() +class Base(DeclarativeBase): + pass class Order(Base): __tablename__ = "order" - order_id = Column(Integer, primary_key=True) - customer_name = Column(String(30), nullable=False) - order_date = Column(DateTime, nullable=False, default=datetime.now()) - order_items = relationship( - "OrderItem", cascade="all, delete-orphan", backref="order" + order_id: Mapped[int] = mapped_column(primary_key=True) + customer_name: Mapped[str] = mapped_column(String(30)) + order_date: Mapped[datetime] = mapped_column(default=datetime.now()) + order_items: Mapped[list[OrderItem]] = relationship( + cascade="all, delete-orphan", backref="order" + ) + items: AssociationProxy[list[Item]] = association_proxy( + "order_items", "item" ) - items = association_proxy("order_items", "item") - def __init__(self, customer_name): + def __init__(self, customer_name: str) -> None: self.customer_name = customer_name class Item(Base): __tablename__ = "item" - item_id = Column(Integer, primary_key=True) - description = Column(String(30), nullable=False) - price = Column(Float, nullable=False) + item_id: Mapped[int] = mapped_column(primary_key=True) + description: Mapped[str] = mapped_column(String(30)) + price: Mapped[float] - def __init__(self, description, price): + def __init__(self, description: str, price: float) -> None: self.description = description self.price = price - def __repr__(self): - return "Item(%r, %r)" % (self.description, self.price) + def __repr__(self) -> str: + return "Item({!r}, {!r})".format(self.description, self.price) class OrderItem(Base): __tablename__ = "orderitem" - order_id = Column(Integer, ForeignKey("order.order_id"), primary_key=True) - item_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True) - price = Column(Float, nullable=False) + order_id: Mapped[int] = mapped_column( + ForeignKey("order.order_id"), primary_key=True + ) + item_id: Mapped[int] = mapped_column( + ForeignKey("item.item_id"), primary_key=True + ) + price: Mapped[float] + + item: Mapped[Item] = relationship(lazy="joined") - def __init__(self, item, price=None): + def __init__(self, item: Item, price: float | None = None): self.item = item self.price = price or item.price - item = relationship(Item, lazy="joined") - if __name__ == "__main__": engine = create_engine("sqlite://") Base.metadata.create_all(engine) - session = Session(engine) - - # create catalog - tshirt, mug, hat, crowbar = ( - Item("SA T-Shirt", 10.99), - Item("SA Mug", 6.50), - Item("SA Hat", 8.99), - Item("MySQL Crowbar", 16.99), - ) - session.add_all([tshirt, mug, hat, crowbar]) - session.commit() - - # create an order - order = Order("john smith") - - # add items via the association proxy. - # the OrderItem is created automatically. - order.items.append(mug) - order.items.append(hat) - - # add an OrderItem explicitly. - order.order_items.append(OrderItem(crowbar, 10.99)) - - session.add(order) - session.commit() - - # query the order, print items - order = session.query(Order).filter_by(customer_name="john smith").one() - - # print items based on the OrderItem collection directly - print( - [ - (assoc.item.description, assoc.price, assoc.item.price) - for assoc in order.order_items - ] - ) - - # print items based on the "proxied" items collection - print([(item.description, item.price) for item in order.items]) - - # print customers who bought 'MySQL Crowbar' on sale - orders = ( - session.query(Order) - .join(OrderItem) - .join(Item) - .filter(Item.description == "MySQL Crowbar") - .filter(Item.price > OrderItem.price) - ) - print([o.customer_name for o in orders]) + with Session(engine) as session: + + # create catalog + tshirt, mug, hat, crowbar = ( + Item("SA T-Shirt", 10.99), + Item("SA Mug", 6.50), + Item("SA Hat", 8.99), + Item("MySQL Crowbar", 16.99), + ) + session.add_all([tshirt, mug, hat, crowbar]) + session.commit() + + # create an order + order = Order("john smith") + + # add items via the association proxy. + # the OrderItem is created automatically. + order.items.append(mug) + order.items.append(hat) + + # add an OrderItem explicitly. + order.order_items.append(OrderItem(crowbar, 10.99)) + + session.add(order) + session.commit() + + # query the order, print items + order = session.scalars( + select(Order).filter_by(customer_name="john smith") + ).one() + + # print items based on the OrderItem collection directly + print( + [ + (assoc.item.description, assoc.price, assoc.item.price) + for assoc in order.order_items + ] + ) + + # print items based on the "proxied" items collection + print([(item.description, item.price) for item in order.items]) + + # print customers who bought 'MySQL Crowbar' on sale + orders_stmt = ( + select(Order) + .join(OrderItem) + .join(Item) + .filter(Item.description == "MySQL Crowbar") + .filter(Item.price > OrderItem.price) + ) + print([o.customer_name for o in session.scalars(orders_stmt)]) diff --git a/examples/dogpile_caching/helloworld.py b/examples/dogpile_caching/helloworld.py index 01934c59fab..df1c2a318ef 100644 --- a/examples/dogpile_caching/helloworld.py +++ b/examples/dogpile_caching/helloworld.py @@ -1,6 +1,4 @@ -"""Illustrate how to load some data, and cache the results. - -""" +"""Illustrate how to load some data, and cache the results.""" from sqlalchemy import select from .caching_query import FromCache diff --git a/examples/dynamic_dict/__init__.py b/examples/dynamic_dict/__init__.py index ed31df062fb..c1d52d3c430 100644 --- a/examples/dynamic_dict/__init__.py +++ b/examples/dynamic_dict/__init__.py @@ -1,4 +1,4 @@ -""" Illustrates how to place a dictionary-like facade on top of a +"""Illustrates how to place a dictionary-like facade on top of a "dynamic" relation, so that dictionary operations (assuming simple string keys) can operate upon a large collection without loading the full collection at once. diff --git a/examples/generic_associations/discriminator_on_association.py b/examples/generic_associations/discriminator_on_association.py index 93c1b29ef98..ed32b7a7884 100644 --- a/examples/generic_associations/discriminator_on_association.py +++ b/examples/generic_associations/discriminator_on_association.py @@ -16,43 +16,48 @@ """ -from sqlalchemy import Column +from __future__ import annotations + +from typing import Any +from typing import TYPE_CHECKING + from sqlalchemy import create_engine from sqlalchemy import ForeignKey -from sqlalchemy import Integer -from sqlalchemy import String from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.ext.declarative import as_declarative -from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.ext.associationproxy import AssociationProxy from sqlalchemy.orm import backref +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import declared_attr +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column from sqlalchemy.orm import relationship from sqlalchemy.orm import Session -@as_declarative() -class Base: +class Base(DeclarativeBase): """Base class which provides automated table name and surrogate primary key column. - """ - @declared_attr - def __tablename__(cls): + @declared_attr.directive + def __tablename__(cls) -> str: return cls.__name__.lower() - id = Column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(primary_key=True) class AddressAssociation(Base): """Associates a collection of Address objects with a particular parent. - """ __tablename__ = "address_association" - discriminator = Column(String) + discriminator: Mapped[str] = mapped_column() """Refers to the type of parent.""" + addresses: Mapped[list[Address]] = relationship( + back_populates="association" + ) __mapper_args__ = {"polymorphic_on": discriminator} @@ -62,18 +67,23 @@ class Address(Base): This represents all address records in a single table. - """ - association_id = Column(Integer, ForeignKey("address_association.id")) - street = Column(String) - city = Column(String) - zip = Column(String) - association = relationship("AddressAssociation", backref="addresses") - - parent = association_proxy("association", "parent") - - def __repr__(self): + association_id: Mapped[int] = mapped_column( + ForeignKey("address_association.id") + ) + street: Mapped[str] + city: Mapped[str] + zip: Mapped[str] + association: Mapped[AddressAssociation] = relationship( + back_populates="addresses" + ) + + parent: AssociationProxy[HasAddresses] = association_proxy( + "association", "parent" + ) + + def __repr__(self) -> str: return "%s(street=%r, city=%r, zip=%r)" % ( self.__class__.__name__, self.street, @@ -85,20 +95,22 @@ def __repr__(self): class HasAddresses: """HasAddresses mixin, creates a relationship to the address_association table for each parent. - """ + if TYPE_CHECKING: + addresses: AssociationProxy[list[Address]] + @declared_attr - def address_association_id(cls): - return Column(Integer, ForeignKey("address_association.id")) + def address_association_id(cls: type[Any]) -> Mapped[int]: + return mapped_column(ForeignKey("address_association.id")) @declared_attr - def address_association(cls): + def address_association(cls: type[Any]) -> Mapped[AddressAssociation]: name = cls.__name__ discriminator = name.lower() assoc_cls = type( - "%sAddressAssociation" % name, + f"{name}AddressAssociation", (AddressAssociation,), dict( __tablename__=None, @@ -117,11 +129,11 @@ def address_association(cls): class Customer(HasAddresses, Base): - name = Column(String) + name: Mapped[str] class Supplier(HasAddresses, Base): - company_name = Column(String) + company_name: Mapped[str] engine = create_engine("sqlite://", echo=True) diff --git a/examples/generic_associations/generic_fk.py b/examples/generic_associations/generic_fk.py index d45166d333f..fd8d067e307 100644 --- a/examples/generic_associations/generic_fk.py +++ b/examples/generic_associations/generic_fk.py @@ -18,33 +18,37 @@ """ +from __future__ import annotations + +from typing import Any +from typing import cast +from typing import TYPE_CHECKING + from sqlalchemy import and_ -from sqlalchemy import Column from sqlalchemy import create_engine from sqlalchemy import event -from sqlalchemy import Integer -from sqlalchemy import String -from sqlalchemy.ext.declarative import as_declarative -from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import backref +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import declared_attr from sqlalchemy.orm import foreign +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column +from sqlalchemy.orm import Mapper from sqlalchemy.orm import relationship from sqlalchemy.orm import remote from sqlalchemy.orm import Session -@as_declarative() -class Base: +class Base(DeclarativeBase): """Base class which provides automated table name and surrogate primary key column. - """ - @declared_attr - def __tablename__(cls): + @declared_attr.directive + def __tablename__(cls) -> str: return cls.__name__.lower() - id = Column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(primary_key=True) class Address(Base): @@ -52,31 +56,31 @@ class Address(Base): This represents all address records in a single table. - """ - street = Column(String) - city = Column(String) - zip = Column(String) + street: Mapped[str] + city: Mapped[str] + zip: Mapped[str] - discriminator = Column(String) + discriminator: Mapped[str] """Refers to the type of parent.""" - parent_id = Column(Integer) + parent_id: Mapped[int] """Refers to the primary key of the parent. This could refer to any table. """ @property - def parent(self): + def parent(self) -> HasAddresses: """Provides in-Python access to the "parent" by choosing the appropriate relationship. - """ - return getattr(self, "parent_%s" % self.discriminator) + return cast( + HasAddresses, getattr(self, f"parent_{self.discriminator}") + ) - def __repr__(self): + def __repr__(self) -> str: return "%s(street=%r, city=%r, zip=%r)" % ( self.__class__.__name__, self.street, @@ -91,9 +95,12 @@ class HasAddresses: """ + if TYPE_CHECKING: + addresses: Mapped[list[Address]] + @event.listens_for(HasAddresses, "mapper_configured", propagate=True) -def setup_listener(mapper, class_): +def setup_listener(mapper: Mapper[Any], class_: type[Any]) -> None: name = class_.__name__ discriminator = name.lower() class_.addresses = relationship( @@ -105,20 +112,24 @@ def setup_listener(mapper, class_): backref=backref( "parent_%s" % discriminator, primaryjoin=remote(class_.id) == foreign(Address.parent_id), + overlaps="addresses, parent_customer", ), + overlaps="addresses", ) @event.listens_for(class_.addresses, "append") - def append_address(target, value, initiator): + def append_address( + target: HasAddresses, value: Address, initiator: Any + ) -> None: value.discriminator = discriminator class Customer(HasAddresses, Base): - name = Column(String) + name: Mapped[str] class Supplier(HasAddresses, Base): - company_name = Column(String) + company_name: Mapped[str] engine = create_engine("sqlite://", echo=True) diff --git a/examples/generic_associations/table_per_association.py b/examples/generic_associations/table_per_association.py index 04786bd49be..2d03532d8fd 100644 --- a/examples/generic_associations/table_per_association.py +++ b/examples/generic_associations/table_per_association.py @@ -12,30 +12,30 @@ """ +from __future__ import annotations + from sqlalchemy import Column from sqlalchemy import create_engine from sqlalchemy import ForeignKey -from sqlalchemy import Integer -from sqlalchemy import String from sqlalchemy import Table -from sqlalchemy.ext.declarative import as_declarative -from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import declared_attr +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column from sqlalchemy.orm import relationship from sqlalchemy.orm import Session -@as_declarative() -class Base: +class Base(DeclarativeBase): """Base class which provides automated table name and surrogate primary key column. - """ - @declared_attr - def __tablename__(cls): + @declared_attr.directive + def __tablename__(cls) -> str: return cls.__name__.lower() - id = Column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(primary_key=True) class Address(Base): @@ -43,14 +43,13 @@ class Address(Base): This represents all address records in a single table. - """ - street = Column(String) - city = Column(String) - zip = Column(String) + street: Mapped[str] + city: Mapped[str] + zip: Mapped[str] - def __repr__(self): + def __repr__(self) -> str: return "%s(street=%r, city=%r, zip=%r)" % ( self.__class__.__name__, self.street, @@ -66,7 +65,7 @@ class HasAddresses: """ @declared_attr - def addresses(cls): + def addresses(cls: type[DeclarativeBase]) -> Mapped[list[Address]]: address_association = Table( "%s_addresses" % cls.__tablename__, cls.metadata, @@ -81,11 +80,11 @@ def addresses(cls): class Customer(HasAddresses, Base): - name = Column(String) + name: Mapped[str] class Supplier(HasAddresses, Base): - company_name = Column(String) + company_name: Mapped[str] engine = create_engine("sqlite://", echo=True) diff --git a/examples/generic_associations/table_per_related.py b/examples/generic_associations/table_per_related.py index 23c75b0b9d6..f84d89e0fed 100644 --- a/examples/generic_associations/table_per_related.py +++ b/examples/generic_associations/table_per_related.py @@ -17,29 +17,34 @@ """ -from sqlalchemy import Column +from __future__ import annotations + +from typing import Any +from typing import ClassVar +from typing import TYPE_CHECKING + from sqlalchemy import create_engine from sqlalchemy import ForeignKey from sqlalchemy import Integer -from sqlalchemy import String -from sqlalchemy.ext.declarative import as_declarative -from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import declared_attr +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column from sqlalchemy.orm import relationship from sqlalchemy.orm import Session -@as_declarative() -class Base: +class Base(DeclarativeBase): """Base class which provides automated table name and surrogate primary key column. """ - @declared_attr - def __tablename__(cls): + @declared_attr.directive + def __tablename__(cls) -> str: return cls.__name__.lower() - id = Column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(primary_key=True) class Address: @@ -52,11 +57,11 @@ class Address: """ - street = Column(String) - city = Column(String) - zip = Column(String) + street: Mapped[str] + city: Mapped[str] + zip: Mapped[str] - def __repr__(self): + def __repr__(self) -> str: return "%s(street=%r, city=%r, zip=%r)" % ( self.__class__.__name__, self.street, @@ -65,34 +70,53 @@ def __repr__(self): ) +if TYPE_CHECKING: + + class AddressWithParent(Address): + """Type stub for Address subclasses created by HasAddresses. + + Inherits street, city, zip from Address. + + Allows mypy to understand when .Address is created, + it will have `parent_id` and `parent` attributes. + If you won't use `parent_id` attribute directly, + there's no need to specify here, included for completeness. + """ + + parent_id: int + parent: HasAddresses + + class HasAddresses: """HasAddresses mixin, creates a new Address class for each parent. """ + Address: ClassVar[type] + @declared_attr - def addresses(cls): + def addresses(cls: type[Any]) -> Mapped[list[AddressWithParent]]: cls.Address = type( - "%sAddress" % cls.__name__, + f"{cls.__name__}Address", (Address, Base), dict( - __tablename__="%s_address" % cls.__tablename__, - parent_id=Column( - Integer, ForeignKey("%s.id" % cls.__tablename__) + __tablename__=f"{cls.__tablename__}_address", + parent_id=mapped_column( + Integer, ForeignKey(f"{cls.__tablename__}.id") ), - parent=relationship(cls), + parent=relationship(cls, overlaps="addresses"), ), ) return relationship(cls.Address) class Customer(HasAddresses, Base): - name = Column(String) + name: Mapped[str] class Supplier(HasAddresses, Base): - company_name = Column(String) + company_name: Mapped[str] engine = create_engine("sqlite://", echo=True) diff --git a/examples/nested_sets/__init__.py b/examples/nested_sets/__init__.py index 5fdfbcedc08..cacab411b9a 100644 --- a/examples/nested_sets/__init__.py +++ b/examples/nested_sets/__init__.py @@ -1,4 +1,4 @@ -""" Illustrates a rudimentary way to implement the "nested sets" +"""Illustrates a rudimentary way to implement the "nested sets" pattern for hierarchical data using the SQLAlchemy ORM. .. autosource:: diff --git a/examples/nested_sets/nested_sets.py b/examples/nested_sets/nested_sets.py index 1492f6abd89..eed7b497a95 100644 --- a/examples/nested_sets/nested_sets.py +++ b/examples/nested_sets/nested_sets.py @@ -44,7 +44,7 @@ def before_insert(mapper, connection, instance): instance.left = 1 instance.right = 2 else: - personnel = mapper.mapped_table + personnel = mapper.persist_selectable right_most_sibling = connection.scalar( select(personnel.c.rgt).where( personnel.c.emp == instance.parent.emp diff --git a/examples/performance/short_selects.py b/examples/performance/short_selects.py index bc6a9c79ac4..6ff187bf0f9 100644 --- a/examples/performance/short_selects.py +++ b/examples/performance/short_selects.py @@ -13,12 +13,10 @@ from sqlalchemy import Integer from sqlalchemy import select from sqlalchemy import String -from sqlalchemy.ext import baked from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.future import select as future_select from sqlalchemy.orm import deferred from sqlalchemy.orm import Session -from sqlalchemy.sql import lambdas from . import Profiler @@ -46,7 +44,7 @@ class Customer(Base): @Profiler.setup def setup_database(dburl, echo, num): global engine - engine = create_engine(dburl, echo=echo) + engine = create_engine(dburl, echo=echo, query_cache_size=0) Base.metadata.drop_all(engine) Base.metadata.create_all(engine) sess = Session(engine) @@ -69,47 +67,45 @@ def setup_database(dburl, echo, num): @Profiler.profile def test_orm_query_classic_style(n): - """classic ORM query of the full entity.""" + """classic ORM query of the full entity, no cache""" session = Session(bind=engine) for id_ in random.sample(ids, n): session.query(Customer).filter(Customer.id == id_).one() @Profiler.profile -def test_orm_query_new_style(n): - """new style ORM select() of the full entity.""" - - session = Session(bind=engine) +def test_orm_query_classic_style_w_cache(n): + """classic ORM query of the full entity, using cache""" + cache = {} + session = Session(bind=engine.execution_options(compiled_cache=cache)) for id_ in random.sample(ids, n): - stmt = future_select(Customer).where(Customer.id == id_) - session.execute(stmt).scalar_one() + session.query(Customer).filter(Customer.id == id_).one() @Profiler.profile -def test_orm_query_new_style_using_embedded_lambdas(n): - """new style ORM select() of the full entity w/ embedded lambdas.""" +def test_orm_query_new_style(n): + """new style ORM select() of the full entity, no cache.""" + session = Session(bind=engine) for id_ in random.sample(ids, n): - stmt = future_select(lambda: Customer).where( - lambda: Customer.id == id_ - ) + stmt = future_select(Customer).where(Customer.id == id_) session.execute(stmt).scalar_one() @Profiler.profile -def test_orm_query_new_style_using_external_lambdas(n): - """new style ORM select() of the full entity w/ external lambdas.""" +def test_orm_query_new_style_cache(n): + """new style ORM select() of the full entity, using cache.""" - session = Session(bind=engine) + cache = {} + session = Session(bind=engine.execution_options(compiled_cache=cache)) for id_ in random.sample(ids, n): - stmt = lambdas.lambda_stmt(lambda: future_select(Customer)) - stmt += lambda s: s.where(Customer.id == id_) + stmt = future_select(Customer).where(Customer.id == id_) session.execute(stmt).scalar_one() @Profiler.profile def test_orm_query_classic_style_cols_only(n): - """classic ORM query against columns""" + """classic ORM query against columns, no cache""" session = Session(bind=engine) for id_ in random.sample(ids, n): session.query(Customer.id, Customer.name, Customer.description).filter( @@ -118,45 +114,19 @@ def test_orm_query_classic_style_cols_only(n): @Profiler.profile -def test_orm_query_new_style_ext_lambdas_cols_only(n): - """new style ORM query w/ external lambdas against columns.""" - s = Session(bind=engine) +def test_orm_query_classic_style_cols_only_cache(n): + """classic ORM query against columns, using cache""" + cache = {} + session = Session(bind=engine.execution_options(compiled_cache=cache)) for id_ in random.sample(ids, n): - stmt = lambdas.lambda_stmt( - lambda: future_select( - Customer.id, Customer.name, Customer.description - ) - ) + (lambda s: s.filter(Customer.id == id_)) - s.execute(stmt).one() - - -@Profiler.profile -def test_baked_query(n): - """test a baked query of the full entity.""" - bakery = baked.bakery() - s = Session(bind=engine) - for id_ in random.sample(ids, n): - q = bakery(lambda s: s.query(Customer)) - q += lambda q: q.filter(Customer.id == bindparam("id")) - q(s).params(id=id_).one() - - -@Profiler.profile -def test_baked_query_cols_only(n): - """test a baked query of only the entity columns.""" - bakery = baked.bakery() - s = Session(bind=engine) - for id_ in random.sample(ids, n): - q = bakery( - lambda s: s.query(Customer.id, Customer.name, Customer.description) - ) - q += lambda q: q.filter(Customer.id == bindparam("id")) - q(s).params(id=id_).one() + session.query(Customer.id, Customer.name, Customer.description).filter( + Customer.id == id_ + ).one() @Profiler.profile def test_core_new_stmt_each_time(n): - """test core, creating a new statement each time.""" + """test core, creating a new statement each time, no cache""" with engine.connect() as conn: for id_ in random.sample(ids, n): @@ -167,7 +137,7 @@ def test_core_new_stmt_each_time(n): @Profiler.profile def test_core_new_stmt_each_time_compiled_cache(n): - """test core, creating a new statement each time, but using the cache.""" + """test core, creating a new statement each time, using cache""" compiled_cache = {} with engine.connect().execution_options( @@ -181,7 +151,7 @@ def test_core_new_stmt_each_time_compiled_cache(n): @Profiler.profile def test_core_reuse_stmt(n): - """test core, reusing the same statement (but recompiling each time).""" + """test core, reusing the same statement, no cache""" stmt = select(Customer.__table__).where(Customer.id == bindparam("id")) with engine.connect() as conn: @@ -192,7 +162,7 @@ def test_core_reuse_stmt(n): @Profiler.profile def test_core_reuse_stmt_compiled_cache(n): - """test core, reusing the same statement + compiled cache.""" + """test core, reusing the same statement, using cache""" stmt = select(Customer.__table__).where(Customer.id == bindparam("id")) compiled_cache = {} diff --git a/examples/versioned_history/history_meta.py b/examples/versioned_history/history_meta.py index 88fb16a0049..ab6e3583dd6 100644 --- a/examples/versioned_history/history_meta.py +++ b/examples/versioned_history/history_meta.py @@ -179,7 +179,7 @@ def default_version_from_history(context): "version", Integer, # if rows are not being deleted from the main table with - # subsequent re-use of primary key, this default can be + # subsequent reuse of primary key, this default can be # "1" instead of running a query per INSERT default=default_version_from_history, nullable=False, diff --git a/lib/sqlalchemy/__init__.py b/lib/sqlalchemy/__init__.py index 53c1dbb7d19..8912040b045 100644 --- a/lib/sqlalchemy/__init__.py +++ b/lib/sqlalchemy/__init__.py @@ -1,5 +1,5 @@ # __init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -57,13 +57,19 @@ from .schema import BaseDDLElement as BaseDDLElement from .schema import BLANK_SCHEMA as BLANK_SCHEMA from .schema import CheckConstraint as CheckConstraint +from .schema import CheckFirst as CheckFirst from .schema import Column as Column from .schema import ColumnDefault as ColumnDefault from .schema import Computed as Computed from .schema import Constraint as Constraint +from .schema import CreateTable as CreateTable +from .schema import CreateTableAs as CreateTableAs +from .schema import CreateView as CreateView from .schema import DDL as DDL from .schema import DDLElement as DDLElement from .schema import DefaultClause as DefaultClause +from .schema import DropTable as DropTable +from .schema import DropView as DropView from .schema import ExecutableDDLElement as ExecutableDDLElement from .schema import FetchedValue as FetchedValue from .schema import ForeignKey as ForeignKey @@ -72,14 +78,18 @@ from .schema import Index as Index from .schema import insert_sentinel as insert_sentinel from .schema import MetaData as MetaData +from .schema import Named as Named from .schema import PrimaryKeyConstraint as PrimaryKeyConstraint from .schema import Sequence as Sequence from .schema import Table as Table +from .schema import TypedColumns as TypedColumns from .schema import UniqueConstraint as UniqueConstraint from .sql import ColumnExpressionArgument as ColumnExpressionArgument from .sql import NotNullable as NotNullable from .sql import Nullable as Nullable from .sql import SelectLabelStyle as SelectLabelStyle +from .sql.expression import aggregate_order_by as aggregate_order_by +from .sql.expression import AggregateOrderBy as AggregateOrderBy from .sql.expression import Alias as Alias from .sql.expression import alias as alias from .sql.expression import AliasedReturnsRows as AliasedReturnsRows @@ -124,6 +134,9 @@ from .sql.expression import extract as extract from .sql.expression import false as false from .sql.expression import False_ as False_ +from .sql.expression import FrameClause as FrameClause +from .sql.expression import FrameClauseType as FrameClauseType +from .sql.expression import from_dml_column as from_dml_column from .sql.expression import FromClause as FromClause from .sql.expression import FromGrouping as FromGrouping from .sql.expression import func as func @@ -168,6 +181,7 @@ from .sql.expression import nullslast as nullslast from .sql.expression import Operators as Operators from .sql.expression import or_ as or_ +from .sql.expression import OrderByList as OrderByList from .sql.expression import outerjoin as outerjoin from .sql.expression import outparam as outparam from .sql.expression import Over as Over @@ -192,6 +206,7 @@ from .sql.expression import TableSample as TableSample from .sql.expression import tablesample as tablesample from .sql.expression import TableValuedAlias as TableValuedAlias +from .sql.expression import TableValuedColumn as TableValuedColumn from .sql.expression import text as text from .sql.expression import TextAsFrom as TextAsFrom from .sql.expression import TextClause as TextClause @@ -200,6 +215,8 @@ from .sql.expression import True_ as True_ from .sql.expression import try_cast as try_cast from .sql.expression import TryCast as TryCast +from .sql.expression import TString as TString +from .sql.expression import tstring as tstring from .sql.expression import Tuple as Tuple from .sql.expression import tuple_ as tuple_ from .sql.expression import type_coerce as type_coerce @@ -267,7 +284,7 @@ from .types import VARBINARY as VARBINARY from .types import VARCHAR as VARCHAR -__version__ = "2.1.0b1" +__version__ = "2.1.0b3" def __go(lcls: Any) -> None: @@ -279,14 +296,3 @@ def __go(lcls: Any) -> None: __go(locals()) - - -def __getattr__(name: str) -> Any: - if name == "SingleonThreadPool": - _util.warn_deprecated( - "SingleonThreadPool was a typo in the v2 series. " - "Please use the correct SingletonThreadPool name.", - "2.0.24", - ) - return SingletonThreadPool - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/lib/sqlalchemy/connectors/__init__.py b/lib/sqlalchemy/connectors/__init__.py index 43cd1035c62..eea1b29a7df 100644 --- a/lib/sqlalchemy/connectors/__init__.py +++ b/lib/sqlalchemy/connectors/__init__.py @@ -1,5 +1,5 @@ # connectors/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/connectors/aioodbc.py b/lib/sqlalchemy/connectors/aioodbc.py index 57a16d72018..e1e4fba6194 100644 --- a/lib/sqlalchemy/connectors/aioodbc.py +++ b/lib/sqlalchemy/connectors/aioodbc.py @@ -1,5 +1,5 @@ # connectors/aioodbc.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -14,6 +14,7 @@ from .asyncio import AsyncAdapt_dbapi_cursor from .asyncio import AsyncAdapt_dbapi_ss_cursor from .pyodbc import PyODBCConnector +from ..connectors.asyncio import AsyncAdapt_dbapi_module from ..util.concurrency import await_ if TYPE_CHECKING: @@ -31,6 +32,14 @@ def setinputsizes(self, *inputsizes): # how it's supposed to work # return await_(self._cursor.setinputsizes(*inputsizes)) + @property + def fast_executemany(self): + return self._cursor._impl.fast_executemany + + @fast_executemany.setter + def fast_executemany(self, value): + self._cursor._impl.fast_executemany = value + class AsyncAdapt_aioodbc_ss_cursor( AsyncAdapt_aioodbc_cursor, AsyncAdapt_dbapi_ss_cursor @@ -92,8 +101,9 @@ def close(self): super().close() -class AsyncAdapt_aioodbc_dbapi: +class AsyncAdapt_aioodbc_dbapi(AsyncAdapt_dbapi_module): def __init__(self, aioodbc, pyodbc): + super().__init__(aioodbc, dbapi_module=pyodbc) self.aioodbc = aioodbc self.pyodbc = pyodbc self.paramstyle = pyodbc.paramstyle @@ -114,6 +124,7 @@ def _init_dbapi_attributes(self): "ProgrammingError", "InternalError", "NotSupportedError", + "SQL_DRIVER_NAME", "NUMBER", "STRING", "DATETIME", @@ -122,15 +133,18 @@ def _init_dbapi_attributes(self): "BinaryNull", "SQL_VARCHAR", "SQL_WVARCHAR", + "SQL_DECIMAL", ): setattr(self, name, getattr(self.pyodbc, name)) def connect(self, *arg, **kw): creator_fn = kw.pop("async_creator_fn", self.aioodbc.connect) - return AsyncAdapt_aioodbc_connection( - self, - await_(creator_fn(*arg, **kw)), + return await_( + AsyncAdapt_aioodbc_connection.create( + self, + creator_fn(*arg, **kw), + ) ) diff --git a/lib/sqlalchemy/connectors/asyncio.py b/lib/sqlalchemy/connectors/asyncio.py index e57f7bfdf21..5a4fc7cc31b 100644 --- a/lib/sqlalchemy/connectors/asyncio.py +++ b/lib/sqlalchemy/connectors/asyncio.py @@ -1,5 +1,5 @@ # connectors/asyncio.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -12,21 +12,32 @@ import asyncio import collections import sys +import types from typing import Any from typing import AsyncIterator +from typing import Awaitable from typing import Deque from typing import Iterator from typing import NoReturn from typing import Optional from typing import Protocol from typing import Sequence +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING from ..engine import AdaptedConnection -from ..engine.interfaces import _DBAPICursorDescription -from ..engine.interfaces import _DBAPIMultiExecuteParams -from ..engine.interfaces import _DBAPISingleExecuteParams +from ..exc import EmulatedDBAPIException +from ..util import EMPTY_DICT from ..util.concurrency import await_ -from ..util.typing import Self +from ..util.concurrency import in_greenlet + +if TYPE_CHECKING: + from ..engine.interfaces import _DBAPICursorDescription + from ..engine.interfaces import _DBAPIMultiExecuteParams + from ..engine.interfaces import _DBAPISingleExecuteParams + from ..engine.interfaces import DBAPIModule + from ..util.typing import Self class AsyncIODBAPIConnection(Protocol): @@ -36,14 +47,19 @@ class AsyncIODBAPIConnection(Protocol): """ - async def close(self) -> None: ... + # note that async DBAPIs dont agree if close() should be awaitable, + # so it is omitted here and picked up by the __getattr__ hook below async def commit(self) -> None: ... - def cursor(self) -> AsyncIODBAPICursor: ... + def cursor(self, *args: Any, **kwargs: Any) -> AsyncIODBAPICursor: ... async def rollback(self) -> None: ... + def __getattr__(self, key: str) -> Any: ... + + def __setattr__(self, key: str, value: Any) -> None: ... + class AsyncIODBAPICursor(Protocol): """protocol representing an async adapted version @@ -101,6 +117,42 @@ async def nextset(self) -> Optional[bool]: ... def __aiter__(self) -> AsyncIterator[Any]: ... +class AsyncAdapt_dbapi_module: + if TYPE_CHECKING: + Error = DBAPIModule.Error + OperationalError = DBAPIModule.OperationalError + InterfaceError = DBAPIModule.InterfaceError + IntegrityError = DBAPIModule.IntegrityError + + def __getattr__(self, key: str) -> Any: ... + + def __init__( + self, + driver: types.ModuleType, + *, + dbapi_module: types.ModuleType | None = None, + ): + self.driver = driver + self.dbapi_module = dbapi_module + + @property + def exceptions_module(self) -> types.ModuleType: + """Return the module which we think will have the exception hierarchy. + + For an asyncio driver that wraps a plain DBAPI like aiomysql, + aioodbc, aiosqlite, etc. these exceptions will be from the + dbapi_module. For a "pure" driver like asyncpg these will come + from the driver module. + + .. versionadded:: 2.1 + + """ + if self.dbapi_module is not None: + return self.dbapi_module + else: + return self.driver + + class AsyncAdapt_dbapi_cursor: server_side = False __slots__ = ( @@ -108,8 +160,11 @@ class AsyncAdapt_dbapi_cursor: "_connection", "_cursor", "_rows", + "_soft_closed_memoized", ) + _awaitable_cursor_close: bool = True + _cursor: AsyncIODBAPICursor _adapt_connection: AsyncAdapt_dbapi_connection _connection: AsyncIODBAPIConnection @@ -121,7 +176,7 @@ def __init__(self, adapt_connection: AsyncAdapt_dbapi_connection): cursor = self._make_new_cursor(self._connection) self._cursor = self._aenter_cursor(cursor) - + self._soft_closed_memoized = EMPTY_DICT if not self.server_side: self._rows = collections.deque() @@ -138,6 +193,8 @@ def _make_new_cursor( @property def description(self) -> Optional[_DBAPICursorDescription]: + if "description" in self._soft_closed_memoized: + return self._soft_closed_memoized["description"] # type: ignore[no-any-return] # noqa: E501 return self._cursor.description @property @@ -156,11 +213,40 @@ def arraysize(self, value: int) -> None: def lastrowid(self) -> int: return self._cursor.lastrowid + async def _async_soft_close(self) -> None: + """close the cursor but keep the results pending, and memoize the + description. + + .. versionadded:: 2.0.44 + + """ + + if not self._awaitable_cursor_close or self.server_side: + return + + self._soft_closed_memoized = self._soft_closed_memoized.union( + { + "description": self._cursor.description, + } + ) + await self._cursor.close() + def close(self) -> None: - # note we aren't actually closing the cursor here, - # we are just letting GC do it. see notes in aiomysql dialect self._rows.clear() + # updated as of 2.0.44 + # try to "close" the cursor based on what we know about the driver + # and if we are able to. otherwise, hope that the asyncio + # extension called _async_soft_close() if the cursor is going into + # a sync context + if self._cursor is None or bool(self._soft_closed_memoized): + return + + if not self._awaitable_cursor_close: + self._cursor.close() # type: ignore[unused-coroutine] + elif in_greenlet(): + await_(self._cursor.close()) + def execute( self, operation: Any, @@ -210,7 +296,7 @@ def nextset(self) -> None: self._rows = collections.deque(await_(self._cursor.fetchall())) def setinputsizes(self, *inputsizes: Any) -> None: - # NOTE: this is overrridden in aioodbc due to + # NOTE: this is overridden in aioodbc due to # see https://github.com/aio-libs/aioodbc/issues/451 # right now @@ -279,6 +365,20 @@ class AsyncAdapt_dbapi_connection(AdaptedConnection): _connection: AsyncIODBAPIConnection + @classmethod + async def create( + cls, + dbapi: Any, + connection_awaitable: Awaitable[AsyncIODBAPIConnection], + **kw: Any, + ) -> Self: + try: + connection = await connection_awaitable + except Exception as error: + cls._handle_exception_no_connection(dbapi, error) + else: + return cls(dbapi, connection, **kw) + def __init__(self, dbapi: Any, connection: AsyncIODBAPIConnection): self.dbapi = dbapi self._connection = connection @@ -300,11 +400,17 @@ def execute( cursor.execute(operation, parameters) return cursor - def _handle_exception(self, error: Exception) -> NoReturn: + @classmethod + def _handle_exception_no_connection( + cls, dbapi: Any, error: Exception + ) -> NoReturn: exc_info = sys.exc_info() raise error.with_traceback(exc_info[2]) + def _handle_exception(self, error: Exception) -> NoReturn: + self._handle_exception_no_connection(self.dbapi, error) + def rollback(self) -> None: try: await_(self._connection.rollback()) @@ -319,3 +425,52 @@ def commit(self) -> None: def close(self) -> None: await_(self._connection.close()) + + +class AsyncAdapt_terminate: + """Mixin for a AsyncAdapt_dbapi_connection to add terminate support.""" + + __slots__ = () + + def terminate(self) -> None: + if in_greenlet(): + # in a greenlet; this is the connection was invalidated case. + try: + # try to gracefully close; see #10717 + await_(asyncio.shield(self._terminate_graceful_close())) + except self._terminate_handled_exceptions() as e: + # in the case where we are recycling an old connection + # that may have already been disconnected, close() will + # fail. In this case, terminate + # the connection without any further waiting. + # see issue #8419 + self._terminate_force_close() + if isinstance(e, asyncio.CancelledError): + # re-raise CancelledError if we were cancelled + raise + else: + # not in a greenlet; this is the gc cleanup case + self._terminate_force_close() + + def _terminate_handled_exceptions(self) -> Tuple[Type[BaseException], ...]: + """Returns the exceptions that should be handled when + calling _graceful_close. + """ + return (asyncio.TimeoutError, asyncio.CancelledError, OSError) + + async def _terminate_graceful_close(self) -> None: + """Try to close connection gracefully""" + raise NotImplementedError + + def _terminate_force_close(self) -> None: + """Terminate the connection""" + raise NotImplementedError + + +class AsyncAdapt_Error(EmulatedDBAPIException): + """Provide for the base of DBAPI ``Error`` base class for dialects + that need to emulate the DBAPI exception hierarchy. + + .. versionadded:: 2.1 + + """ diff --git a/lib/sqlalchemy/connectors/pyodbc.py b/lib/sqlalchemy/connectors/pyodbc.py index 3a32d19c8bb..1e90ed35631 100644 --- a/lib/sqlalchemy/connectors/pyodbc.py +++ b/lib/sqlalchemy/connectors/pyodbc.py @@ -1,5 +1,5 @@ # connectors/pyodbc.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -8,7 +8,6 @@ from __future__ import annotations import re -from types import ModuleType import typing from typing import Any from typing import Dict @@ -28,6 +27,7 @@ from ..sql.type_api import TypeEngine if typing.TYPE_CHECKING: + from ..engine.interfaces import DBAPIModule from ..engine.interfaces import IsolationLevel @@ -47,15 +47,13 @@ class PyODBCConnector(Connector): # hold the desired driver name pyodbc_driver_name: Optional[str] = None - dbapi: ModuleType - def __init__(self, use_setinputsizes: bool = False, **kw: Any): super().__init__(**kw) if use_setinputsizes: self.bind_typing = interfaces.BindTyping.SETINPUTSIZES @classmethod - def import_dbapi(cls) -> ModuleType: + def import_dbapi(cls) -> DBAPIModule: return __import__("pyodbc") def create_connect_args(self, url: URL) -> ConnectArgsType: @@ -150,7 +148,7 @@ def is_disconnect( ], cursor: Optional[interfaces.DBAPICursor], ) -> bool: - if isinstance(e, self.dbapi.ProgrammingError): + if isinstance(e, self.loaded_dbapi.ProgrammingError): return "The cursor's connection has been closed." in str( e ) or "Attempt to use a closed connection." in str(e) @@ -227,11 +225,9 @@ def do_set_input_sizes( ) def get_isolation_level_values( - self, dbapi_connection: interfaces.DBAPIConnection + self, dbapi_conn: interfaces.DBAPIConnection ) -> List[IsolationLevel]: - return super().get_isolation_level_values(dbapi_connection) + [ - "AUTOCOMMIT" - ] + return [*super().get_isolation_level_values(dbapi_conn), "AUTOCOMMIT"] def set_isolation_level( self, @@ -247,3 +243,8 @@ def set_isolation_level( else: dbapi_connection.autocommit = False super().set_isolation_level(dbapi_connection, level) + + def detect_autocommit_setting( + self, dbapi_conn: interfaces.DBAPIConnection + ) -> bool: + return bool(dbapi_conn.autocommit) diff --git a/lib/sqlalchemy/dialects/__init__.py b/lib/sqlalchemy/dialects/__init__.py index 31ce6d64b52..d6336c1aa55 100644 --- a/lib/sqlalchemy/dialects/__init__.py +++ b/lib/sqlalchemy/dialects/__init__.py @@ -1,5 +1,5 @@ # dialects/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -7,6 +7,7 @@ from __future__ import annotations +from typing import Any from typing import Callable from typing import Optional from typing import Type @@ -39,7 +40,7 @@ def _auto_fn(name: str) -> Optional[Callable[[], Type[Dialect]]]: # hardcoded. if mysql / mariadb etc were third party dialects # they would just publish all the entrypoints, which would actually # look much nicer. - module = __import__( + module: Any = __import__( "sqlalchemy.dialects.mysql.mariadb" ).dialects.mysql.mariadb return module.loader(driver) # type: ignore diff --git a/lib/sqlalchemy/dialects/_typing.py b/lib/sqlalchemy/dialects/_typing.py index 4dd40d7220f..bf9b67a0a06 100644 --- a/lib/sqlalchemy/dialects/_typing.py +++ b/lib/sqlalchemy/dialects/_typing.py @@ -1,5 +1,5 @@ # dialects/_typing.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/dialects/mssql/__init__.py b/lib/sqlalchemy/dialects/mssql/__init__.py index 20140fdddb3..3ad714921c8 100644 --- a/lib/sqlalchemy/dialects/mssql/__init__.py +++ b/lib/sqlalchemy/dialects/mssql/__init__.py @@ -1,5 +1,5 @@ # dialects/mssql/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -8,6 +8,7 @@ from . import aioodbc # noqa from . import base # noqa +from . import mssqlpython # noqa from . import pymssql # noqa from . import pyodbc # noqa from .base import BIGINT diff --git a/lib/sqlalchemy/dialects/mssql/aioodbc.py b/lib/sqlalchemy/dialects/mssql/aioodbc.py index 522ad1d6b0d..1139f37f798 100644 --- a/lib/sqlalchemy/dialects/mssql/aioodbc.py +++ b/lib/sqlalchemy/dialects/mssql/aioodbc.py @@ -1,5 +1,5 @@ # dialects/mssql/aioodbc.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -42,12 +42,12 @@ from __future__ import annotations +from .base import MSExecutionContext from .pyodbc import MSDialect_pyodbc -from .pyodbc import MSExecutionContext_pyodbc from ...connectors.aioodbc import aiodbcConnector -class MSExecutionContext_aioodbc(MSExecutionContext_pyodbc): +class MSExecutionContext_aioodbc(MSExecutionContext): def create_server_side_cursor(self): return self._dbapi_connection.cursor(server_side=True) diff --git a/lib/sqlalchemy/dialects/mssql/base.py b/lib/sqlalchemy/dialects/mssql/base.py index a2b9d37dadd..1dda1e9dc6c 100644 --- a/lib/sqlalchemy/dialects/mssql/base.py +++ b/lib/sqlalchemy/dialects/mssql/base.py @@ -1,5 +1,5 @@ # dialects/mssql/base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -100,14 +100,6 @@ ``dialect_options`` key in :meth:`_reflection.Inspector.get_columns`. Use the information in the ``identity`` key instead. -.. deprecated:: 1.3 - - The use of :class:`.Sequence` to specify IDENTITY characteristics is - deprecated and will be removed in a future release. Please use - the :class:`_schema.Identity` object parameters - :paramref:`_schema.Identity.start` and - :paramref:`_schema.Identity.increment`. - .. versionchanged:: 1.4 Removed the ability to use a :class:`.Sequence` object to modify IDENTITY characteristics. :class:`.Sequence` objects now only manipulate true T-SQL SEQUENCE types. @@ -168,13 +160,6 @@ addition to ``start`` and ``increment``. These are not supported by SQL Server and will be ignored when generating the CREATE TABLE ddl. -.. versionchanged:: 1.3.19 The :class:`_schema.Identity` object is - now used to affect the - ``IDENTITY`` generator for a :class:`_schema.Column` under SQL Server. - Previously, the :class:`.Sequence` object was used. As SQL Server now - supports real sequences as a separate construct, :class:`.Sequence` will be - functional in the normal way starting from SQLAlchemy version 1.4. - Using IDENTITY with Non-Integer numeric types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -717,10 +702,6 @@ def _reset_mssql(dbapi_connection, connection_record, reset_state): schema="[MyDataBase.Period].[MyOwner.Dot]", ) -.. versionchanged:: 1.2 the SQL Server dialect now treats brackets as - identifier delimiters splitting the schema into separate database - and owner tokens, to allow dots within either name itself. - .. _legacy_schema_rendering: Legacy Schema Mode @@ -880,8 +861,6 @@ def _reset_mssql(dbapi_connection, connection_record, reset_state): would render the index as ``CREATE INDEX my_index ON table (x) WHERE x > 10``. -.. versionadded:: 1.3.4 - Index ordering ^^^^^^^^^^^^^^ @@ -984,6 +963,8 @@ class MyClass(Base): import datetime import operator import re +from typing import Any +from typing import Literal from typing import overload from typing import TYPE_CHECKING from uuid import UUID as _python_UUID @@ -1014,6 +995,7 @@ class MyClass(Base): from ...sql import try_cast as try_cast # noqa: F401 from ...sql import util as sql_util from ...sql._typing import is_sql_compiler +from ...sql.compiler import AggregateOrderByStyle from ...sql.compiler import InsertmanyvaluesSentinelOpts from ...sql.elements import TryCast as TryCast # noqa: F401 from ...types import BIGINT @@ -1031,9 +1013,9 @@ class MyClass(Base): from ...types import TEXT from ...types import VARCHAR from ...util import update_wrapper -from ...util.typing import Literal if TYPE_CHECKING: + from ...sql.ddl import DropIndex from ...sql.dml import DMLState from ...sql.selectable import TableClause @@ -1407,8 +1389,6 @@ class TIMESTAMP(sqltypes._Binary): TIMESTAMP type, which is not supported by SQL Server. It is a read-only datatype that does not support INSERT of values. - .. versionadded:: 1.2 - .. seealso:: :class:`_mssql.ROWVERSION` @@ -1426,8 +1406,6 @@ def __init__(self, convert_int=False): :param convert_int: if True, binary integer values will be converted to integers on read. - .. versionadded:: 1.2 - """ self.convert_int = convert_int @@ -1461,8 +1439,6 @@ class ROWVERSION(TIMESTAMP): This is a read-only datatype that does not support INSERT of values. - .. versionadded:: 1.2 - .. seealso:: :class:`_mssql.TIMESTAMP` @@ -1624,7 +1600,7 @@ def __init__(self, as_uuid: bool = True): as Python uuid objects, converting to/from string via the DBAPI. - .. versionchanged: 2.0 Added direct "uuid" support to the + .. versionchanged:: 2.0 Added direct "uuid" support to the :class:`_mssql.UNIQUEIDENTIFIER` datatype; uuid interpretation defaults to ``True``. @@ -1862,6 +1838,7 @@ def visit_SQL_VARIANT(self, type_, **kw): class MSExecutionContext(default.DefaultExecutionContext): + _embedded_scope_identity = False _enable_identity_insert = False _select_lastrowid = False _lastrowid = None @@ -1924,15 +1901,46 @@ def pre_exec(self): self, ) + # don't embed the scope_identity select into an + # "INSERT .. DEFAULT VALUES" + if ( + self._select_lastrowid + and self.dialect.scope_identity_must_be_embedded + and self.dialect.use_scope_identity + and len(self.parameters[0]) + ): + self._embedded_scope_identity = True + + self.statement += "; select scope_identity()" + def post_exec(self): - """Disable IDENTITY_INSERT if enabled.""" conn = self.root_connection if self.isinsert or self.isupdate or self.isdelete: self._rowcount = self.cursor.rowcount - if self._select_lastrowid: + # handle INSERT with embedded SELECT SCOPE_IDENTITY() call + if self._embedded_scope_identity: + # Fetch the last inserted id from the manipulated statement + # We may have to skip over a number of result sets with + # no data (due to triggers, etc.) so run up to three times + + row = None + for _ in range(3): + if self.cursor.description: + rows = self.cursor.fetchall() + if rows: + row = rows[0] + break + else: + self.cursor.nextset() + + self._lastrowid = int(row[0]) if row else None + + self.cursor_fetch_strategy = _cursor._NO_CURSOR_DML + + elif self._select_lastrowid: if self.dialect.use_scope_identity: conn._cursor_execute( self.cursor, @@ -1963,6 +1971,7 @@ def post_exec(self): ) if self._enable_identity_insert: + # Disable IDENTITY_INSERT if enabled. if TYPE_CHECKING: assert is_sql_compiler(self.compiled) assert isinstance(self.compiled.compile_state, DMLState) @@ -2062,10 +2071,19 @@ def visit_char_length_func(self, fn, **kw): return "LEN%s" % self.function_argspec(fn, **kw) def visit_aggregate_strings_func(self, fn, **kw): - expr = fn.clauses.clauses[0]._compiler_dispatch(self, **kw) - kw["literal_execute"] = True - delimeter = fn.clauses.clauses[1]._compiler_dispatch(self, **kw) - return f"string_agg({expr}, {delimeter})" + cl = list(fn.clauses) + expr, delimiter = cl[0:2] + + literal_exec = dict(kw) + literal_exec["literal_execute"] = True + + return ( + f"string_agg({expr._compiler_dispatch(self, **kw)}, " + f"{delimiter._compiler_dispatch(self, **literal_exec)})" + ) + + def visit_pow_func(self, fn, **kw): + return f"POWER{self.function_argspec(fn)}" def visit_concat_op_expression_clauselist( self, clauselist, operator, **kw @@ -2503,7 +2521,12 @@ def _render_json_extract_from_binary(self, binary, operator, **kw): # the NULL handling is particularly weird with boolean, so # explicitly return numeric (BIT) constants type_expression = ( - "WHEN 'true' THEN 1 WHEN 'false' THEN 0 ELSE NULL" + "WHEN 'true' THEN 1 WHEN 'false' THEN 0 ELSE " + "CAST(JSON_VALUE(%s, %s) AS BIT)" + % ( + self.process(binary.left, **kw), + self.process(binary.right, **kw), + ) ) elif binary.type._type_affinity is sqltypes.String: # TODO: does this comment (from mysql) apply to here, too? @@ -2700,11 +2723,54 @@ def visit_create_index(self, create, include_schema=False, **kw): return text - def visit_drop_index(self, drop, **kw): - return "\nDROP INDEX %s ON %s" % ( - self._prepared_index_name(drop.element, include_schema=False), - self.preparer.format_table(drop.element.table), + def visit_drop_index(self, drop: DropIndex, **kw: Any) -> str: + index_name = self._prepared_index_name( + drop.element, include_schema=False ) + table_name = self.preparer.format_table(drop.element.table) + if_exists = " IF EXISTS" if drop.if_exists else "" + return f"\nDROP INDEX{if_exists} {index_name} ON {table_name}" + + def visit_create_table_as(self, element, **kw): + prep = self.preparer + + # SQL Server doesn't support CREATE TABLE AS, use SELECT INTO instead + # Format: SELECT columns INTO new_table FROM source WHERE ... + + qualified = prep.format_table(element.table) + + # Get the inner SELECT SQL + inner_kw = dict(kw) + inner_kw["literal_binds"] = True + select_sql = self.sql_compiler.process(element.selectable, **inner_kw) + + # Inject INTO clause before FROM keyword + # Find FROM position (case-insensitive) + select_upper = select_sql.upper() + from_idx = select_upper.find(" FROM ") + if from_idx == -1: + from_idx = select_upper.find("\nFROM ") + + if from_idx == -1: + raise exc.CompileError( + "Could not find FROM keyword in selectable for CREATE TABLE AS" + ) + + # Insert INTO clause before FROM + result = ( + select_sql[:from_idx] + + f"INTO {qualified} " + + select_sql[from_idx:] + ) + + return result + + def visit_create_view(self, create, **kw): + # SQL Server uses CREATE OR ALTER instead of CREATE OR REPLACE + result = super().visit_create_view(create, **kw) + if create.or_replace: + result = result.replace("CREATE OR REPLACE", "CREATE OR ALTER") + return result def visit_primary_key_constraint(self, constraint, **kw): if len(constraint) == 0: @@ -2851,23 +2917,9 @@ def _escape_identifier(self, value): def _unescape_identifier(self, value): return value.replace("]]", "]") - def quote_schema(self, schema, force=None): + def quote_schema(self, schema): """Prepare a quoted table and schema name.""" - # need to re-implement the deprecation warning entirely - if force is not None: - # not using the util.deprecated_params() decorator in this - # case because of the additional function call overhead on this - # very performance-critical spot. - util.warn_deprecated( - "The IdentifierPreparer.quote_schema.force parameter is " - "deprecated and will be removed in a future release. This " - "flag has no effect on the behavior of the " - "IdentifierPreparer.quote method; please refer to " - "quoted_name().", - version="1.3", - ) - dbname, owner = _schema_elements(schema) if dbname: result = "%s.%s" % (self.quote(dbname), self.quote(owner)) @@ -3007,6 +3059,7 @@ class MSDialect(default.DefaultDialect): supports_default_values = True supports_empty_insert = False favor_returning_over_lastrowid = True + scope_identity_must_be_embedded = False returns_native_bytes = True @@ -3018,6 +3071,8 @@ class MSDialect(default.DefaultDialect): """ + aggregate_order_by_style = AggregateOrderByStyle.WITHIN_GROUP + # supports_native_uuid is partial here, so we implement our # own impl type @@ -3238,6 +3293,7 @@ def get_isolation_level(self, dbapi_connection): 'attempting to query the "{}" view.'.format(err, view_name) ) from err else: + row = cursor.fetchone() return row[0].upper() finally: @@ -3250,13 +3306,6 @@ def initialize(self, connection): self._setup_supports_comments(connection) def _setup_version_attributes(self): - if self.server_version_info[0] not in list(range(8, 17)): - util.warn( - "Unrecognized server version info '%s'. Some SQL Server " - "features may not function properly." - % ".".join(str(x) for x in self.server_version_info) - ) - if self.server_version_info >= MS_2008_VERSION: self.supports_multivalues_insert = True else: @@ -3503,6 +3552,9 @@ def get_indexes(self, connection, tablename, dbname, owner, schema, **kw): where tab.name = :tabname and sch.name = :schname +order by + ind_col.index_id, + ind_col.key_ordinal """ ) .bindparams( @@ -3632,27 +3684,37 @@ def _get_internal_temp_table_name(self, connection, tablename): @reflection.cache @_db_plus_owner def get_columns(self, connection, tablename, dbname, owner, schema, **kw): + sys_columns = ischema.sys_columns + sys_types = ischema.sys_types + sys_base_types = ischema.sys_types.alias("base_types") + sys_default_constraints = ischema.sys_default_constraints + computed_cols = ischema.computed_columns + identity_cols = ischema.identity_columns + extended_properties = ischema.extended_properties + + # to access sys tables, need an object_id. + # object_id() can normally match to the unquoted name even if it + # has special characters. however it also accepts quoted names, + # which means for the special case that the name itself has + # "quotes" (e.g. brackets for SQL Server) we need to "quote" (e.g. + # bracket) that name anyway. Fixed as part of #12654 + is_temp_table = tablename.startswith("#") if is_temp_table: owner, tablename = self._get_internal_temp_table_name( connection, tablename ) - columns = ischema.mssql_temp_table_columns - else: - columns = ischema.columns - - computed_cols = ischema.computed_columns - identity_cols = ischema.identity_columns + object_id_tokens = [self.identifier_preparer.quote(tablename)] if owner: - whereclause = sql.and_( - columns.c.table_name == tablename, - columns.c.table_schema == owner, - ) - full_name = columns.c.table_schema + "." + columns.c.table_name - else: - whereclause = columns.c.table_name == tablename - full_name = columns.c.table_name + object_id_tokens.insert(0, self.identifier_preparer.quote(owner)) + + if is_temp_table: + object_id_tokens.insert(0, "tempdb") + + object_id = func.object_id(".".join(object_id_tokens)) + + whereclause = sys_columns.c.object_id == object_id if self._supports_nvarchar_max: computed_definition = computed_cols.c.definition @@ -3662,100 +3724,147 @@ def get_columns(self, connection, tablename, dbname, owner, schema, **kw): computed_cols.c.definition, NVARCHAR(4000) ) - object_id = func.object_id(full_name) - s = ( sql.select( - columns.c.column_name, - columns.c.data_type, - columns.c.is_nullable, - columns.c.character_maximum_length, - columns.c.numeric_precision, - columns.c.numeric_scale, - columns.c.column_default, - columns.c.collation_name, + sys_columns.c.name, + sys_types.c.name, + sys_base_types.c.name.label("base_type"), + sys_columns.c.is_nullable, + sys_columns.c.max_length, + sys_columns.c.precision, + sys_columns.c.scale, + sys_default_constraints.c.definition, + sys_columns.c.collation_name, computed_definition, computed_cols.c.is_persisted, identity_cols.c.is_identity, identity_cols.c.seed_value, identity_cols.c.increment_value, - ischema.extended_properties.c.value.label("comment"), + extended_properties.c.value.label("comment"), + ) + .select_from(sys_columns) + .join( + sys_types, + onclause=sys_columns.c.user_type_id + == sys_types.c.user_type_id, + ) + .outerjoin( + sys_base_types, + onclause=sql.and_( + sys_types.c.system_type_id + == sys_base_types.c.system_type_id, + sys_base_types.c.user_type_id + == sys_base_types.c.system_type_id, + ), + ) + .outerjoin( + sys_default_constraints, + sql.and_( + sys_default_constraints.c.object_id + == sys_columns.c.default_object_id, + sys_default_constraints.c.parent_column_id + == sys_columns.c.column_id, + ), ) - .select_from(columns) .outerjoin( computed_cols, onclause=sql.and_( - computed_cols.c.object_id == object_id, - computed_cols.c.name - == columns.c.column_name.collate("DATABASE_DEFAULT"), + computed_cols.c.object_id == sys_columns.c.object_id, + computed_cols.c.column_id == sys_columns.c.column_id, ), ) .outerjoin( identity_cols, onclause=sql.and_( - identity_cols.c.object_id == object_id, - identity_cols.c.name - == columns.c.column_name.collate("DATABASE_DEFAULT"), + identity_cols.c.object_id == sys_columns.c.object_id, + identity_cols.c.column_id == sys_columns.c.column_id, ), ) .outerjoin( - ischema.extended_properties, + extended_properties, onclause=sql.and_( - ischema.extended_properties.c["class"] == 1, - ischema.extended_properties.c.major_id == object_id, - ischema.extended_properties.c.minor_id - == columns.c.ordinal_position, - ischema.extended_properties.c.name == "MS_Description", + extended_properties.c["class"] == 1, + extended_properties.c.name == "MS_Description", + sys_columns.c.object_id == extended_properties.c.major_id, + sys_columns.c.column_id == extended_properties.c.minor_id, ), ) .where(whereclause) - .order_by(columns.c.ordinal_position) + .order_by(sys_columns.c.column_id) ) - c = connection.execution_options(future_result=True).execute(s) + if is_temp_table: + exec_opts = {"schema_translate_map": {"sys": "tempdb.sys"}} + else: + exec_opts = {"schema_translate_map": {}} + c = connection.execution_options(**exec_opts).execute(s) cols = [] for row in c.mappings(): - name = row[columns.c.column_name] - type_ = row[columns.c.data_type] - nullable = row[columns.c.is_nullable] == "YES" - charlen = row[columns.c.character_maximum_length] - numericprec = row[columns.c.numeric_precision] - numericscale = row[columns.c.numeric_scale] - default = row[columns.c.column_default] - collation = row[columns.c.collation_name] + name = row[sys_columns.c.name] + type_ = row[sys_types.c.name] + base_type = row["base_type"] + nullable = row[sys_columns.c.is_nullable] == 1 + maxlen = row[sys_columns.c.max_length] + numericprec = row[sys_columns.c.precision] + numericscale = row[sys_columns.c.scale] + default = row[sys_default_constraints.c.definition] + collation = row[sys_columns.c.collation_name] definition = row[computed_definition] is_persisted = row[computed_cols.c.is_persisted] is_identity = row[identity_cols.c.is_identity] identity_start = row[identity_cols.c.seed_value] identity_increment = row[identity_cols.c.increment_value] - comment = row[ischema.extended_properties.c.value] + comment = row[extended_properties.c.value] + # Try to resolve the user type first (e.g., "sysname"), + # then fall back to the base type (e.g., "nvarchar"). + # base_type may be None for CLR types (geography, geometry, + # hierarchyid) which have no corresponding base type. coltype = self.ischema_names.get(type_, None) + if ( + coltype is None + and base_type is not None + and base_type != type_ + ): + coltype = self.ischema_names.get(base_type, None) kwargs = {} + if coltype in ( + MSBinary, + MSVarBinary, + sqltypes.LargeBinary, + ): + kwargs["length"] = maxlen if maxlen != -1 else None + elif coltype in ( MSString, MSChar, + MSText, + ): + kwargs["length"] = maxlen if maxlen != -1 else None + if collation: + kwargs["collation"] = collation + elif coltype in ( MSNVarchar, MSNChar, - MSText, MSNText, - MSBinary, - MSVarBinary, - sqltypes.LargeBinary, ): - if charlen == -1: - charlen = None - kwargs["length"] = charlen + kwargs["length"] = maxlen // 2 if maxlen != -1 else None if collation: kwargs["collation"] = collation if coltype is None: - util.warn( - "Did not recognize type '%s' of column '%s'" - % (type_, name) - ) + if base_type is not None and base_type != type_: + util.warn( + "Did not recognize type '%s' (user type) or '%s' " + "(base type) of column '%s'" % (type_, base_type, name) + ) + else: + util.warn( + "Did not recognize type '%s' of column '%s'" + % (type_, name) + ) coltype = sqltypes.NULLTYPE else: if issubclass(coltype, sqltypes.NumericCommon): @@ -3969,6 +4078,8 @@ def get_foreign_keys( index_info.index_schema = fk_info.unique_constraint_schema AND index_info.index_name = fk_info.unique_constraint_name AND index_info.ordinal_position = fk_info.ordinal_position + AND NOT (index_info.table_schema = fk_info.table_schema + AND index_info.table_name = fk_info.table_name) ORDER BY fk_info.constraint_schema, fk_info.constraint_name, fk_info.ordinal_position @@ -3991,10 +4102,8 @@ def get_foreign_keys( ) # group rows by constraint ID, to handle multi-column FKs - fkeys = [] - - def fkey_rec(): - return { + fkeys = util.defaultdict( + lambda: { "name": None, "constrained_columns": [], "referred_schema": None, @@ -4002,8 +4111,7 @@ def fkey_rec(): "referred_columns": [], "options": {}, } - - fkeys = util.defaultdict(fkey_rec) + ) for r in connection.execute(s).all(): ( diff --git a/lib/sqlalchemy/dialects/mssql/information_schema.py b/lib/sqlalchemy/dialects/mssql/information_schema.py index b60bb158b46..d739d718239 100644 --- a/lib/sqlalchemy/dialects/mssql/information_schema.py +++ b/lib/sqlalchemy/dialects/mssql/information_schema.py @@ -1,5 +1,5 @@ # dialects/mssql/information_schema.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -88,23 +88,41 @@ def _compile(element, compiler, **kw): schema="INFORMATION_SCHEMA", ) -mssql_temp_table_columns = Table( - "COLUMNS", +sys_columns = Table( + "columns", ischema, - Column("TABLE_SCHEMA", CoerceUnicode, key="table_schema"), - Column("TABLE_NAME", CoerceUnicode, key="table_name"), - Column("COLUMN_NAME", CoerceUnicode, key="column_name"), - Column("IS_NULLABLE", Integer, key="is_nullable"), - Column("DATA_TYPE", String, key="data_type"), - Column("ORDINAL_POSITION", Integer, key="ordinal_position"), - Column( - "CHARACTER_MAXIMUM_LENGTH", Integer, key="character_maximum_length" - ), - Column("NUMERIC_PRECISION", Integer, key="numeric_precision"), - Column("NUMERIC_SCALE", Integer, key="numeric_scale"), - Column("COLUMN_DEFAULT", Integer, key="column_default"), - Column("COLLATION_NAME", String, key="collation_name"), - schema="tempdb.INFORMATION_SCHEMA", + Column("object_id", Integer), + Column("name", CoerceUnicode), + Column("column_id", Integer), + Column("default_object_id", Integer), + Column("user_type_id", Integer), + Column("is_nullable", Integer), + Column("ordinal_position", Integer), + Column("max_length", Integer), + Column("precision", Integer), + Column("scale", Integer), + Column("collation_name", String), + schema="sys", +) + +sys_types = Table( + "types", + ischema, + Column("name", CoerceUnicode, key="name"), + Column("system_type_id", Integer, key="system_type_id"), + Column("user_type_id", Integer, key="user_type_id"), + Column("schema_id", Integer, key="schema_id"), + Column("max_length", Integer, key="max_length"), + Column("precision", Integer, key="precision"), + Column("scale", Integer, key="scale"), + Column("collation_name", CoerceUnicode, key="collation_name"), + Column("is_nullable", Boolean, key="is_nullable"), + Column("is_user_defined", Boolean, key="is_user_defined"), + Column("is_assembly_type", Boolean, key="is_assembly_type"), + Column("default_object_id", Integer, key="default_object_id"), + Column("rule_object_id", Integer, key="rule_object_id"), + Column("is_table_type", Boolean, key="is_table_type"), + schema="sys", ) constraints = Table( @@ -117,6 +135,17 @@ def _compile(element, compiler, **kw): schema="INFORMATION_SCHEMA", ) +sys_default_constraints = Table( + "default_constraints", + ischema, + Column("object_id", Integer), + Column("name", CoerceUnicode), + Column("schema_id", Integer), + Column("parent_column_id", Integer), + Column("definition", CoerceUnicode), + schema="sys", +) + column_constraints = Table( "CONSTRAINT_COLUMN_USAGE", ischema, @@ -182,6 +211,7 @@ def _compile(element, compiler, **kw): ischema, Column("object_id", Integer), Column("name", CoerceUnicode), + Column("column_id", Integer), Column("is_computed", Boolean), Column("is_persisted", Boolean), Column("definition", CoerceUnicode), @@ -220,6 +250,7 @@ def column_expression(self, colexpr): ischema, Column("object_id", Integer), Column("name", CoerceUnicode), + Column("column_id", Integer), Column("is_identity", Boolean), Column("seed_value", NumericSqlVariant), Column("increment_value", NumericSqlVariant), diff --git a/lib/sqlalchemy/dialects/mssql/json.py b/lib/sqlalchemy/dialects/mssql/json.py index a2d3ce81469..4c128326a40 100644 --- a/lib/sqlalchemy/dialects/mssql/json.py +++ b/lib/sqlalchemy/dialects/mssql/json.py @@ -1,12 +1,21 @@ # dialects/mssql/json.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors +from __future__ import annotations + +from typing import Any +from typing import TYPE_CHECKING from ... import types as sqltypes +from ...sql.sqltypes import _T_JSON + +if TYPE_CHECKING: + from ...engine.interfaces import Dialect + from ...sql.type_api import _BindProcessorType + from ...sql.type_api import _LiteralProcessorType # technically, all the dialect-specific datatypes that don't have any special # behaviors would be private with names like _MSJson. However, we haven't been @@ -16,7 +25,7 @@ # package-private at once. -class JSON(sqltypes.JSON): +class JSON(sqltypes.JSON[_T_JSON]): """MSSQL JSON type. MSSQL supports JSON-formatted data as of SQL Server 2016. @@ -82,13 +91,13 @@ class JSON(sqltypes.JSON): # these are not generalizable to all JSON implementations, remain separately # implemented for each dialect. class _FormatTypeMixin: - def _format_value(self, value): + def _format_value(self, value: Any) -> str: raise NotImplementedError() - def bind_processor(self, dialect): - super_proc = self.string_bind_processor(dialect) + def bind_processor(self, dialect: Dialect) -> _BindProcessorType[Any]: + super_proc = self.string_bind_processor(dialect) # type: ignore[attr-defined] # noqa: E501 - def process(value): + def process(value: Any) -> Any: value = self._format_value(value) if super_proc: value = super_proc(value) @@ -96,29 +105,31 @@ def process(value): return process - def literal_processor(self, dialect): - super_proc = self.string_literal_processor(dialect) + def literal_processor( + self, dialect: Dialect + ) -> _LiteralProcessorType[Any]: + super_proc = self.string_literal_processor(dialect) # type: ignore[attr-defined] # noqa: E501 - def process(value): + def process(value: Any) -> str: value = self._format_value(value) if super_proc: value = super_proc(value) - return value + return value # type: ignore[no-any-return] return process class JSONIndexType(_FormatTypeMixin, sqltypes.JSON.JSONIndexType): - def _format_value(self, value): + def _format_value(self, value: Any) -> str: if isinstance(value, int): - value = "$[%s]" % value + formatted_value = "$[%s]" % value else: - value = '$."%s"' % value - return value + formatted_value = '$."%s"' % value + return formatted_value class JSONPathType(_FormatTypeMixin, sqltypes.JSON.JSONPathType): - def _format_value(self, value): + def _format_value(self, value: Any) -> str: return "$%s" % ( "".join( [ diff --git a/lib/sqlalchemy/dialects/mssql/mssqlpython.py b/lib/sqlalchemy/dialects/mssql/mssqlpython.py new file mode 100644 index 00000000000..80a89fdf014 --- /dev/null +++ b/lib/sqlalchemy/dialects/mssql/mssqlpython.py @@ -0,0 +1,220 @@ +# dialects/mssql/mssqlpython.py +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors +# +# +# This module is part of SQLAlchemy and is released under +# the MIT License: https://www.opensource.org/licenses/mit-license.php +# mypy: ignore-errors + +""" +.. dialect:: mssql+mssqlpython + :name: mssqlpython + :dbapi: mssql-python + :connectstring: mssql+mssqlpython://:@:/ + :url: https://github.com/microsoft/mssql-python + +mssql-python is a driver for Microsoft SQL Server produced by Microsoft. + +.. versionadded:: 2.1.0b2 + + +The driver is generally similar to pyodbc in most aspects as it is based +on the same ODBC framework. + +Connection Strings +------------------ + +Examples of connecting with the mssql-python driver:: + + from sqlalchemy import create_engine + + # Basic connection + engine = create_engine( + "mssql+mssqlpython://user:password@hostname/database" + ) + + # With Windows Authentication + engine = create_engine( + "mssql+mssqlpython://hostname/database?authentication=ActiveDirectoryIntegrated" + ) + +""" # noqa + +from __future__ import annotations + +import re +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union + +from .base import MSDialect +from .pyodbc import _ms_numeric_pyodbc +from ... import util +from ...sql import sqltypes + +if TYPE_CHECKING: + from ... import pool + from ...engine import interfaces + from ...engine.interfaces import ConnectArgsType + from ...engine.interfaces import DBAPIModule + from ...engine.interfaces import IsolationLevel + from ...engine.interfaces import URL + + +class _MSNumeric_mssqlpython(_ms_numeric_pyodbc, sqltypes.Numeric): + pass + + +class _MSFloat_mssqlpython(_ms_numeric_pyodbc, sqltypes.Float): + pass + + +class MSDialect_mssqlpython(MSDialect): + driver = "mssqlpython" + supports_statement_cache = True + + supports_sane_rowcount_returning = True + supports_sane_multi_rowcount = True + supports_native_uuid = True + scope_identity_must_be_embedded = True + + supports_native_decimal = True + + # used by pyodbc _ms_numeric_pyodbc class + _need_decimal_fix = True + + colspecs = util.update_copy( + MSDialect.colspecs, + { + sqltypes.Numeric: _MSNumeric_mssqlpython, + sqltypes.Float: _MSFloat_mssqlpython, + }, + ) + + def __init__(self, enable_pooling=False, **kw): + super().__init__(**kw) + if not enable_pooling and self.dbapi is not None: + self.loaded_dbapi.pooling(enabled=False) + + @classmethod + def import_dbapi(cls) -> DBAPIModule: + return __import__("mssql_python") + + def create_connect_args(self, url: URL) -> ConnectArgsType: + opts = url.translate_connect_args(username="user") + opts.update(url.query) + + keys = opts + + query = url.query + + connect_args: Dict[str, Any] = {} + connectors: List[str] + + def check_quote(token: str) -> str: + if ";" in str(token) or str(token).startswith("{"): + token = "{%s}" % token.replace("}", "}}") + return token + + keys = {k: check_quote(v) for k, v in keys.items()} + + port = "" + if "port" in keys and "port" not in query: + port = ",%d" % int(keys.pop("port")) + + connectors = [] + + connectors.extend( + [ + "Server=%s%s" % (keys.pop("host", ""), port), + "Database=%s" % keys.pop("database", ""), + ] + ) + + user = keys.pop("user", None) + if user: + connectors.append("UID=%s" % user) + pwd = keys.pop("password", "") + if pwd: + connectors.append("PWD=%s" % pwd) + else: + authentication = keys.pop("authentication", None) + if authentication: + connectors.append("Authentication=%s" % authentication) + + connectors.extend(["%s=%s" % (k, v) for k, v in keys.items()]) + + return ((";".join(connectors),), connect_args) + + def is_disconnect( + self, + e: Exception, + connection: Optional[ + Union[pool.PoolProxiedConnection, interfaces.DBAPIConnection] + ], + cursor: Optional[interfaces.DBAPICursor], + ) -> bool: + if isinstance(e, self.loaded_dbapi.ProgrammingError): + return ( + "The cursor's connection has been closed." in str(e) + or "Attempt to use a closed connection." in str(e) + or "Driver Error: Operation cannot be performed" in str(e) + ) + elif isinstance(e, self.loaded_dbapi.InterfaceError): + return bool(re.search(r"Cannot .* on closed connection", str(e))) + else: + return False + + def _dbapi_version(self) -> interfaces.VersionInfoType: + if not self.dbapi: + return () + return self._parse_dbapi_version(self.dbapi.version) + + def _parse_dbapi_version(self, vers: str) -> interfaces.VersionInfoType: + m = re.match(r"(?:py.*-)?([\d\.]+)(?:-(\w+))?", vers) + if not m: + return () + vers_tuple: interfaces.VersionInfoType = tuple( + [int(x) for x in m.group(1).split(".")] + ) + if m.group(2): + vers_tuple += (m.group(2),) + return vers_tuple + + def _get_server_version_info(self, connection): + vers = connection.exec_driver_sql("select @@version").scalar() + m = re.match(r"Microsoft .*? - (\d+)\.(\d+)\.(\d+)\.(\d+)", vers) + if m: + return tuple(int(x) for x in m.group(1, 2, 3, 4)) + else: + return None + + def get_isolation_level_values( + self, dbapi_connection: interfaces.DBAPIConnection + ) -> List[IsolationLevel]: + return [ + *super().get_isolation_level_values(dbapi_connection), + "AUTOCOMMIT", + ] + + def set_isolation_level( + self, + dbapi_connection: interfaces.DBAPIConnection, + level: IsolationLevel, + ) -> None: + if level == "AUTOCOMMIT": + dbapi_connection.autocommit = True + else: + dbapi_connection.autocommit = False + super().set_isolation_level(dbapi_connection, level) + + def detect_autocommit_setting( + self, dbapi_conn: interfaces.DBAPIConnection + ) -> bool: + return bool(dbapi_conn.autocommit) + + +dialect = MSDialect_mssqlpython diff --git a/lib/sqlalchemy/dialects/mssql/provision.py b/lib/sqlalchemy/dialects/mssql/provision.py index 10165856e1a..823a13c95d4 100644 --- a/lib/sqlalchemy/dialects/mssql/provision.py +++ b/lib/sqlalchemy/dialects/mssql/provision.py @@ -1,5 +1,5 @@ # dialects/mssql/provision.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -16,6 +16,7 @@ from ...schema import MetaData from ...schema import Table from ...testing.provision import create_db +from ...testing.provision import dbapi_error from ...testing.provision import drop_all_schema_objects_pre_tables from ...testing.provision import drop_db from ...testing.provision import generate_driver_url @@ -39,8 +40,10 @@ def generate_driver_url(url, driver, query_str): new_url = url.set(drivername="%s+%s" % (backend, driver)) - if driver not in ("pyodbc", "aioodbc"): + if driver == "pymssql" and url.get_driver_name() != "pymssql": new_url = new_url.set(query="") + elif driver == "mssqlpython" and url.get_driver_name() != "mssqlpython": + new_url = new_url.set(query={"Encrypt": "No"}) if driver == "aioodbc": new_url = new_url.update_query_dict({"MARS_Connection": "Yes"}) @@ -136,6 +139,29 @@ def _mssql_get_temp_table_name(cfg, eng, base_name): def drop_all_schema_objects_pre_tables(cfg, eng): with eng.connect().execution_options(isolation_level="AUTOCOMMIT") as conn: inspector = inspect(conn) + + # Drop all full-text indexes before dropping catalogs + fulltext_indexes = conn.exec_driver_sql( + "SELECT OBJECT_SCHEMA_NAME(object_id) AS schema_name, " + "OBJECT_NAME(object_id) AS table_name " + "FROM sys.fulltext_indexes" + ).fetchall() + + for schema_name, table_name in fulltext_indexes: + if schema_name: + qualified_name = f"[{schema_name}].[{table_name}]" + else: + qualified_name = f"[{table_name}]" + conn.exec_driver_sql(f"DROP FULLTEXT INDEX ON {qualified_name}") + + # Now drop all full-text catalogs + fulltext_catalogs = conn.exec_driver_sql( + "SELECT name FROM sys.fulltext_catalogs" + ).fetchall() + + for (catalog_name,) in fulltext_catalogs: + conn.exec_driver_sql(f"DROP FULLTEXT CATALOG [{catalog_name}]") + for schema in (None, "dbo", cfg.test_schema, cfg.test_schema_2): for tname in inspector.get_table_names(schema=schema): tb = Table( @@ -150,7 +176,7 @@ def drop_all_schema_objects_pre_tables(cfg, eng): DropConstraint( ForeignKeyConstraint( [tb.c.x], [tb.c.y], name=fk["name"] - ) + ), ) ) @@ -160,3 +186,11 @@ def normalize_sequence(cfg, sequence): if sequence.start is None: sequence.start = 1 return sequence + + +@dbapi_error.for_db("mssql") +def dbapi_error(cfg, cls, message): + if cfg.db.driver == "mssqlpython": + return cls(message, "placeholder for mssqlpython") + else: + return cls(message) diff --git a/lib/sqlalchemy/dialects/mssql/pymssql.py b/lib/sqlalchemy/dialects/mssql/pymssql.py index 301a98eb417..7244bb0d3e3 100644 --- a/lib/sqlalchemy/dialects/mssql/pymssql.py +++ b/lib/sqlalchemy/dialects/mssql/pymssql.py @@ -1,5 +1,5 @@ # dialects/mssql/pymssql.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/dialects/mssql/pyodbc.py b/lib/sqlalchemy/dialects/mssql/pyodbc.py index cbf0adbfe08..07b9eef60be 100644 --- a/lib/sqlalchemy/dialects/mssql/pyodbc.py +++ b/lib/sqlalchemy/dialects/mssql/pyodbc.py @@ -1,5 +1,5 @@ # dialects/mssql/pyodbc.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -325,8 +325,6 @@ def provide_token(dialect, conn_rec, cargs, cparams): feature would cause ``fast_executemany`` to not be used in most cases even if specified. -.. versionadded:: 1.3 - .. seealso:: `fast executemany `_ @@ -371,7 +369,6 @@ def provide_token(dialect, conn_rec, cargs, cparams): from .base import BINARY from .base import DATETIMEOFFSET from .base import MSDialect -from .base import MSExecutionContext from .base import VARBINARY from .json import JSON as _MSJson from .json import JSONIndexType as _MSJsonIndexType @@ -380,7 +377,6 @@ def provide_token(dialect, conn_rec, cargs, cparams): from ... import types as sqltypes from ... import util from ...connectors.pyodbc import PyODBCConnector -from ...engine import cursor as _cursor class _ms_numeric_pyodbc: @@ -562,73 +558,15 @@ def get_dbapi_type(self, dbapi): return dbapi.SQL_WVARCHAR -class MSExecutionContext_pyodbc(MSExecutionContext): - _embedded_scope_identity = False - - def pre_exec(self): - """where appropriate, issue "select scope_identity()" in the same - statement. - - Background on why "scope_identity()" is preferable to "@@identity": - https://msdn.microsoft.com/en-us/library/ms190315.aspx - - Background on why we attempt to embed "scope_identity()" into the same - statement as the INSERT: - https://code.google.com/p/pyodbc/wiki/FAQs#How_do_I_retrieve_autogenerated/identity_values? - - """ - - super().pre_exec() - - # don't embed the scope_identity select into an - # "INSERT .. DEFAULT VALUES" - if ( - self._select_lastrowid - and self.dialect.use_scope_identity - and len(self.parameters[0]) - ): - self._embedded_scope_identity = True - - self.statement += "; select scope_identity()" - - def post_exec(self): - if self._embedded_scope_identity: - # Fetch the last inserted id from the manipulated statement - # We may have to skip over a number of result sets with - # no data (due to triggers, etc.) - while True: - try: - # fetchall() ensures the cursor is consumed - # without closing it (FreeTDS particularly) - rows = self.cursor.fetchall() - except self.dialect.dbapi.Error: - # no way around this - nextset() consumes the previous set - # so we need to just keep flipping - self.cursor.nextset() - else: - if not rows: - # async adapter drivers just return None here - self.cursor.nextset() - continue - row = rows[0] - break - - self._lastrowid = int(row[0]) - - self.cursor_fetch_strategy = _cursor._NO_CURSOR_DML - else: - super().post_exec() - - class MSDialect_pyodbc(PyODBCConnector, MSDialect): supports_statement_cache = True + scope_identity_must_be_embedded = True + # note this parameter is no longer used by the ORM or default dialect # see #9414 supports_sane_rowcount_returning = False - execution_ctx_cls = MSExecutionContext_pyodbc - colspecs = util.update_copy( MSDialect.colspecs, { diff --git a/lib/sqlalchemy/dialects/mysql/__init__.py b/lib/sqlalchemy/dialects/mysql/__init__.py index d722c1d30ca..e2c939e4ff6 100644 --- a/lib/sqlalchemy/dialects/mysql/__init__.py +++ b/lib/sqlalchemy/dialects/mysql/__init__.py @@ -1,5 +1,5 @@ # dialects/mysql/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -102,4 +102,5 @@ "insert", "Insert", "match", + "limit", ) diff --git a/lib/sqlalchemy/dialects/mysql/_mariadb_shim.py b/lib/sqlalchemy/dialects/mysql/_mariadb_shim.py new file mode 100644 index 00000000000..fdf210c1fca --- /dev/null +++ b/lib/sqlalchemy/dialects/mysql/_mariadb_shim.py @@ -0,0 +1,312 @@ +# dialects/mysql/_mariadb_shim.py +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors +# +# +# This module is part of SQLAlchemy and is released under +# the MIT License: https://www.opensource.org/licenses/mit-license.php + +from __future__ import annotations + +from typing import Any +from typing import cast +from typing import Optional +from typing import Type +from typing import TYPE_CHECKING + +from .reserved_words import RESERVED_WORDS_MARIADB +from ... import exc +from ... import schema as sa_schema +from ... import util +from ...engine import cursor as _cursor +from ...engine import default +from ...engine.default import DefaultDialect +from ...engine.interfaces import TypeCompiler +from ...sql import elements +from ...sql import sqltypes +from ...sql.compiler import DDLCompiler +from ...sql.compiler import IdentifierPreparer +from ...sql.compiler import SQLCompiler +from ...sql.schema import SchemaConst +from ...sql.sqltypes import _UUID_RETURN +from ...sql.sqltypes import UUID +from ...sql.sqltypes import Uuid + +if TYPE_CHECKING: + from .base import MySQLIdentifierPreparer + from .mariadb import INET4 + from .mariadb import INET6 + from ...engine import URL + from ...engine.base import Connection + from ...sql import ddl + from ...sql.schema import IdentityOptions + from ...sql.schema import Sequence as Sequence_SchemaItem + from ...sql.type_api import _BindProcessorType + + +class _MariaDBUUID(UUID[_UUID_RETURN]): + def __init__(self, as_uuid: bool = True, native_uuid: bool = True): + self.as_uuid = as_uuid + + # the _MariaDBUUID internal type is only invoked for a Uuid() with + # native_uuid=True. for non-native uuid type, the plain Uuid + # returns itself due to the workings of the Emulated superclass. + assert native_uuid + + # for internal type, force string conversion for result_processor() as + # current drivers are returning a string, not a Python UUID object + self.native_uuid = False + + @property + def native(self) -> bool: # type: ignore[override] + # override to return True, this is a native type, just turning + # off native_uuid for internal data handling + return True + + def bind_processor(self, dialect: MariaDBShim) -> Optional[_BindProcessorType[_UUID_RETURN]]: # type: ignore[override] # noqa: E501 + if not dialect.supports_native_uuid or not dialect._allows_uuid_binds: + return super().bind_processor(dialect) # type: ignore[return-value] # noqa: E501 + else: + return None + + +class MariaDBTypeCompilerShim(TypeCompiler): + def visit_INET4(self, type_: INET4, **kwargs: Any) -> str: + return "INET4" + + def visit_INET6(self, type_: INET6, **kwargs: Any) -> str: + return "INET6" + + +class MariadbExecutionContextShim(default.DefaultExecutionContext): + def post_exec(self) -> None: + if ( + self.isdelete + and cast(SQLCompiler, self.compiled).effective_returning + and not self.cursor.description + ): + # All MySQL/mariadb drivers appear to not include + # cursor.description for DELETE..RETURNING with no rows if the + # WHERE criteria is a straight "false" condition such as our EMPTY + # IN condition. manufacture an empty result in this case (issue + # #10505) + # + # taken from cx_Oracle implementation + self.cursor_fetch_strategy = ( + _cursor.FullyBufferedCursorFetchStrategy( + self.cursor, + [ + (entry.keyname, None) # type: ignore[misc] + for entry in cast( + SQLCompiler, self.compiled + )._result_columns + ], + [], + ) + ) + + def fire_sequence( + self, seq: Sequence_SchemaItem, type_: sqltypes.Integer + ) -> int: + return self._execute_scalar( # type: ignore[no-any-return] + ( + "select nextval(%s)" + % self.identifier_preparer.format_sequence(seq) + ), + type_, + ) + + +class MariaDBIdentifierPreparerShim(IdentifierPreparer): + def _set_mariadb(self) -> None: + self.reserved_words = RESERVED_WORDS_MARIADB + + +class MariaDBSQLCompilerShim(SQLCompiler): + def visit_sequence(self, sequence: sa_schema.Sequence, **kw: Any) -> str: + return "nextval(%s)" % self.preparer.format_sequence(sequence) + + def _mariadb_regexp_flags( + self, flags: str, pattern: elements.ColumnElement[Any], **kw: Any + ) -> str: + return "CONCAT('(?', %s, ')', %s)" % ( + self.render_literal_value(flags, sqltypes.STRINGTYPE), + self.process(pattern, **kw), + ) + + def _mariadb_regexp_match( + self, + op_string: str, + binary: elements.BinaryExpression[Any], + operator: Any, + **kw: Any, + ) -> str: + flags = binary.modifiers["flags"] + return "%s%s%s" % ( + self.process(binary.left, **kw), + op_string, + self._mariadb_regexp_flags(flags, binary.right), + ) + + def _mariadb_regexp_replace_op_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: + flags = binary.modifiers["flags"] + return "REGEXP_REPLACE(%s, %s, %s)" % ( + self.process(binary.left, **kw), + self._mariadb_regexp_flags(flags, binary.right.clauses[0]), + self.process(binary.right.clauses[1], **kw), + ) + + def _mariadb_visit_drop_check_constraint( + self, drop: ddl.DropConstraint, **kw: Any + ) -> str: + constraint = drop.element + qual = "CONSTRAINT " + const = self.preparer.format_constraint(constraint) + return "ALTER TABLE %s DROP %s%s" % ( + self.preparer.format_table(constraint.table), + qual, + const, + ) + + +class MariaDBDDLCompilerShim(DDLCompiler): + dialect: MariaDBShim + + def _mariadb_get_column_specification( + self, column: sa_schema.Column[Any], **kw: Any + ) -> str: + + if ( + column.computed is not None + and column._user_defined_nullable is SchemaConst.NULL_UNSPECIFIED + ): + kw["_force_column_to_nullable"] = True + + return self._mysql_get_column_specification(column, **kw) + + def _mysql_get_column_specification( + self, + column: sa_schema.Column[Any], + *, + _force_column_to_nullable: bool = False, + **kw: Any, + ) -> str: + raise NotImplementedError() + + def get_identity_options(self, identity_options: IdentityOptions) -> str: + text = super().get_identity_options(identity_options) + text = text.replace("NO CYCLE", "NOCYCLE") + return text + + def _mariadb_visit_drop_check_constraint( + self, drop: ddl.DropConstraint, **kw: Any + ) -> str: + constraint = drop.element + qual = "CONSTRAINT " + const = self.preparer.format_constraint(constraint) + return "ALTER TABLE %s DROP %s%s" % ( + self.preparer.format_table(constraint.table), + qual, + const, + ) + + +class MariaDBShim(DefaultDialect): + server_version_info: tuple[int, ...] + is_mariadb: bool + _allows_uuid_binds = False + + identifier_preparer: MySQLIdentifierPreparer + preparer: Type[MySQLIdentifierPreparer] + + def _set_mariadb( + self, is_mariadb: Optional[bool], server_version_info: tuple[int, ...] + ) -> None: + if is_mariadb is None: + return + + if not is_mariadb and self.is_mariadb: + raise exc.InvalidRequestError( + "MySQL version %s is not a MariaDB variant." + % (".".join(map(str, server_version_info)),) + ) + if is_mariadb: + assert isinstance(self.colspecs, dict) + self.colspecs = util.update_copy( + self.colspecs, {Uuid: _MariaDBUUID} + ) + + self.identifier_preparer = self.preparer(self) + self.identifier_preparer._set_mariadb() + + # this will be updated on first connect in initialize() + # if using older mariadb version + self.delete_returning = True + self.insert_returning = True + + self.is_mariadb = is_mariadb + + @property + def _mariadb_normalized_version_info(self) -> tuple[int, ...]: + return self.server_version_info + + @property + def _is_mariadb(self) -> bool: + return self.is_mariadb + + @classmethod + def _is_mariadb_from_url(cls, url: URL) -> bool: + dbapi = cls.import_dbapi() + dialect = cls(dbapi=dbapi) + + cargs, cparams = dialect.create_connect_args(url) + conn = dialect.connect(*cargs, **cparams) + try: + cursor = conn.cursor() + cursor.execute("SELECT VERSION() LIKE '%MariaDB%'") + val = cursor.fetchone()[0] # type: ignore[index] + except: + raise + else: + return bool(val) + finally: + conn.close() + + def _initialize_mariadb(self, connection: Connection) -> None: + assert self.is_mariadb + + self.supports_sequences = self.server_version_info >= (10, 3) + + self.delete_returning = self.server_version_info >= (10, 0, 5) + + self.insert_returning = self.server_version_info >= (10, 5) + + self._warn_for_known_db_issues() + + self.supports_native_uuid = ( + self.server_version_info is not None + and self.server_version_info >= (10, 7) + ) + self._allows_uuid_binds = True + + # ref https://mariadb.com/kb/en/mariadb-1021-release-notes/ + self._support_default_function = self.server_version_info >= (10, 2, 1) + + # ref https://mariadb.com/kb/en/mariadb-1045-release-notes/ + self._support_float_cast = self.server_version_info >= (10, 4, 5) + + def _warn_for_known_db_issues(self) -> None: + if self.is_mariadb: + mdb_version = self.server_version_info + assert mdb_version is not None + if mdb_version > (10, 2) and mdb_version < (10, 2, 9): + util.warn( + "MariaDB %r before 10.2.9 has known issues regarding " + "CHECK constraints, which impact handling of NULL values " + "with SQLAlchemy's boolean datatype (MDEV-13596). An " + "additional issue prevents proper migrations of columns " + "with CHECK constraints (MDEV-11114). Please upgrade to " + "MariaDB 10.2.9 or greater, or use the MariaDB 10.1 " + "series, to avoid these issues." % (mdb_version,) + ) diff --git a/lib/sqlalchemy/dialects/mysql/aiomysql.py b/lib/sqlalchemy/dialects/mysql/aiomysql.py index 66dd9111043..09db0d7b48f 100644 --- a/lib/sqlalchemy/dialects/mysql/aiomysql.py +++ b/lib/sqlalchemy/dialects/mysql/aiomysql.py @@ -1,10 +1,9 @@ # dialects/mysql/aiomysql.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors r""" .. dialect:: mysql+aiomysql @@ -29,17 +28,40 @@ ) """ # noqa +from __future__ import annotations + +from types import ModuleType +from typing import Any +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union + from .pymysql import MySQLDialect_pymysql from ...connectors.asyncio import AsyncAdapt_dbapi_connection from ...connectors.asyncio import AsyncAdapt_dbapi_cursor +from ...connectors.asyncio import AsyncAdapt_dbapi_module from ...connectors.asyncio import AsyncAdapt_dbapi_ss_cursor +from ...connectors.asyncio import AsyncAdapt_terminate from ...util.concurrency import await_ +if TYPE_CHECKING: + + from ...connectors.asyncio import AsyncIODBAPIConnection + from ...connectors.asyncio import AsyncIODBAPICursor + from ...engine.interfaces import ConnectArgsType + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import DBAPICursor + from ...engine.interfaces import DBAPIModule + from ...engine.interfaces import PoolProxiedConnection + from ...engine.url import URL + class AsyncAdapt_aiomysql_cursor(AsyncAdapt_dbapi_cursor): __slots__ = () - def _make_new_cursor(self, connection): + def _make_new_cursor( + self, connection: AsyncIODBAPIConnection + ) -> AsyncIODBAPICursor: return connection.cursor(self._adapt_connection.dbapi.Cursor) @@ -48,45 +70,56 @@ class AsyncAdapt_aiomysql_ss_cursor( ): __slots__ = () - def _make_new_cursor(self, connection): + def _make_new_cursor( + self, connection: AsyncIODBAPIConnection + ) -> AsyncIODBAPICursor: return connection.cursor( self._adapt_connection.dbapi.aiomysql.cursors.SSCursor ) -class AsyncAdapt_aiomysql_connection(AsyncAdapt_dbapi_connection): +class AsyncAdapt_aiomysql_connection( + AsyncAdapt_terminate, AsyncAdapt_dbapi_connection +): __slots__ = () _cursor_cls = AsyncAdapt_aiomysql_cursor _ss_cursor_cls = AsyncAdapt_aiomysql_ss_cursor - def ping(self, reconnect): + def ping(self, reconnect: bool) -> None: assert not reconnect - return await_(self._connection.ping(reconnect)) + await_(self._connection.ping(reconnect)) - def character_set_name(self): - return self._connection.character_set_name() + def character_set_name(self) -> Optional[str]: + return self._connection.character_set_name() # type: ignore[no-any-return] # noqa: E501 - def autocommit(self, value): + def autocommit(self, value: Any) -> None: await_(self._connection.autocommit(value)) - def terminate(self): - # it's not awaitable. - self._connection.close() + def get_autocommit(self) -> bool: + return self._connection.get_autocommit() # type: ignore def close(self) -> None: await_(self._connection.ensure_closed()) + async def _terminate_graceful_close(self) -> None: + await self._connection.ensure_closed() + + def _terminate_force_close(self) -> None: + # it's not awaitable. + self._connection.close() + -class AsyncAdapt_aiomysql_dbapi: - def __init__(self, aiomysql, pymysql): +class AsyncAdapt_aiomysql_dbapi(AsyncAdapt_dbapi_module): + def __init__(self, aiomysql: ModuleType, pymysql: ModuleType): + super().__init__(aiomysql, dbapi_module=pymysql) self.aiomysql = aiomysql self.pymysql = pymysql self.paramstyle = "format" self._init_dbapi_attributes() self.Cursor, self.SSCursor = self._init_cursors_subclasses() - def _init_dbapi_attributes(self): + def _init_dbapi_attributes(self) -> None: for name in ( "Warning", "Error", @@ -112,25 +145,33 @@ def _init_dbapi_attributes(self): ): setattr(self, name, getattr(self.pymysql, name)) - def connect(self, *arg, **kw): + def connect(self, *arg: Any, **kw: Any) -> AsyncAdapt_aiomysql_connection: creator_fn = kw.pop("async_creator_fn", self.aiomysql.connect) - return AsyncAdapt_aiomysql_connection( - self, - await_(creator_fn(*arg, **kw)), + return await_( + AsyncAdapt_aiomysql_connection.create( + self, + creator_fn(*arg, **kw), + ) ) - def _init_cursors_subclasses(self): + def _init_cursors_subclasses( + self, + ) -> tuple[AsyncIODBAPICursor, AsyncIODBAPICursor]: # suppress unconditional warning emitted by aiomysql - class Cursor(self.aiomysql.Cursor): - async def _show_warnings(self, conn): + class Cursor(self.aiomysql.Cursor): # type: ignore[misc, name-defined] + async def _show_warnings( + self, conn: AsyncIODBAPIConnection + ) -> None: pass - class SSCursor(self.aiomysql.SSCursor): - async def _show_warnings(self, conn): + class SSCursor(self.aiomysql.SSCursor): # type: ignore[misc, name-defined] # noqa: E501 + async def _show_warnings( + self, conn: AsyncIODBAPIConnection + ) -> None: pass - return Cursor, SSCursor + return Cursor, SSCursor # type: ignore[return-value] class MySQLDialect_aiomysql(MySQLDialect_pymysql): @@ -144,33 +185,42 @@ class MySQLDialect_aiomysql(MySQLDialect_pymysql): has_terminate = True @classmethod - def import_dbapi(cls): + def import_dbapi(cls) -> AsyncAdapt_aiomysql_dbapi: return AsyncAdapt_aiomysql_dbapi( __import__("aiomysql"), __import__("pymysql") ) - def do_terminate(self, dbapi_connection) -> None: + def do_terminate(self, dbapi_connection: DBAPIConnection) -> None: dbapi_connection.terminate() - def create_connect_args(self, url): + def create_connect_args( + self, url: URL, _translate_args: Optional[dict[str, Any]] = None + ) -> ConnectArgsType: return super().create_connect_args( url, _translate_args=dict(username="user", database="db") ) - def is_disconnect(self, e, connection, cursor): + def is_disconnect( + self, + e: DBAPIModule.Error, + connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]], + cursor: Optional[DBAPICursor], + ) -> bool: if super().is_disconnect(e, connection, cursor): return True else: str_e = str(e).lower() return "not connected" in str_e - def _found_rows_client_flag(self): - from pymysql.constants import CLIENT + def _found_rows_client_flag(self) -> int: + from pymysql.constants import CLIENT # type: ignore - return CLIENT.FOUND_ROWS + return CLIENT.FOUND_ROWS # type: ignore[no-any-return] - def get_driver_connection(self, connection): - return connection._connection + def get_driver_connection( + self, connection: DBAPIConnection + ) -> AsyncIODBAPIConnection: + return connection._connection # type: ignore[no-any-return] dialect = MySQLDialect_aiomysql diff --git a/lib/sqlalchemy/dialects/mysql/asyncmy.py b/lib/sqlalchemy/dialects/mysql/asyncmy.py index 86c78d65d5b..a7fa2f7d780 100644 --- a/lib/sqlalchemy/dialects/mysql/asyncmy.py +++ b/lib/sqlalchemy/dialects/mysql/asyncmy.py @@ -1,10 +1,9 @@ # dialects/mysql/asyncmy.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors r""" .. dialect:: mysql+asyncmy @@ -29,13 +28,33 @@ """ # noqa from __future__ import annotations +from types import ModuleType +from typing import Any +from typing import NoReturn +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union + from .pymysql import MySQLDialect_pymysql from ... import util from ...connectors.asyncio import AsyncAdapt_dbapi_connection from ...connectors.asyncio import AsyncAdapt_dbapi_cursor +from ...connectors.asyncio import AsyncAdapt_dbapi_module from ...connectors.asyncio import AsyncAdapt_dbapi_ss_cursor +from ...connectors.asyncio import AsyncAdapt_terminate from ...util.concurrency import await_ +if TYPE_CHECKING: + + from ...connectors.asyncio import AsyncIODBAPIConnection + from ...connectors.asyncio import AsyncIODBAPICursor + from ...engine.interfaces import ConnectArgsType + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import DBAPICursor + from ...engine.interfaces import DBAPIModule + from ...engine.interfaces import PoolProxiedConnection + from ...engine.url import URL + class AsyncAdapt_asyncmy_cursor(AsyncAdapt_dbapi_cursor): __slots__ = () @@ -46,63 +65,72 @@ class AsyncAdapt_asyncmy_ss_cursor( ): __slots__ = () - def _make_new_cursor(self, connection): + def _make_new_cursor( + self, connection: AsyncIODBAPIConnection + ) -> AsyncIODBAPICursor: return connection.cursor( self._adapt_connection.dbapi.asyncmy.cursors.SSCursor ) -class AsyncAdapt_asyncmy_connection(AsyncAdapt_dbapi_connection): +class AsyncAdapt_asyncmy_connection( + AsyncAdapt_terminate, AsyncAdapt_dbapi_connection +): __slots__ = () _cursor_cls = AsyncAdapt_asyncmy_cursor _ss_cursor_cls = AsyncAdapt_asyncmy_ss_cursor - def _handle_exception(self, error): + @classmethod + def _handle_exception_no_connection( + cls, dbapi: Any, error: Exception + ) -> NoReturn: if isinstance(error, AttributeError): - raise self.dbapi.InternalError( + raise dbapi.InternalError( "network operation failed due to asyncmy attribute error" ) raise error - def ping(self, reconnect): + def ping(self, reconnect: bool) -> None: assert not reconnect return await_(self._do_ping()) - async def _do_ping(self): + async def _do_ping(self) -> None: try: async with self._execute_mutex: - return await self._connection.ping(False) + await self._connection.ping(False) except Exception as error: self._handle_exception(error) - def character_set_name(self): - return self._connection.character_set_name() + def character_set_name(self) -> Optional[str]: + return self._connection.character_set_name() # type: ignore[no-any-return] # noqa: E501 - def autocommit(self, value): + def autocommit(self, value: Any) -> None: await_(self._connection.autocommit(value)) - def terminate(self): - # it's not awaitable. - self._connection.close() + def get_autocommit(self) -> bool: + return self._connection.get_autocommit() # type: ignore def close(self) -> None: await_(self._connection.ensure_closed()) + async def _terminate_graceful_close(self) -> None: + await self._connection.ensure_closed() -def _Binary(x): - """Return x as a binary type.""" - return bytes(x) + def _terminate_force_close(self) -> None: + # it's not awaitable. + self._connection.close() -class AsyncAdapt_asyncmy_dbapi: - def __init__(self, asyncmy): +class AsyncAdapt_asyncmy_dbapi(AsyncAdapt_dbapi_module): + def __init__(self, asyncmy: ModuleType): + super().__init__(asyncmy) self.asyncmy = asyncmy self.paramstyle = "format" self._init_dbapi_attributes() - def _init_dbapi_attributes(self): + def _init_dbapi_attributes(self) -> None: for name in ( "Warning", "Error", @@ -123,14 +151,16 @@ def _init_dbapi_attributes(self): BINARY = util.symbol("BINARY") DATETIME = util.symbol("DATETIME") TIMESTAMP = util.symbol("TIMESTAMP") - Binary = staticmethod(_Binary) + Binary = staticmethod(bytes) - def connect(self, *arg, **kw): + def connect(self, *arg: Any, **kw: Any) -> AsyncAdapt_asyncmy_connection: creator_fn = kw.pop("async_creator_fn", self.asyncmy.connect) - return AsyncAdapt_asyncmy_connection( - self, - await_(creator_fn(*arg, **kw)), + return await_( + AsyncAdapt_asyncmy_connection.create( + self, + creator_fn(*arg, **kw), + ) ) @@ -145,18 +175,23 @@ class MySQLDialect_asyncmy(MySQLDialect_pymysql): has_terminate = True @classmethod - def import_dbapi(cls): + def import_dbapi(cls) -> DBAPIModule: return AsyncAdapt_asyncmy_dbapi(__import__("asyncmy")) - def do_terminate(self, dbapi_connection) -> None: + def do_terminate(self, dbapi_connection: DBAPIConnection) -> None: dbapi_connection.terminate() - def create_connect_args(self, url): + def create_connect_args(self, url: URL) -> ConnectArgsType: # type: ignore[override] # noqa: E501 return super().create_connect_args( url, _translate_args=dict(username="user", database="db") ) - def is_disconnect(self, e, connection, cursor): + def is_disconnect( + self, + e: DBAPIModule.Error, + connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]], + cursor: Optional[DBAPICursor], + ) -> bool: if super().is_disconnect(e, connection, cursor): return True else: @@ -165,13 +200,15 @@ def is_disconnect(self, e, connection, cursor): "not connected" in str_e or "network operation failed" in str_e ) - def _found_rows_client_flag(self): - from asyncmy.constants import CLIENT + def _found_rows_client_flag(self) -> int: + from asyncmy.constants import CLIENT # type: ignore - return CLIENT.FOUND_ROWS + return CLIENT.FOUND_ROWS # type: ignore[no-any-return] - def get_driver_connection(self, connection): - return connection._connection + def get_driver_connection( + self, connection: DBAPIConnection + ) -> AsyncIODBAPIConnection: + return connection._connection # type: ignore[no-any-return] dialect = MySQLDialect_asyncmy diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index fd60d7ba65c..908d1b5cb0e 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -1,10 +1,9 @@ # dialects/mysql/base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors r""" @@ -672,9 +671,6 @@ def connect(dbapi_connection, connection_record): {printsql}INSERT INTO my_table (id, data) VALUES (%s, %s) ON DUPLICATE KEY UPDATE data = %s, updated_at = CURRENT_TIMESTAMP -.. versionchanged:: 1.3 support for parameter-ordered UPDATE clause within - MySQL ON DUPLICATE KEY UPDATE - .. warning:: The :meth:`_mysql.Insert.on_duplicate_key_update` @@ -709,10 +705,6 @@ def connect(dbapi_connection, connection_record): When rendered, the "inserted" namespace will produce the expression ``VALUES()``. -.. versionadded:: 1.2 Added support for MySQL ON DUPLICATE KEY UPDATE clause - - - rowcount Support ---------------- @@ -817,9 +809,6 @@ def connect(dbapi_connection, connection_record): mariadb_with_parser="ngram", ) -.. versionadded:: 1.3 - - .. _mysql_foreign_keys: MySQL / MariaDB Foreign Keys @@ -1075,19 +1064,27 @@ class MyClass(Base): """ # noqa from __future__ import annotations -from array import array as _array from collections import defaultdict from itertools import compress import re +from typing import Any +from typing import Callable from typing import cast - +from typing import NoReturn +from typing import Optional +from typing import overload +from typing import Sequence +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union + +from . import _mariadb_shim from . import reflection as _reflection from .enumerated import ENUM from .enumerated import SET from .json import JSON from .json import JSONIndexType from .json import JSONPathType -from .reserved_words import RESERVED_WORDS_MARIADB from .reserved_words import RESERVED_WORDS_MYSQL from .types import _FloatType from .types import _IntegerType @@ -1123,11 +1120,9 @@ class MyClass(Base): from .types import YEAR from ... import exc from ... import literal_column -from ... import log from ... import schema as sa_schema from ... import sql from ... import util -from ...engine import cursor as _cursor from ...engine import default from ...engine import reflection from ...engine.reflection import ReflectionDefaults @@ -1141,17 +1136,54 @@ class MyClass(Base): from ...sql import util as sql_util from ...sql import visitors from ...sql.compiler import InsertmanyvaluesSentinelOpts -from ...sql.compiler import SQLCompiler -from ...sql.schema import SchemaConst from ...types import BINARY from ...types import BLOB from ...types import BOOLEAN from ...types import DATE +from ...types import LargeBinary from ...types import UUID from ...types import VARBINARY from ...util import topological - +if TYPE_CHECKING: + + from ...dialects.mysql import expression + from ...dialects.mysql.dml import DMLLimitClause + from ...dialects.mysql.dml import OnDuplicateClause + from ...engine.base import Connection + from ...engine.cursor import CursorResult + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import DBAPICursor + from ...engine.interfaces import DBAPIModule + from ...engine.interfaces import IsolationLevel + from ...engine.interfaces import PoolProxiedConnection + from ...engine.interfaces import ReflectedCheckConstraint + from ...engine.interfaces import ReflectedColumn + from ...engine.interfaces import ReflectedForeignKeyConstraint + from ...engine.interfaces import ReflectedIndex + from ...engine.interfaces import ReflectedPrimaryKeyConstraint + from ...engine.interfaces import ReflectedTableComment + from ...engine.interfaces import ReflectedUniqueConstraint + from ...engine.result import _Ts + from ...engine.row import Row + from ...schema import Table + from ...sql import ddl + from ...sql import selectable + from ...sql.dml import _DMLTableElement + from ...sql.dml import Delete + from ...sql.dml import Update + from ...sql.dml import ValuesBase + from ...sql.functions import aggregate_strings + from ...sql.functions import random + from ...sql.functions import rollup + from ...sql.functions import sysdate + from ...sql.sqltypes import _JSON_VALUE + from ...sql.type_api import TypeEngine + from ...sql.visitors import ExternallyTraversible + from ...util.typing import TupleAny + from ...util.typing import Unpack + +_T = TypeVar("_T", bound=Any) SET_RE = re.compile( r"\s*SET\s+(?:(?:GLOBAL|SESSION)\s+)?\w", re.I | re.UNICODE ) @@ -1245,90 +1277,81 @@ class MyClass(Base): } -class MySQLExecutionContext(default.DefaultExecutionContext): - def post_exec(self): - if ( - self.isdelete - and cast(SQLCompiler, self.compiled).effective_returning - and not self.cursor.description - ): - # All MySQL/mariadb drivers appear to not include - # cursor.description for DELETE..RETURNING with no rows if the - # WHERE criteria is a straight "false" condition such as our EMPTY - # IN condition. manufacture an empty result in this case (issue - # #10505) - # - # taken from cx_Oracle implementation - self.cursor_fetch_strategy = ( - _cursor.FullyBufferedCursorFetchStrategy( - self.cursor, - [ - (entry.keyname, None) - for entry in cast( - SQLCompiler, self.compiled - )._result_columns - ], - [], - ) - ) - - def create_server_side_cursor(self): +class MySQLExecutionContext( + _mariadb_shim.MariadbExecutionContextShim, default.DefaultExecutionContext +): + def create_server_side_cursor(self) -> DBAPICursor: if self.dialect.supports_server_side_cursors: - return self._dbapi_connection.cursor(self.dialect._sscursor) + return self._dbapi_connection.cursor( + self.dialect._sscursor # type: ignore[attr-defined] + ) else: raise NotImplementedError() - def fire_sequence(self, seq, type_): - return self._execute_scalar( - ( - "select nextval(%s)" - % self.identifier_preparer.format_sequence(seq) - ), - type_, - ) - -class MySQLCompiler(compiler.SQLCompiler): +class MySQLCompiler( + _mariadb_shim.MariaDBSQLCompilerShim, compiler.SQLCompiler +): + dialect: MySQLDialect render_table_with_column_in_update_from = True """Overridden from base SQLCompiler value""" extract_map = compiler.SQLCompiler.extract_map.copy() extract_map.update({"milliseconds": "millisecond"}) - def default_from(self): + def default_from(self) -> str: """Called when a ``SELECT`` statement has no froms, and no ``FROM`` clause is to be appended. """ if self.stack: stmt = self.stack[-1]["selectable"] - if stmt._where_criteria: + if stmt._where_criteria: # type: ignore[attr-defined] return " FROM DUAL" return "" - def visit_random_func(self, fn, **kw): + def visit_random_func(self, fn: random, **kw: Any) -> str: return "rand%s" % self.function_argspec(fn) - def visit_rollup_func(self, fn, **kw): + def visit_rollup_func(self, fn: rollup[Any], **kw: Any) -> str: clause = ", ".join( elem._compiler_dispatch(self, **kw) for elem in fn.clauses ) return f"{clause} WITH ROLLUP" - def visit_aggregate_strings_func(self, fn, **kw): - expr, delimeter = ( - elem._compiler_dispatch(self, **kw) for elem in fn.clauses - ) - return f"group_concat({expr} SEPARATOR {delimeter})" + def visit_aggregate_strings_func( + self, fn: aggregate_strings, **kw: Any + ) -> str: + + order_by = getattr(fn.clauses, "aggregate_order_by", None) + + cl = list(fn.clauses) + expr, delimiter = cl[0:2] - def visit_sequence(self, seq, **kw): - return "nextval(%s)" % self.preparer.format_sequence(seq) + literal_exec = dict(kw) + literal_exec["literal_execute"] = True - def visit_sysdate_func(self, fn, **kw): + if order_by is not None: + return ( + f"group_concat({expr._compiler_dispatch(self, **kw)} " + f"ORDER BY {order_by._compiler_dispatch(self, **kw)} " + "SEPARATOR " + f"{delimiter._compiler_dispatch(self, **literal_exec)})" + ) + else: + return ( + f"group_concat({expr._compiler_dispatch(self, **kw)} " + "SEPARATOR " + f"{delimiter._compiler_dispatch(self, **literal_exec)})" + ) + + def visit_sysdate_func(self, fn: sysdate, **kw: Any) -> str: return "SYSDATE()" - def _render_json_extract_from_binary(self, binary, operator, **kw): + def _render_json_extract_from_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: # note we are intentionally calling upon the process() calls in the # order in which they appear in the SQL String as this is used # by positional parameter rendering @@ -1355,9 +1378,10 @@ def _render_json_extract_from_binary(self, binary, operator, **kw): ) ) elif binary.type._type_affinity in (sqltypes.Numeric, sqltypes.Float): + binary_type = cast(sqltypes.Numeric[Any], binary.type) if ( - binary.type.scale is not None - and binary.type.precision is not None + binary_type.scale is not None + and binary_type.precision is not None ): # using DECIMAL here because MySQL does not recognize NUMERIC type_expression = ( @@ -1365,8 +1389,8 @@ def _render_json_extract_from_binary(self, binary, operator, **kw): % ( self.process(binary.left, **kw), self.process(binary.right, **kw), - binary.type.precision, - binary.type.scale, + binary_type.precision, + binary_type.scale, ) ) else: @@ -1400,15 +1424,22 @@ def _render_json_extract_from_binary(self, binary, operator, **kw): return case_expression + " " + type_expression + " END" - def visit_json_getitem_op_binary(self, binary, operator, **kw): + def visit_json_getitem_op_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: return self._render_json_extract_from_binary(binary, operator, **kw) - def visit_json_path_getitem_op_binary(self, binary, operator, **kw): + def visit_json_path_getitem_op_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: return self._render_json_extract_from_binary(binary, operator, **kw) - def visit_on_duplicate_key_update(self, on_duplicate, **kw): - statement = self.current_executable + def visit_on_duplicate_key_update( + self, on_duplicate: OnDuplicateClause, **kw: Any + ) -> str: + statement: ValuesBase = self.current_executable + cols: list[elements.KeyedColumnElement[Any]] if on_duplicate._parameter_ordering: parameter_ordering = [ coercions.expect(roles.DMLColumnRole, key) @@ -1421,7 +1452,7 @@ def visit_on_duplicate_key_update(self, on_duplicate, **kw): if key in statement.table.c ] + [c for c in statement.table.c if c.key not in ordered_keys] else: - cols = statement.table.c + cols = list(statement.table.c) clauses = [] @@ -1430,7 +1461,7 @@ def visit_on_duplicate_key_update(self, on_duplicate, **kw): ) if requires_mysql8_alias: - if statement.table.name.lower() == "new": + if statement.table.name.lower() == "new": # type: ignore[union-attr] # noqa: E501 _on_dup_alias_name = "new_1" else: _on_dup_alias_name = "new" @@ -1444,24 +1475,26 @@ def visit_on_duplicate_key_update(self, on_duplicate, **kw): for column in (col for col in cols if col.key in on_duplicate_update): val = on_duplicate_update[column.key] - def replace(obj): + def replace( + element: ExternallyTraversible, **kw: Any + ) -> Optional[ExternallyTraversible]: if ( - isinstance(obj, elements.BindParameter) - and obj.type._isnull + isinstance(element, elements.BindParameter) + and element.type._isnull ): - return obj._with_binary_element_type(column.type) + return element._with_binary_element_type(column.type) elif ( - isinstance(obj, elements.ColumnClause) - and obj.table is on_duplicate.inserted_alias + isinstance(element, elements.ColumnClause) + and element.table is on_duplicate.inserted_alias ): if requires_mysql8_alias: column_literal_clause = ( f"{_on_dup_alias_name}." - f"{self.preparer.quote(obj.name)}" + f"{self.preparer.quote(element.name)}" ) else: column_literal_clause = ( - f"VALUES({self.preparer.quote(obj.name)})" + f"VALUES({self.preparer.quote(element.name)})" ) return literal_column(column_literal_clause) else: @@ -1480,8 +1513,8 @@ def replace(obj): "Additional column names not matching " "any column keys in table '%s': %s" % ( - self.statement.table.name, - (", ".join("'%s'" % c for c in non_matching)), + self.statement.table.name, # type: ignore[union-attr] + ", ".join("'%s'" % c for c in non_matching), ) ) @@ -1494,13 +1527,15 @@ def replace(obj): return f"ON DUPLICATE KEY UPDATE {', '.join(clauses)}" def visit_concat_op_expression_clauselist( - self, clauselist, operator, **kw - ): - return "concat(%s)" % ( - ", ".join(self.process(elem, **kw) for elem in clauselist.clauses) + self, clauselist: elements.ClauseList, operator: Any, **kw: Any + ) -> str: + return "concat(%s)" % ", ".join( + self.process(elem, **kw) for elem in clauselist.clauses ) - def visit_concat_op_binary(self, binary, operator, **kw): + def visit_concat_op_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: return "concat(%s, %s)" % ( self.process(binary.left, **kw), self.process(binary.right, **kw), @@ -1523,10 +1558,12 @@ def visit_concat_op_binary(self, binary, operator, **kw): "WITH QUERY EXPANSION", ) - def visit_mysql_match(self, element, **kw): + def visit_mysql_match(self, element: expression.match, **kw: Any) -> str: return self.visit_match_op_binary(element, element.operator, **kw) - def visit_match_op_binary(self, binary, operator, **kw): + def visit_match_op_binary( + self, binary: expression.match, operator: Any, **kw: Any + ) -> str: """ Note that `mysql_boolean_mode` is enabled by default because of backward compatibility @@ -1547,12 +1584,11 @@ def visit_match_op_binary(self, binary, operator, **kw): "with_query_expansion=%s" % query_expansion, ) - flags = ", ".join(flags) + flags_str = ", ".join(flags) - raise exc.CompileError("Invalid MySQL match flags: %s" % flags) + raise exc.CompileError("Invalid MySQL match flags: %s" % flags_str) - match_clause = binary.left - match_clause = self.process(match_clause, **kw) + match_clause = self.process(binary.left, **kw) against_clause = self.process(binary.right, **kw) if any(flag_combination): @@ -1561,21 +1597,25 @@ def visit_match_op_binary(self, binary, operator, **kw): flag_combination, ) - against_clause = [against_clause] - against_clause.extend(flag_expressions) - - against_clause = " ".join(against_clause) + against_clause = " ".join([against_clause, *flag_expressions]) return "MATCH (%s) AGAINST (%s)" % (match_clause, against_clause) - def get_from_hint_text(self, table, text): + def get_from_hint_text( + self, table: selectable.FromClause, text: Optional[str] + ) -> Optional[str]: return text - def visit_typeclause(self, typeclause, type_=None, **kw): + def visit_typeclause( + self, + typeclause: elements.TypeClause, + type_: Optional[TypeEngine[Any]] = None, + **kw: Any, + ) -> Optional[str]: if type_ is None: type_ = typeclause.type.dialect_impl(self.dialect) if isinstance(type_, sqltypes.TypeDecorator): - return self.visit_typeclause(typeclause, type_.impl, **kw) + return self.visit_typeclause(typeclause, type_.impl, **kw) # type: ignore[arg-type] # noqa: E501 elif isinstance(type_, sqltypes.Integer): if getattr(type_, "unsigned", False): return "UNSIGNED INTEGER" @@ -1614,7 +1654,7 @@ def visit_typeclause(self, typeclause, type_=None, **kw): else: return None - def visit_cast(self, cast, **kw): + def visit_cast(self, cast: elements.Cast[Any], **kw: Any) -> str: type_ = self.process(cast.typeclause) if type_ is None: util.warn( @@ -1628,7 +1668,9 @@ def visit_cast(self, cast, **kw): return "CAST(%s AS %s)" % (self.process(cast.clause, **kw), type_) - def render_literal_value(self, value, type_): + def render_literal_value( + self, value: Optional[str], type_: TypeEngine[Any] + ) -> str: value = super().render_literal_value(value, type_) if self.dialect._backslash_escapes: value = value.replace("\\", "\\\\") @@ -1636,13 +1678,15 @@ def render_literal_value(self, value, type_): # override native_boolean=False behavior here, as # MySQL still supports native boolean - def visit_true(self, element, **kw): + def visit_true(self, expr: elements.True_, **kw: Any) -> str: return "true" - def visit_false(self, element, **kw): + def visit_false(self, expr: elements.False_, **kw: Any) -> str: return "false" - def get_select_precolumns(self, select, **kw): + def get_select_precolumns( + self, select: selectable.Select[Any], **kw: Any + ) -> str: """Add special MySQL keywords in place of DISTINCT. .. deprecated:: 1.4 This usage is deprecated. @@ -1662,7 +1706,13 @@ def get_select_precolumns(self, select, **kw): return super().get_select_precolumns(select, **kw) - def visit_join(self, join, asfrom=False, from_linter=None, **kwargs): + def visit_join( + self, + join: selectable.Join, + asfrom: bool = False, + from_linter: Optional[compiler.FromLinter] = None, + **kwargs: Any, + ) -> str: if from_linter: from_linter.edges.add((join.left, join.right)) @@ -1683,18 +1733,24 @@ def visit_join(self, join, asfrom=False, from_linter=None, **kwargs): join.right, asfrom=True, from_linter=from_linter, **kwargs ), " ON ", - self.process(join.onclause, from_linter=from_linter, **kwargs), + self.process(join.onclause, from_linter=from_linter, **kwargs), # type: ignore[arg-type] # noqa: E501 ) ) - def for_update_clause(self, select, **kw): + def for_update_clause( + self, select: selectable.GenerativeSelect, **kw: Any + ) -> str: + assert select._for_update_arg is not None if select._for_update_arg.read: - tmp = " LOCK IN SHARE MODE" + if self.dialect.use_mysql_for_share: + tmp = " FOR SHARE" + else: + tmp = " LOCK IN SHARE MODE" else: tmp = " FOR UPDATE" if select._for_update_arg.of and self.dialect.supports_for_update_of: - tables = util.OrderedSet() + tables: util.OrderedSet[elements.ClauseElement] = util.OrderedSet() for c in select._for_update_arg.of: tables.update(sql_util.surface_selectables_only(c)) @@ -1711,7 +1767,9 @@ def for_update_clause(self, select, **kw): return tmp - def limit_clause(self, select, **kw): + def limit_clause( + self, select: selectable.GenerativeSelect, **kw: Any + ) -> str: # MySQL supports: # LIMIT # LIMIT , @@ -1747,10 +1805,13 @@ def limit_clause(self, select, **kw): self.process(limit_clause, **kw), ) else: + assert limit_clause is not None # No offset provided, so just use the limit return " \n LIMIT %s" % (self.process(limit_clause, **kw),) - def update_post_criteria_clause(self, update_stmt, **kw): + def update_post_criteria_clause( + self, update_stmt: Update, **kw: Any + ) -> Optional[str]: limit = update_stmt.kwargs.get("%s_limit" % self.dialect.name, None) supertext = super().update_post_criteria_clause(update_stmt, **kw) @@ -1763,7 +1824,9 @@ def update_post_criteria_clause(self, update_stmt, **kw): else: return supertext - def delete_post_criteria_clause(self, delete_stmt, **kw): + def delete_post_criteria_clause( + self, delete_stmt: Delete, **kw: Any + ) -> Optional[str]: limit = delete_stmt.kwargs.get("%s_limit" % self.dialect.name, None) supertext = super().delete_post_criteria_clause(delete_stmt, **kw) @@ -1776,11 +1839,19 @@ def delete_post_criteria_clause(self, delete_stmt, **kw): else: return supertext - def visit_mysql_dml_limit_clause(self, element, **kw): + def visit_mysql_dml_limit_clause( + self, element: DMLLimitClause, **kw: Any + ) -> str: kw["literal_execute"] = True return f"LIMIT {self.process(element._limit_clause, **kw)}" - def update_tables_clause(self, update_stmt, from_table, extra_froms, **kw): + def update_tables_clause( + self, + update_stmt: Update, + from_table: _DMLTableElement, + extra_froms: list[selectable.FromClause], + **kw: Any, + ) -> str: kw["asfrom"] = True return ", ".join( t._compiler_dispatch(self, **kw) @@ -1788,11 +1859,22 @@ def update_tables_clause(self, update_stmt, from_table, extra_froms, **kw): ) def update_from_clause( - self, update_stmt, from_table, extra_froms, from_hints, **kw - ): + self, + update_stmt: Update, + from_table: _DMLTableElement, + extra_froms: list[selectable.FromClause], + from_hints: Any, + **kw: Any, + ) -> None: return None - def delete_table_clause(self, delete_stmt, from_table, extra_froms, **kw): + def delete_table_clause( + self, + delete_stmt: Delete, + from_table: _DMLTableElement, + extra_froms: list[selectable.FromClause], + **kw: Any, + ) -> str: """If we have extra froms make sure we render any alias as hint.""" ashint = False if extra_froms: @@ -1802,8 +1884,13 @@ def delete_table_clause(self, delete_stmt, from_table, extra_froms, **kw): ) def delete_extra_from_clause( - self, delete_stmt, from_table, extra_froms, from_hints, **kw - ): + self, + delete_stmt: Delete, + from_table: _DMLTableElement, + extra_froms: list[selectable.FromClause], + from_hints: Any, + **kw: Any, + ) -> str: """Render the DELETE .. USING clause specific to MySQL.""" kw["asfrom"] = True return "USING " + ", ".join( @@ -1811,10 +1898,11 @@ def delete_extra_from_clause( for t in [from_table] + extra_froms ) - def visit_empty_set_expr(self, element_types, **kw): + def visit_empty_set_expr( + self, element_types: list[TypeEngine[Any]], **kw: Any + ) -> str: return ( - "SELECT %(outer)s FROM (SELECT %(inner)s) " - "as _empty_set WHERE 1!=1" + "SELECT %(outer)s FROM (SELECT %(inner)s) as _empty_set WHERE 1!=1" % { "inner": ", ".join( "1 AS _in_%s" % idx @@ -1826,81 +1914,128 @@ def visit_empty_set_expr(self, element_types, **kw): } ) - def visit_is_distinct_from_binary(self, binary, operator, **kw): + def visit_is_distinct_from_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: return "NOT (%s <=> %s)" % ( self.process(binary.left), self.process(binary.right), ) - def visit_is_not_distinct_from_binary(self, binary, operator, **kw): + def visit_is_not_distinct_from_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: return "%s <=> %s" % ( self.process(binary.left), self.process(binary.right), ) - def _mariadb_regexp_flags(self, flags, pattern, **kw): - return "CONCAT('(?', %s, ')', %s)" % ( + def _mysql_regexp_match( + self, + op_string: str, + binary: elements.BinaryExpression[Any], + operator: Any, + **kw: Any, + ) -> str: + flags = binary.modifiers["flags"] + + text = "REGEXP_LIKE(%s, %s, %s)" % ( + self.process(binary.left, **kw), + self.process(binary.right, **kw), self.render_literal_value(flags, sqltypes.STRINGTYPE), - self.process(pattern, **kw), ) + if op_string == " NOT REGEXP ": + return "NOT %s" % text + else: + return text - def _regexp_match(self, op_string, binary, operator, **kw): + def _regexp_match( + self, + op_string: str, + binary: elements.BinaryExpression[Any], + operator: Any, + **kw: Any, + ) -> str: + assert binary.modifiers is not None flags = binary.modifiers["flags"] if flags is None: return self._generate_generic_binary(binary, op_string, **kw) - elif self.dialect.is_mariadb: - return "%s%s%s" % ( - self.process(binary.left, **kw), - op_string, - self._mariadb_regexp_flags(flags, binary.right), - ) else: - text = "REGEXP_LIKE(%s, %s, %s)" % ( - self.process(binary.left, **kw), - self.process(binary.right, **kw), - self.render_literal_value(flags, sqltypes.STRINGTYPE), + return self.dialect._dispatch_for_vendor( + self._mysql_regexp_match, + self._mariadb_regexp_match, + op_string, + binary, + operator, + **kw, ) - if op_string == " NOT REGEXP ": - return "NOT %s" % text - else: - return text - def visit_regexp_match_op_binary(self, binary, operator, **kw): + def visit_regexp_match_op_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: return self._regexp_match(" REGEXP ", binary, operator, **kw) - def visit_not_regexp_match_op_binary(self, binary, operator, **kw): + def visit_not_regexp_match_op_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: return self._regexp_match(" NOT REGEXP ", binary, operator, **kw) - def visit_regexp_replace_op_binary(self, binary, operator, **kw): + def visit_regexp_replace_op_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: + assert binary.modifiers is not None flags = binary.modifiers["flags"] if flags is None: return "REGEXP_REPLACE(%s, %s)" % ( self.process(binary.left, **kw), self.process(binary.right, **kw), ) - elif self.dialect.is_mariadb: - return "REGEXP_REPLACE(%s, %s, %s)" % ( - self.process(binary.left, **kw), - self._mariadb_regexp_flags(flags, binary.right.clauses[0]), - self.process(binary.right.clauses[1], **kw), - ) else: - return "REGEXP_REPLACE(%s, %s, %s)" % ( - self.process(binary.left, **kw), - self.process(binary.right, **kw), - self.render_literal_value(flags, sqltypes.STRINGTYPE), + return self.dialect._dispatch_for_vendor( + self._mysql_regexp_replace_op_binary, + self._mariadb_regexp_replace_op_binary, + binary, + operator, + **kw, ) + def _mysql_regexp_replace_op_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: + flags = binary.modifiers["flags"] + + return "REGEXP_REPLACE(%s, %s, %s)" % ( + self.process(binary.left, **kw), + self.process(binary.right, **kw), + self.render_literal_value(flags, sqltypes.STRINGTYPE), + ) -class MySQLDDLCompiler(compiler.DDLCompiler): - def get_column_specification(self, column, **kw): + +class MySQLDDLCompiler( + _mariadb_shim.MariaDBDDLCompilerShim, compiler.DDLCompiler +): + dialect: MySQLDialect + + def get_column_specification( + self, column: sa_schema.Column[Any], **kw: Any + ) -> str: """Builds column DDL.""" - if ( - self.dialect.is_mariadb is True - and column.computed is not None - and column._user_defined_nullable is SchemaConst.NULL_UNSPECIFIED - ): - column.nullable = True + + return self.dialect._dispatch_for_vendor( + self._mysql_get_column_specification, + self._mariadb_get_column_specification, + column, + **kw, + ) + + def _mysql_get_column_specification( + self, + column: sa_schema.Column[Any], + *, + _force_column_to_nullable: bool = False, + **kw: Any, + ) -> str: + colspec = [ self.preparer.format_column(column), self.dialect.type_compiler_instance.process( @@ -1916,7 +2051,7 @@ def get_column_specification(self, column, **kw): sqltypes.TIMESTAMP, ) - if not column.nullable: + if not column.nullable and not _force_column_to_nullable: colspec.append("NOT NULL") # see: https://docs.sqlalchemy.org/en/latest/dialects/mysql.html#mysql_timestamp_null # noqa @@ -1946,19 +2081,25 @@ def get_column_specification(self, column, **kw): colspec.append("AUTO_INCREMENT") else: default = self.get_column_default_string(column) + if default is not None: if ( - isinstance( - column.server_default.arg, functions.FunctionElement + self.dialect._support_default_function + and not re.match(r"^\s*[\'\"\(]", default) + and not re.search(r"ON +UPDATE", default, re.I) + and not re.match( + r"\bnow\(\d+\)|\bcurrent_timestamp\(\d+\)", + default, + re.I, ) - and self.dialect._support_default_function + and re.match(r".*\W.*", default) ): colspec.append(f"DEFAULT ({default})") else: colspec.append("DEFAULT " + default) return " ".join(colspec) - def post_create_table(self, table): + def post_create_table(self, table: sa_schema.Table) -> str: """Build table-level CREATE options like ENGINE and COLLATE.""" table_opts = [] @@ -2042,16 +2183,16 @@ def post_create_table(self, table): return " ".join(table_opts) - def visit_create_index(self, create, **kw): + def visit_create_index(self, create: ddl.CreateIndex, **kw: Any) -> str: # type: ignore[override] # noqa: E501 index = create.element self._verify_index_table(index) preparer = self.preparer - table = preparer.format_table(index.table) + table = preparer.format_table(index.table) # type: ignore[arg-type] columns = [ self.sql_compiler.process( ( - elements.Grouping(expr) + elements.Grouping(expr) # type: ignore[arg-type] if ( isinstance(expr, elements.BinaryExpression) or ( @@ -2075,7 +2216,7 @@ def visit_create_index(self, create, **kw): if index.unique: text += "UNIQUE " - index_prefix = index.kwargs.get("%s_prefix" % self.dialect.name, None) + index_prefix = index.get_dialect_option(self.dialect, "prefix") if index_prefix: text += index_prefix + " " @@ -2084,16 +2225,16 @@ def visit_create_index(self, create, **kw): text += "IF NOT EXISTS " text += "%s ON %s " % (name, table) - length = index.dialect_options[self.dialect.name]["length"] + length = index.get_dialect_option(self.dialect, "length") if length is not None: if isinstance(length, dict): # length value can be a (column_name --> integer value) # mapping specifying the prefix length for each column of the # index - columns = ", ".join( + columns_str = ", ".join( ( - "%s(%d)" % (expr, length[col.name]) - if col.name in length + "%s(%d)" % (expr, length[col.name]) # type: ignore[union-attr] # noqa: E501 + if col.name in length # type: ignore[union-attr] else ( "%s(%d)" % (expr, length[expr]) if expr in length @@ -2105,31 +2246,39 @@ def visit_create_index(self, create, **kw): else: # or can be an integer value specifying the same # prefix length for all columns of the index - columns = ", ".join( + columns_str = ", ".join( "%s(%d)" % (col, length) for col in columns ) else: - columns = ", ".join(columns) - text += "(%s)" % columns + columns_str = ", ".join(columns) + text += "(%s)" % columns_str - parser = index.dialect_options["mysql"]["with_parser"] + parser = index.get_dialect_option( + self.dialect, "with_parser", deprecated_fallback="mysql" + ) if parser is not None: text += " WITH PARSER %s" % (parser,) - using = index.dialect_options["mysql"]["using"] + using = index.get_dialect_option( + self.dialect, "using", deprecated_fallback="mysql" + ) if using is not None: text += " USING %s" % (preparer.quote(using)) return text - def visit_primary_key_constraint(self, constraint, **kw): + def visit_primary_key_constraint( + self, constraint: sa_schema.PrimaryKeyConstraint, **kw: Any + ) -> str: text = super().visit_primary_key_constraint(constraint) - using = constraint.dialect_options["mysql"]["using"] + using = constraint.get_dialect_option( + self.dialect, "using", deprecated_fallback="mysql" + ) if using: text += " USING %s" % (self.preparer.quote(using)) return text - def visit_drop_index(self, drop, **kw): + def visit_drop_index(self, drop: ddl.DropIndex, **kw: Any) -> str: index = drop.element text = "\nDROP INDEX " if drop.if_exists: @@ -2137,10 +2286,12 @@ def visit_drop_index(self, drop, **kw): return text + "%s ON %s" % ( self._prepared_index_name(index, include_schema=False), - self.preparer.format_table(index.table), + self.preparer.format_table(index.table), # type: ignore[arg-type] ) - def visit_drop_constraint(self, drop, **kw): + def visit_drop_constraint( + self, drop: ddl.DropConstraint, **kw: Any + ) -> str: constraint = drop.element if isinstance(constraint, sa_schema.ForeignKeyConstraint): qual = "FOREIGN KEY " @@ -2152,11 +2303,12 @@ def visit_drop_constraint(self, drop, **kw): qual = "INDEX " const = self.preparer.format_constraint(constraint) elif isinstance(constraint, sa_schema.CheckConstraint): - if self.dialect.is_mariadb: - qual = "CONSTRAINT " - else: - qual = "CHECK " - const = self.preparer.format_constraint(constraint) + return self.dialect._dispatch_for_vendor( + self._mysql_visit_drop_check_constraint, + self._mariadb_visit_drop_check_constraint, + drop, + **kw, + ) else: qual = "" const = self.preparer.format_constraint(constraint) @@ -2166,7 +2318,21 @@ def visit_drop_constraint(self, drop, **kw): const, ) - def define_constraint_match(self, constraint): + def _mysql_visit_drop_check_constraint( + self, drop: ddl.DropConstraint, **kw: Any + ) -> str: + constraint = drop.element + qual = "CHECK " + const = self.preparer.format_constraint(constraint) + return "ALTER TABLE %s DROP %s%s" % ( + self.preparer.format_table(constraint.table), + qual, + const, + ) + + def define_constraint_match( + self, constraint: sa_schema.ForeignKeyConstraint + ) -> str: if constraint.match is not None: raise exc.CompileError( "MySQL ignores the 'MATCH' keyword while at the same time " @@ -2174,7 +2340,9 @@ def define_constraint_match(self, constraint): ) return "" - def visit_set_table_comment(self, create, **kw): + def visit_set_table_comment( + self, create: ddl.SetTableComment, **kw: Any + ) -> str: return "ALTER TABLE %s COMMENT %s" % ( self.preparer.format_table(create.element), self.sql_compiler.render_literal_value( @@ -2182,12 +2350,16 @@ def visit_set_table_comment(self, create, **kw): ), ) - def visit_drop_table_comment(self, create, **kw): + def visit_drop_table_comment( + self, drop: ddl.DropTableComment, **kw: Any + ) -> str: return "ALTER TABLE %s COMMENT ''" % ( - self.preparer.format_table(create.element) + self.preparer.format_table(drop.element) ) - def visit_set_column_comment(self, create, **kw): + def visit_set_column_comment( + self, create: ddl.SetColumnComment, **kw: Any + ) -> str: return "ALTER TABLE %s CHANGE %s %s" % ( self.preparer.format_table(create.element.table), self.preparer.format_column(create.element), @@ -2195,8 +2367,10 @@ def visit_set_column_comment(self, create, **kw): ) -class MySQLTypeCompiler(compiler.GenericTypeCompiler): - def _extend_numeric(self, type_, spec): +class MySQLTypeCompiler( + _mariadb_shim.MariaDBTypeCompilerShim, compiler.GenericTypeCompiler +): + def _extend_numeric(self, type_: _NumericCommonType, spec: str) -> str: "Extend a numeric-type declaration with MySQL specific extensions." if not self._mysql_type(type_): @@ -2208,13 +2382,15 @@ def _extend_numeric(self, type_, spec): spec += " ZEROFILL" return spec - def _extend_string(self, type_, defaults, spec): + def _extend_string( + self, type_: _StringType, defaults: dict[str, Any], spec: str + ) -> str: """Extend a string-type declaration with standard SQL CHARACTER SET / COLLATE annotations and MySQL specific extensions. """ - def attr(name): + def attr(name: str) -> Any: return getattr(type_, name, defaults.get(name)) if attr("charset"): @@ -2224,6 +2400,7 @@ def attr(name): elif attr("unicode"): charset = "UNICODE" else: + charset = None if attr("collation"): @@ -2242,10 +2419,10 @@ def attr(name): [c for c in (spec, charset, collation) if c is not None] ) - def _mysql_type(self, type_): + def _mysql_type(self, type_: Any) -> bool: return isinstance(type_, (_StringType, _NumericCommonType)) - def visit_NUMERIC(self, type_, **kw): + def visit_NUMERIC(self, type_: NUMERIC, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if type_.precision is None: return self._extend_numeric(type_, "NUMERIC") elif type_.scale is None: @@ -2260,7 +2437,7 @@ def visit_NUMERIC(self, type_, **kw): % {"precision": type_.precision, "scale": type_.scale}, ) - def visit_DECIMAL(self, type_, **kw): + def visit_DECIMAL(self, type_: DECIMAL, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if type_.precision is None: return self._extend_numeric(type_, "DECIMAL") elif type_.scale is None: @@ -2275,7 +2452,7 @@ def visit_DECIMAL(self, type_, **kw): % {"precision": type_.precision, "scale": type_.scale}, ) - def visit_DOUBLE(self, type_, **kw): + def visit_DOUBLE(self, type_: DOUBLE, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if type_.precision is not None and type_.scale is not None: return self._extend_numeric( type_, @@ -2285,7 +2462,7 @@ def visit_DOUBLE(self, type_, **kw): else: return self._extend_numeric(type_, "DOUBLE") - def visit_REAL(self, type_, **kw): + def visit_REAL(self, type_: REAL, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if type_.precision is not None and type_.scale is not None: return self._extend_numeric( type_, @@ -2295,7 +2472,7 @@ def visit_REAL(self, type_, **kw): else: return self._extend_numeric(type_, "REAL") - def visit_FLOAT(self, type_, **kw): + def visit_FLOAT(self, type_: FLOAT, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if ( self._mysql_type(type_) and type_.scale is not None @@ -2311,7 +2488,7 @@ def visit_FLOAT(self, type_, **kw): else: return self._extend_numeric(type_, "FLOAT") - def visit_INTEGER(self, type_, **kw): + def visit_INTEGER(self, type_: INTEGER, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if self._mysql_type(type_) and type_.display_width is not None: return self._extend_numeric( type_, @@ -2321,7 +2498,7 @@ def visit_INTEGER(self, type_, **kw): else: return self._extend_numeric(type_, "INTEGER") - def visit_BIGINT(self, type_, **kw): + def visit_BIGINT(self, type_: BIGINT, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if self._mysql_type(type_) and type_.display_width is not None: return self._extend_numeric( type_, @@ -2331,7 +2508,7 @@ def visit_BIGINT(self, type_, **kw): else: return self._extend_numeric(type_, "BIGINT") - def visit_MEDIUMINT(self, type_, **kw): + def visit_MEDIUMINT(self, type_: MEDIUMINT, **kw: Any) -> str: if self._mysql_type(type_) and type_.display_width is not None: return self._extend_numeric( type_, @@ -2341,7 +2518,7 @@ def visit_MEDIUMINT(self, type_, **kw): else: return self._extend_numeric(type_, "MEDIUMINT") - def visit_TINYINT(self, type_, **kw): + def visit_TINYINT(self, type_: TINYINT, **kw: Any) -> str: if self._mysql_type(type_) and type_.display_width is not None: return self._extend_numeric( type_, "TINYINT(%s)" % type_.display_width @@ -2349,7 +2526,7 @@ def visit_TINYINT(self, type_, **kw): else: return self._extend_numeric(type_, "TINYINT") - def visit_SMALLINT(self, type_, **kw): + def visit_SMALLINT(self, type_: SMALLINT, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if self._mysql_type(type_) and type_.display_width is not None: return self._extend_numeric( type_, @@ -2359,55 +2536,55 @@ def visit_SMALLINT(self, type_, **kw): else: return self._extend_numeric(type_, "SMALLINT") - def visit_BIT(self, type_, **kw): + def visit_BIT(self, type_: BIT, **kw: Any) -> str: if type_.length is not None: return "BIT(%s)" % type_.length else: return "BIT" - def visit_DATETIME(self, type_, **kw): + def visit_DATETIME(self, type_: DATETIME, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if getattr(type_, "fsp", None): - return "DATETIME(%d)" % type_.fsp + return "DATETIME(%d)" % type_.fsp # type: ignore[str-format] else: return "DATETIME" - def visit_DATE(self, type_, **kw): + def visit_DATE(self, type_: DATE, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 return "DATE" - def visit_TIME(self, type_, **kw): + def visit_TIME(self, type_: TIME, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if getattr(type_, "fsp", None): - return "TIME(%d)" % type_.fsp + return "TIME(%d)" % type_.fsp # type: ignore[str-format] else: return "TIME" - def visit_TIMESTAMP(self, type_, **kw): + def visit_TIMESTAMP(self, type_: TIMESTAMP, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if getattr(type_, "fsp", None): - return "TIMESTAMP(%d)" % type_.fsp + return "TIMESTAMP(%d)" % type_.fsp # type: ignore[str-format] else: return "TIMESTAMP" - def visit_YEAR(self, type_, **kw): + def visit_YEAR(self, type_: YEAR, **kw: Any) -> str: if type_.display_width is None: return "YEAR" else: return "YEAR(%s)" % type_.display_width - def visit_TEXT(self, type_, **kw): + def visit_TEXT(self, type_: TEXT, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if type_.length is not None: return self._extend_string(type_, {}, "TEXT(%d)" % type_.length) else: return self._extend_string(type_, {}, "TEXT") - def visit_TINYTEXT(self, type_, **kw): + def visit_TINYTEXT(self, type_: TINYTEXT, **kw: Any) -> str: return self._extend_string(type_, {}, "TINYTEXT") - def visit_MEDIUMTEXT(self, type_, **kw): + def visit_MEDIUMTEXT(self, type_: MEDIUMTEXT, **kw: Any) -> str: return self._extend_string(type_, {}, "MEDIUMTEXT") - def visit_LONGTEXT(self, type_, **kw): + def visit_LONGTEXT(self, type_: LONGTEXT, **kw: Any) -> str: return self._extend_string(type_, {}, "LONGTEXT") - def visit_VARCHAR(self, type_, **kw): + def visit_VARCHAR(self, type_: VARCHAR, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if type_.length is not None: return self._extend_string(type_, {}, "VARCHAR(%d)" % type_.length) else: @@ -2415,7 +2592,7 @@ def visit_VARCHAR(self, type_, **kw): "VARCHAR requires a length on dialect %s" % self.dialect.name ) - def visit_CHAR(self, type_, **kw): + def visit_CHAR(self, type_: CHAR, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if type_.length is not None: return self._extend_string( type_, {}, "CHAR(%(length)s)" % {"length": type_.length} @@ -2423,7 +2600,7 @@ def visit_CHAR(self, type_, **kw): else: return self._extend_string(type_, {}, "CHAR") - def visit_NVARCHAR(self, type_, **kw): + def visit_NVARCHAR(self, type_: NVARCHAR, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 # We'll actually generate the equiv. "NATIONAL VARCHAR" instead # of "NVARCHAR". if type_.length is not None: @@ -2437,7 +2614,7 @@ def visit_NVARCHAR(self, type_, **kw): "NVARCHAR requires a length on dialect %s" % self.dialect.name ) - def visit_NCHAR(self, type_, **kw): + def visit_NCHAR(self, type_: NCHAR, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 # We'll actually generate the equiv. # "NATIONAL CHAR" instead of "NCHAR". if type_.length is not None: @@ -2449,40 +2626,42 @@ def visit_NCHAR(self, type_, **kw): else: return self._extend_string(type_, {"national": True}, "CHAR") - def visit_UUID(self, type_, **kw): + def visit_UUID(self, type_: UUID[Any], **kw: Any) -> str: # type: ignore[override] # NOQA: E501 return "UUID" - def visit_VARBINARY(self, type_, **kw): - return "VARBINARY(%d)" % type_.length + def visit_VARBINARY(self, type_: VARBINARY, **kw: Any) -> str: + return "VARBINARY(%d)" % type_.length # type: ignore[str-format] - def visit_JSON(self, type_, **kw): + def visit_JSON(self, type_: JSON[Any], **kw: Any) -> str: return "JSON" - def visit_large_binary(self, type_, **kw): + def visit_large_binary(self, type_: LargeBinary, **kw: Any) -> str: return self.visit_BLOB(type_) - def visit_enum(self, type_, **kw): + def visit_enum(self, type_: ENUM, **kw: Any) -> str: # type: ignore[override] # NOQA: E501 if not type_.native_enum: return super().visit_enum(type_) else: return self._visit_enumerated_values("ENUM", type_, type_.enums) - def visit_BLOB(self, type_, **kw): + def visit_BLOB(self, type_: LargeBinary, **kw: Any) -> str: if type_.length is not None: return "BLOB(%d)" % type_.length else: return "BLOB" - def visit_TINYBLOB(self, type_, **kw): + def visit_TINYBLOB(self, type_: TINYBLOB, **kw: Any) -> str: return "TINYBLOB" - def visit_MEDIUMBLOB(self, type_, **kw): + def visit_MEDIUMBLOB(self, type_: MEDIUMBLOB, **kw: Any) -> str: return "MEDIUMBLOB" - def visit_LONGBLOB(self, type_, **kw): + def visit_LONGBLOB(self, type_: LONGBLOB, **kw: Any) -> str: return "LONGBLOB" - def _visit_enumerated_values(self, name, type_, enumerated_values): + def _visit_enumerated_values( + self, name: str, type_: _StringType, enumerated_values: Sequence[str] + ) -> str: quoted_enums = [] for e in enumerated_values: if self.dialect.identifier_preparer._double_percents: @@ -2492,20 +2671,27 @@ def _visit_enumerated_values(self, name, type_, enumerated_values): type_, {}, "%s(%s)" % (name, ",".join(quoted_enums)) ) - def visit_ENUM(self, type_, **kw): + def visit_ENUM(self, type_: ENUM, **kw: Any) -> str: return self._visit_enumerated_values("ENUM", type_, type_.enums) - def visit_SET(self, type_, **kw): + def visit_SET(self, type_: SET, **kw: Any) -> str: return self._visit_enumerated_values("SET", type_, type_.values) - def visit_BOOLEAN(self, type_, **kw): + def visit_BOOLEAN(self, type_: sqltypes.Boolean, **kw: Any) -> str: return "BOOL" -class MySQLIdentifierPreparer(compiler.IdentifierPreparer): +class MySQLIdentifierPreparer( + _mariadb_shim.MariaDBIdentifierPreparerShim, compiler.IdentifierPreparer +): reserved_words = RESERVED_WORDS_MYSQL - def __init__(self, dialect, server_ansiquotes=False, **kw): + def __init__( + self, + dialect: default.DefaultDialect, + server_ansiquotes: bool = False, + **kw: Any, + ): if not server_ansiquotes: quote = "`" else: @@ -2513,23 +2699,21 @@ def __init__(self, dialect, server_ansiquotes=False, **kw): super().__init__(dialect, initial_quote=quote, escape_quote=quote) - def _quote_free_identifiers(self, *ids): + def _quote_free_identifiers(self, *ids: Optional[str]) -> tuple[str, ...]: """Unilaterally identifier-quote any number of strings.""" return tuple([self.quote_identifier(i) for i in ids if i is not None]) -class MariaDBIdentifierPreparer(MySQLIdentifierPreparer): - reserved_words = RESERVED_WORDS_MARIADB - - -@log.class_logger -class MySQLDialect(default.DefaultDialect): +class MySQLDialect(_mariadb_shim.MariaDBShim, default.DefaultDialect): """Details of the MySQL dialect. Not used directly in application code. """ name = "mysql" + + is_mariadb = False + supports_statement_cache = True supports_alter = True @@ -2553,16 +2737,19 @@ class MySQLDialect(default.DefaultDialect): returns_native_bytes = True - supports_sequences = False # default for MySQL ... # ... may be updated to True for MariaDB 10.3+ in initialize() + supports_sequences = False sequences_optional = False - supports_for_update_of = False # default for MySQL ... # ... may be updated to True for MySQL 8+ in initialize() + supports_for_update_of = False - _requires_alias_for_on_duplicate_key = False # Only available ... - # ... in MySQL 8+ + # mysql 8.0.1 uses this syntax + use_mysql_for_share = False + + # Only available ... ... in MySQL 8+ + _requires_alias_for_on_duplicate_key = False # MySQL doesn't support "DEFAULT VALUES" but *does* support # "VALUES (DEFAULT)" @@ -2590,16 +2777,19 @@ class MySQLDialect(default.DefaultDialect): ddl_compiler = MySQLDDLCompiler type_compiler_cls = MySQLTypeCompiler ischema_names = ischema_names - preparer = MySQLIdentifierPreparer - - is_mariadb = False - _mariadb_normalized_version_info = None + preparer: type[MySQLIdentifierPreparer] = MySQLIdentifierPreparer # default SQL compilation settings - # these are modified upon initialize(), # i.e. first connect _backslash_escapes = True _server_ansiquotes = False + _casing = 0 + _support_default_function = True + _support_float_cast = False + + server_version_info: tuple[int, ...] + identifier_preparer: MySQLIdentifierPreparer construct_arguments = [ (sa_schema.Table, {"*": None}), @@ -2619,18 +2809,20 @@ class MySQLDialect(default.DefaultDialect): def __init__( self, - json_serializer=None, - json_deserializer=None, - is_mariadb=None, - **kwargs, - ): + json_serializer: Callable[[_JSON_VALUE], str] | None = None, + json_deserializer: Callable[[str], _JSON_VALUE] | None = None, + is_mariadb: Optional[bool] = None, + **kwargs: Any, + ) -> None: kwargs.pop("use_ansiquotes", None) # legacy default.DefaultDialect.__init__(self, **kwargs) self._json_serializer = json_serializer self._json_deserializer = json_deserializer - self._set_mariadb(is_mariadb, None) + self._set_mariadb(is_mariadb, ()) - def get_isolation_level_values(self, dbapi_conn): + def get_isolation_level_values( + self, dbapi_conn: DBAPIConnection + ) -> Sequence[IsolationLevel]: return ( "SERIALIZABLE", "READ UNCOMMITTED", @@ -2638,13 +2830,17 @@ def get_isolation_level_values(self, dbapi_conn): "REPEATABLE READ", ) - def set_isolation_level(self, dbapi_connection, level): + def set_isolation_level( + self, dbapi_connection: DBAPIConnection, level: IsolationLevel + ) -> None: cursor = dbapi_connection.cursor() cursor.execute(f"SET SESSION TRANSACTION ISOLATION LEVEL {level}") cursor.execute("COMMIT") cursor.close() - def get_isolation_level(self, dbapi_connection): + def get_isolation_level( + self, dbapi_connection: DBAPIConnection + ) -> IsolationLevel: cursor = dbapi_connection.cursor() if self._is_mysql and self.server_version_info >= (5, 7, 20): cursor.execute("SELECT @@transaction_isolation") @@ -2661,46 +2857,33 @@ def get_isolation_level(self, dbapi_connection): cursor.close() if isinstance(val, bytes): val = val.decode() - return val.upper().replace("-", " ") - - @classmethod - def _is_mariadb_from_url(cls, url): - dbapi = cls.import_dbapi() - dialect = cls(dbapi=dbapi) - - cargs, cparams = dialect.create_connect_args(url) - conn = dialect.connect(*cargs, **cparams) - try: - cursor = conn.cursor() - cursor.execute("SELECT VERSION() LIKE '%MariaDB%'") - val = cursor.fetchone()[0] - except: - raise - else: - return bool(val) - finally: - conn.close() + return val.upper().replace("-", " ") # type: ignore[no-any-return] - def _get_server_version_info(self, connection): + def _get_server_version_info( + self, connection: Connection + ) -> tuple[int, ...]: # get database server version info explicitly over the wire # to avoid proxy servers like MaxScale getting in the # way with their own values, see #4205 dbapi_con = connection.connection cursor = dbapi_con.cursor() cursor.execute("SELECT VERSION()") - val = cursor.fetchone()[0] + + val = cursor.fetchone()[0] # type: ignore[index] cursor.close() if isinstance(val, bytes): val = val.decode() return self._parse_server_version(val) - def _parse_server_version(self, val): - version = [] + def _parse_server_version(self, val: str) -> tuple[int, ...]: + version: list[int] = [] is_mariadb = False r = re.compile(r"[.\-+]") tokens = r.split(val) + + _mariadb_normalized_version_info = None for token in tokens: parsed_token = re.match( r"^(?:(\d+)(?:a|b|c)?|(MariaDB\w*))$", token @@ -2708,21 +2891,21 @@ def _parse_server_version(self, val): if not parsed_token: continue elif parsed_token.group(2): - self._mariadb_normalized_version_info = tuple(version[-3:]) + _mariadb_normalized_version_info = tuple(version[-3:]) is_mariadb = True else: digit = int(parsed_token.group(1)) version.append(digit) - server_version_info = tuple(version) + if _mariadb_normalized_version_info: + server_version_info = _mariadb_normalized_version_info + else: + server_version_info = tuple(version) self._set_mariadb( - server_version_info and is_mariadb, server_version_info + bool(server_version_info and is_mariadb), server_version_info ) - if not is_mariadb: - self._mariadb_normalized_version_info = server_version_info - if server_version_info < (5, 0, 2): raise NotImplementedError( "the MySQL/MariaDB dialect supports server " @@ -2733,62 +2916,54 @@ def _parse_server_version(self, val): self.server_version_info = server_version_info return server_version_info - def _set_mariadb(self, is_mariadb, server_version_info): - if is_mariadb is None: - return - - if not is_mariadb and self.is_mariadb: - raise exc.InvalidRequestError( - "MySQL version %s is not a MariaDB variant." - % (".".join(map(str, server_version_info)),) - ) - if is_mariadb: - - if not issubclass(self.preparer, MariaDBIdentifierPreparer): - self.preparer = MariaDBIdentifierPreparer - # this would have been set by the default dialect already, - # so set it again - self.identifier_preparer = self.preparer(self) - - # this will be updated on first connect in initialize() - # if using older mariadb version - self.delete_returning = True - self.insert_returning = True - - self.is_mariadb = is_mariadb - - def do_begin_twophase(self, connection, xid): + def do_begin_twophase(self, connection: Connection, xid: Any) -> None: connection.execute(sql.text("XA BEGIN :xid"), dict(xid=xid)) - def do_prepare_twophase(self, connection, xid): + def do_prepare_twophase(self, connection: Connection, xid: Any) -> None: connection.execute(sql.text("XA END :xid"), dict(xid=xid)) connection.execute(sql.text("XA PREPARE :xid"), dict(xid=xid)) def do_rollback_twophase( - self, connection, xid, is_prepared=True, recover=False - ): + self, + connection: Connection, + xid: Any, + is_prepared: bool = True, + recover: bool = False, + ) -> None: if not is_prepared: connection.execute(sql.text("XA END :xid"), dict(xid=xid)) connection.execute(sql.text("XA ROLLBACK :xid"), dict(xid=xid)) def do_commit_twophase( - self, connection, xid, is_prepared=True, recover=False - ): + self, + connection: Connection, + xid: Any, + is_prepared: bool = True, + recover: bool = False, + ) -> None: if not is_prepared: self.do_prepare_twophase(connection, xid) connection.execute(sql.text("XA COMMIT :xid"), dict(xid=xid)) - def do_recover_twophase(self, connection): + def do_recover_twophase(self, connection: Connection) -> list[Any]: resultset = connection.exec_driver_sql("XA RECOVER") - return [row["data"][0 : row["gtrid_length"]] for row in resultset] + return [ + row["data"][0 : row["gtrid_length"]] + for row in resultset.mappings() + ] - def is_disconnect(self, e, connection, cursor): + def is_disconnect( + self, + e: DBAPIModule.Error, + connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]], + cursor: Optional[DBAPICursor], + ) -> bool: if isinstance( e, ( - self.dbapi.OperationalError, - self.dbapi.ProgrammingError, - self.dbapi.InterfaceError, + self.dbapi.OperationalError, # type: ignore + self.dbapi.ProgrammingError, # type: ignore + self.dbapi.InterfaceError, # type: ignore ), ) and self._extract_error_code(e) in ( 1927, @@ -2801,7 +2976,7 @@ def is_disconnect(self, e, connection, cursor): ): return True elif isinstance( - e, (self.dbapi.InterfaceError, self.dbapi.InternalError) + e, (self.dbapi.InterfaceError, self.dbapi.InternalError) # type: ignore # noqa: E501 ): # if underlying connection is closed, # this is the error you get @@ -2809,13 +2984,17 @@ def is_disconnect(self, e, connection, cursor): else: return False - def _compat_fetchall(self, rp, charset=None): + def _compat_fetchall( + self, rp: CursorResult[Unpack[TupleAny]], charset: Optional[str] = None + ) -> Union[Sequence[Row[Unpack[TupleAny]]], Sequence[_DecodingRow]]: """Proxy result rows to smooth over MySQL-Python driver inconsistencies.""" return [_DecodingRow(row, charset) for row in rp.fetchall()] - def _compat_fetchone(self, rp, charset=None): + def _compat_fetchone( + self, rp: CursorResult[Unpack[TupleAny]], charset: Optional[str] = None + ) -> Union[Row[Unpack[TupleAny]], None, _DecodingRow]: """Proxy a result row to smooth over MySQL-Python driver inconsistencies.""" @@ -2825,7 +3004,9 @@ def _compat_fetchone(self, rp, charset=None): else: return None - def _compat_first(self, rp, charset=None): + def _compat_first( + self, rp: CursorResult[Unpack[TupleAny]], charset: Optional[str] = None + ) -> Optional[_DecodingRow]: """Proxy a result row to smooth over MySQL-Python driver inconsistencies.""" @@ -2835,14 +3016,22 @@ def _compat_first(self, rp, charset=None): else: return None - def _extract_error_code(self, exception): + def _extract_error_code( + self, exception: DBAPIModule.Error + ) -> Optional[int]: raise NotImplementedError() - def _get_default_schema_name(self, connection): - return connection.exec_driver_sql("SELECT DATABASE()").scalar() + def _get_default_schema_name(self, connection: Connection) -> str: + return connection.exec_driver_sql("SELECT DATABASE()").scalar() # type: ignore[return-value] # noqa: E501 @reflection.cache - def has_table(self, connection, table_name, schema=None, **kw): + def has_table( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> bool: self._ensure_has_table_connection(connection) if schema is None: @@ -2883,12 +3072,18 @@ def has_table(self, connection, table_name, schema=None, **kw): # # there's more "doesn't exist" kinds of messages but they are # less clear if mysql 8 would suddenly start using one of those - if self._extract_error_code(e.orig) in (1146, 1049, 1051): + if self._extract_error_code(e.orig) in (1146, 1049, 1051): # type: ignore # noqa: E501 return False raise @reflection.cache - def has_sequence(self, connection, sequence_name, schema=None, **kw): + def has_sequence( + self, + connection: Connection, + sequence_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> bool: if not self.supports_sequences: self._sequences_not_supported() if not schema: @@ -2908,14 +3103,16 @@ def has_sequence(self, connection, sequence_name, schema=None, **kw): ) return cursor.first() is not None - def _sequences_not_supported(self): + def _sequences_not_supported(self) -> NoReturn: raise NotImplementedError( "Sequences are supported only by the " "MariaDB series 10.3 or greater" ) @reflection.cache - def get_sequence_names(self, connection, schema=None, **kw): + def get_sequence_names( + self, connection: Connection, schema: Optional[str] = None, **kw: Any + ) -> list[str]: if not self.supports_sequences: self._sequences_not_supported() if not schema: @@ -2935,10 +3132,24 @@ def get_sequence_names(self, connection, schema=None, **kw): ) ] - def initialize(self, connection): + def _dispatch_for_vendor( + self, + mysql_callable: Callable[..., _T], + mariadb_callable: Callable[..., _T], + *arg: Any, + **kw: Any, + ) -> _T: + if not self.is_mariadb: + return mysql_callable(*arg, **kw) + else: + return mariadb_callable(*arg, **kw) + + def initialize(self, connection: Connection) -> None: # this is driver-based, does not need server version info # and is fairly critical for even basic SQL operations - self._connection_charset = self._detect_charset(connection) + self._connection_charset: Optional[str] = self._detect_charset( + connection + ) # call super().initialize() because we need to have # server_version_info set up. in 1.4 under python 2 only this does the @@ -2956,95 +3167,49 @@ def initialize(self, connection): self, server_ansiquotes=self._server_ansiquotes ) - self.supports_sequences = ( - self.is_mariadb and self.server_version_info >= (10, 3) + self._dispatch_for_vendor( + self._initialize_mysql, self._initialize_mariadb, connection ) - self.supports_for_update_of = ( - self._is_mysql and self.server_version_info >= (8,) - ) + def _initialize_mysql(self, connection: Connection) -> None: + assert not self.is_mariadb - self._needs_correct_for_88718_96365 = ( - not self.is_mariadb and self.server_version_info >= (8,) - ) + self.supports_for_update_of = self.server_version_info >= (8,) - self.delete_returning = ( - self.is_mariadb and self.server_version_info >= (10, 0, 5) - ) + self.use_mysql_for_share = self.server_version_info >= (8, 0, 1) - self.insert_returning = ( - self.is_mariadb and self.server_version_info >= (10, 5) - ) + self._needs_correct_for_88718_96365 = self.server_version_info >= ( + 8, + ) and (self.server_version_info < (8, 0, 14) or self._casing == 2) self._requires_alias_for_on_duplicate_key = ( - self._is_mysql and self.server_version_info >= (8, 0, 20) + self.server_version_info >= (8, 0, 20) ) - self._warn_for_known_db_issues() + # ref https://dev.mysql.com/doc/refman/8.0/en/data-type-defaults.html # noqa + self._support_default_function = self.server_version_info >= (8, 0, 13) - def _warn_for_known_db_issues(self): - if self.is_mariadb: - mdb_version = self._mariadb_normalized_version_info - if mdb_version > (10, 2) and mdb_version < (10, 2, 9): - util.warn( - "MariaDB %r before 10.2.9 has known issues regarding " - "CHECK constraints, which impact handling of NULL values " - "with SQLAlchemy's boolean datatype (MDEV-13596). An " - "additional issue prevents proper migrations of columns " - "with CHECK constraints (MDEV-11114). Please upgrade to " - "MariaDB 10.2.9 or greater, or use the MariaDB 10.1 " - "series, to avoid these issues." % (mdb_version,) - ) - - @property - def _support_float_cast(self): - if not self.server_version_info: - return False - elif self.is_mariadb: - # ref https://mariadb.com/kb/en/mariadb-1045-release-notes/ - return self.server_version_info >= (10, 4, 5) - else: - # ref https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-17.html#mysqld-8-0-17-feature # noqa - return self.server_version_info >= (8, 0, 17) + # ref https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-17.html#mysqld-8-0-17-feature # noqa + self._support_float_cast = self.server_version_info >= (8, 0, 17) @property - def _support_default_function(self): - if not self.server_version_info: - return False - elif self.is_mariadb: - # ref https://mariadb.com/kb/en/mariadb-1021-release-notes/ - return self.server_version_info >= (10, 2, 1) - else: - # ref https://dev.mysql.com/doc/refman/8.0/en/data-type-defaults.html # noqa - return self.server_version_info >= (8, 0, 13) - - @property - def _is_mariadb(self): - return self.is_mariadb - - @property - def _is_mysql(self): + def _is_mysql(self) -> bool: return not self.is_mariadb - @property - def _is_mariadb_102(self): - return self.is_mariadb and self._mariadb_normalized_version_info > ( - 10, - 2, - ) - @reflection.cache - def get_schema_names(self, connection, **kw): + def get_schema_names(self, connection: Connection, **kw: Any) -> list[str]: rp = connection.exec_driver_sql("SHOW schemas") return [r[0] for r in rp] @reflection.cache - def get_table_names(self, connection, schema=None, **kw): + def get_table_names( + self, connection: Connection, schema: Optional[str] = None, **kw: Any + ) -> list[str]: """Return a Unicode SHOW TABLES from a given schema.""" if schema is not None: - current_schema = schema + current_schema: str = schema else: - current_schema = self.default_schema_name + current_schema = self.default_schema_name # type: ignore charset = self._connection_charset @@ -3060,9 +3225,12 @@ def get_table_names(self, connection, schema=None, **kw): ] @reflection.cache - def get_view_names(self, connection, schema=None, **kw): + def get_view_names( + self, connection: Connection, schema: Optional[str] = None, **kw: Any + ) -> list[str]: if schema is None: schema = self.default_schema_name + assert schema is not None charset = self._connection_charset rp = connection.exec_driver_sql( "SHOW FULL TABLES FROM %s" @@ -3075,7 +3243,13 @@ def get_view_names(self, connection, schema=None, **kw): ] @reflection.cache - def get_table_options(self, connection, table_name, schema=None, **kw): + def get_table_options( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> dict[str, Any]: parsed_state = self._parsed_state_or_create( connection, table_name, schema, **kw ) @@ -3085,7 +3259,13 @@ def get_table_options(self, connection, table_name, schema=None, **kw): return ReflectionDefaults.table_options() @reflection.cache - def get_columns(self, connection, table_name, schema=None, **kw): + def get_columns( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> list[ReflectedColumn]: parsed_state = self._parsed_state_or_create( connection, table_name, schema, **kw ) @@ -3095,7 +3275,13 @@ def get_columns(self, connection, table_name, schema=None, **kw): return ReflectionDefaults.columns() @reflection.cache - def get_pk_constraint(self, connection, table_name, schema=None, **kw): + def get_pk_constraint( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> ReflectedPrimaryKeyConstraint: parsed_state = self._parsed_state_or_create( connection, table_name, schema, **kw ) @@ -3107,13 +3293,19 @@ def get_pk_constraint(self, connection, table_name, schema=None, **kw): return ReflectionDefaults.pk_constraint() @reflection.cache - def get_foreign_keys(self, connection, table_name, schema=None, **kw): + def get_foreign_keys( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> list[ReflectedForeignKeyConstraint]: parsed_state = self._parsed_state_or_create( connection, table_name, schema, **kw ) default_schema = None - fkeys = [] + fkeys: list[ReflectedForeignKeyConstraint] = [] for spec in parsed_state.fk_constraints: ref_name = spec["table"][-1] @@ -3133,7 +3325,7 @@ def get_foreign_keys(self, connection, table_name, schema=None, **kw): if spec.get(opt, False) not in ("NO ACTION", None): con_kw[opt] = spec[opt] - fkey_d = { + fkey_d: ReflectedForeignKeyConstraint = { "name": spec["name"], "constrained_columns": loc_names, "referred_schema": ref_schema, @@ -3143,12 +3335,16 @@ def get_foreign_keys(self, connection, table_name, schema=None, **kw): } fkeys.append(fkey_d) - if self._needs_correct_for_88718_96365: + if self._is_mysql and self._needs_correct_for_88718_96365: self._correct_for_mysql_bugs_88718_96365(fkeys, connection) return fkeys if fkeys else ReflectionDefaults.foreign_keys() - def _correct_for_mysql_bugs_88718_96365(self, fkeys, connection): + def _correct_for_mysql_bugs_88718_96365( + self, + fkeys: list[ReflectedForeignKeyConstraint], + connection: Connection, + ) -> None: # Foreign key is always in lower case (MySQL 8.0) # https://bugs.mysql.com/bug.php?id=88718 # issue #4344 for SQLAlchemy @@ -3164,22 +3360,24 @@ def _correct_for_mysql_bugs_88718_96365(self, fkeys, connection): if self._casing in (1, 2): - def lower(s): + def lower(s: str) -> str: return s.lower() else: # if on case sensitive, there can be two tables referenced # with the same name different casing, so we need to use # case-sensitive matching. - def lower(s): + def lower(s: str) -> str: return s - default_schema_name = connection.dialect.default_schema_name + default_schema_name: str = connection.dialect.default_schema_name # type: ignore # noqa: E501 # NOTE: using (table_schema, table_name, lower(column_name)) in (...) # is very slow since mysql does not seem able to properly use indexse. # Unpack the where condition instead. - schema_by_table_by_column = defaultdict(lambda: defaultdict(list)) + schema_by_table_by_column: defaultdict[ + str, defaultdict[str, list[str]] + ] = defaultdict(lambda: defaultdict(list)) for rec in fkeys: sch = lower(rec["referred_schema"] or default_schema_name) tbl = lower(rec["referred_table"]) @@ -3214,7 +3412,9 @@ def lower(s): _info_columns.c.column_name, ).where(condition) - correct_for_wrong_fk_case = connection.execute(select) + correct_for_wrong_fk_case: CursorResult[str, str, str] = ( + connection.execute(select) + ) # in casing=0, table name and schema name come back in their # exact case. @@ -3226,35 +3426,41 @@ def lower(s): # SHOW CREATE TABLE converts them to *lower case*, therefore # not matching. So for this case, case-insensitive lookup # is necessary - d = defaultdict(dict) + d: defaultdict[tuple[str, str], dict[str, str]] = defaultdict(dict) for schema, tname, cname in correct_for_wrong_fk_case: d[(lower(schema), lower(tname))]["SCHEMANAME"] = schema d[(lower(schema), lower(tname))]["TABLENAME"] = tname d[(lower(schema), lower(tname))][cname.lower()] = cname for fkey in fkeys: - rec = d[ + rec_b = d[ ( lower(fkey["referred_schema"] or default_schema_name), lower(fkey["referred_table"]), ) ] - fkey["referred_table"] = rec["TABLENAME"] + fkey["referred_table"] = rec_b["TABLENAME"] if fkey["referred_schema"] is not None: - fkey["referred_schema"] = rec["SCHEMANAME"] + fkey["referred_schema"] = rec_b["SCHEMANAME"] fkey["referred_columns"] = [ - rec[col.lower()] for col in fkey["referred_columns"] + rec_b[col.lower()] for col in fkey["referred_columns"] ] @reflection.cache - def get_check_constraints(self, connection, table_name, schema=None, **kw): + def get_check_constraints( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> list[ReflectedCheckConstraint]: parsed_state = self._parsed_state_or_create( connection, table_name, schema, **kw ) - cks = [ + cks: list[ReflectedCheckConstraint] = [ {"name": spec["name"], "sqltext": spec["sqltext"]} for spec in parsed_state.ck_constraints ] @@ -3262,7 +3468,13 @@ def get_check_constraints(self, connection, table_name, schema=None, **kw): return cks if cks else ReflectionDefaults.check_constraints() @reflection.cache - def get_table_comment(self, connection, table_name, schema=None, **kw): + def get_table_comment( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> ReflectedTableComment: parsed_state = self._parsed_state_or_create( connection, table_name, schema, **kw ) @@ -3273,12 +3485,18 @@ def get_table_comment(self, connection, table_name, schema=None, **kw): return ReflectionDefaults.table_comment() @reflection.cache - def get_indexes(self, connection, table_name, schema=None, **kw): + def get_indexes( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> list[ReflectedIndex]: parsed_state = self._parsed_state_or_create( connection, table_name, schema, **kw ) - indexes = [] + indexes: list[ReflectedIndex] = [] for spec in parsed_state.keys: dialect_options = {} @@ -3289,33 +3507,26 @@ def get_indexes(self, connection, table_name, schema=None, **kw): if flavor == "UNIQUE": unique = True elif flavor in ("FULLTEXT", "SPATIAL"): - dialect_options["%s_prefix" % self.name] = flavor - elif flavor is None: - pass - else: - self.logger.info( - "Converting unknown KEY type %s to a plain KEY", flavor + dialect_options[f"{self.name}_prefix"] = flavor + elif flavor is not None: + util.warn( + f"Converting unknown KEY type {flavor} to a plain KEY" ) - pass if spec["parser"]: - dialect_options["%s_with_parser" % (self.name)] = spec[ - "parser" - ] + dialect_options[f"{self.name}_with_parser"] = spec["parser"] - index_d = {} + index_d: ReflectedIndex = { + "name": spec["name"], + "column_names": [s[0] for s in spec["columns"]], + "unique": unique, + } - index_d["name"] = spec["name"] - index_d["column_names"] = [s[0] for s in spec["columns"]] mysql_length = { s[0]: s[1] for s in spec["columns"] if s[1] is not None } if mysql_length: - dialect_options["%s_length" % self.name] = mysql_length - - index_d["unique"] = unique - if flavor: - index_d["type"] = flavor + dialect_options[f"{self.name}_length"] = mysql_length if dialect_options: index_d["dialect_options"] = dialect_options @@ -3326,13 +3537,17 @@ def get_indexes(self, connection, table_name, schema=None, **kw): @reflection.cache def get_unique_constraints( - self, connection, table_name, schema=None, **kw - ): + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> list[ReflectedUniqueConstraint]: parsed_state = self._parsed_state_or_create( connection, table_name, schema, **kw ) - ucs = [ + ucs: list[ReflectedUniqueConstraint] = [ { "name": key["name"], "column_names": [col[0] for col in key["columns"]], @@ -3348,7 +3563,13 @@ def get_unique_constraints( return ReflectionDefaults.unique_constraints() @reflection.cache - def get_view_definition(self, connection, view_name, schema=None, **kw): + def get_view_definition( + self, + connection: Connection, + view_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> str: charset = self._connection_charset full_name = ".".join( self.identifier_preparer._quote_free_identifiers(schema, view_name) @@ -3362,8 +3583,12 @@ def get_view_definition(self, connection, view_name, schema=None, **kw): return sql def _parsed_state_or_create( - self, connection, table_name, schema=None, **kw - ): + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> _reflection.ReflectedState: return self._setup_parser( connection, table_name, @@ -3372,7 +3597,7 @@ def _parsed_state_or_create( ) @util.memoized_property - def _tabledef_parser(self): + def _tabledef_parser(self) -> _reflection.MySQLTableDefinitionParser: """return the MySQLTableDefinitionParser, generate if needed. The deferred creation ensures that the dialect has @@ -3383,7 +3608,13 @@ def _tabledef_parser(self): return _reflection.MySQLTableDefinitionParser(self, preparer) @reflection.cache - def _setup_parser(self, connection, table_name, schema=None, **kw): + def _setup_parser( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> _reflection.ReflectedState: charset = self._connection_charset parser = self._tabledef_parser full_name = ".".join( @@ -3399,10 +3630,14 @@ def _setup_parser(self, connection, table_name, schema=None, **kw): columns = self._describe_table( connection, None, charset, full_name=full_name ) - sql = parser._describe_to_create(table_name, columns) + sql = parser._describe_to_create( + table_name, columns # type: ignore[arg-type] + ) return parser.parse(sql, charset) - def _fetch_setting(self, connection, setting_name): + def _fetch_setting( + self, connection: Connection, setting_name: str + ) -> Optional[str]: charset = self._connection_charset if self.server_version_info and self.server_version_info < (5, 6): @@ -3417,12 +3652,12 @@ def _fetch_setting(self, connection, setting_name): if not row: return None else: - return row[fetch_col] + return cast(Optional[str], row[fetch_col]) - def _detect_charset(self, connection): + def _detect_charset(self, connection: Connection) -> str: raise NotImplementedError() - def _detect_casing(self, connection): + def _detect_casing(self, connection: Connection) -> int: """Sniff out identifier case sensitivity. Cached per-connection. This value can not change without a server @@ -3446,7 +3681,7 @@ def _detect_casing(self, connection): self._casing = cs return cs - def _detect_collations(self, connection): + def _detect_collations(self, connection: Connection) -> dict[str, str]: """Pull the active COLLATIONS list from the server. Cached per-connection. @@ -3459,7 +3694,7 @@ def _detect_collations(self, connection): collations[row[0]] = row[1] return collations - def _detect_sql_mode(self, connection): + def _detect_sql_mode(self, connection: Connection) -> None: setting = self._fetch_setting(connection, "sql_mode") if setting is None: @@ -3471,7 +3706,7 @@ def _detect_sql_mode(self, connection): else: self._sql_mode = setting or "" - def _detect_ansiquotes(self, connection): + def _detect_ansiquotes(self, connection: Connection) -> None: """Detect and adjust for the ANSI_QUOTES sql mode.""" mode = self._sql_mode @@ -3486,34 +3721,81 @@ def _detect_ansiquotes(self, connection): # as of MySQL 5.0.1 self._backslash_escapes = "NO_BACKSLASH_ESCAPES" not in mode + @overload def _show_create_table( - self, connection, table, charset=None, full_name=None - ): + self, + connection: Connection, + table: Optional[Table], + charset: Optional[str], + full_name: str, + ) -> str: ... + + @overload + def _show_create_table( + self, + connection: Connection, + table: Table, + charset: Optional[str] = None, + full_name: None = None, + ) -> str: ... + + def _show_create_table( + self, + connection: Connection, + table: Optional[Table], + charset: Optional[str] = None, + full_name: Optional[str] = None, + ) -> str: """Run SHOW CREATE TABLE for a ``Table``.""" if full_name is None: + assert table is not None full_name = self.identifier_preparer.format_table(table) st = "SHOW CREATE TABLE %s" % full_name - rp = None try: rp = connection.execution_options( skip_user_error_events=True ).exec_driver_sql(st) except exc.DBAPIError as e: - if self._extract_error_code(e.orig) == 1146: + if self._extract_error_code(e.orig) == 1146: # type: ignore[arg-type] # noqa: E501 raise exc.NoSuchTableError(full_name) from e else: raise row = self._compat_first(rp, charset=charset) if not row: raise exc.NoSuchTableError(full_name) - return row[1].strip() + return cast(str, row[1]).strip() - def _describe_table(self, connection, table, charset=None, full_name=None): + @overload + def _describe_table( + self, + connection: Connection, + table: Optional[Table], + charset: Optional[str], + full_name: str, + ) -> Union[Sequence[Row[Unpack[TupleAny]]], Sequence[_DecodingRow]]: ... + + @overload + def _describe_table( + self, + connection: Connection, + table: Table, + charset: Optional[str] = None, + full_name: None = None, + ) -> Union[Sequence[Row[Unpack[TupleAny]]], Sequence[_DecodingRow]]: ... + + def _describe_table( + self, + connection: Connection, + table: Optional[Table], + charset: Optional[str] = None, + full_name: Optional[str] = None, + ) -> Union[Sequence[Row[Unpack[TupleAny]]], Sequence[_DecodingRow]]: """Run DESCRIBE for a ``Table`` and return processed rows.""" if full_name is None: + assert table is not None full_name = self.identifier_preparer.format_table(table) st = "DESCRIBE %s" % full_name @@ -3524,14 +3806,14 @@ def _describe_table(self, connection, table, charset=None, full_name=None): skip_user_error_events=True ).exec_driver_sql(st) except exc.DBAPIError as e: - code = self._extract_error_code(e.orig) + code = self._extract_error_code(e.orig) # type: ignore[arg-type] # noqa: E501 if code == 1146: raise exc.NoSuchTableError(full_name) from e elif code == 1356: raise exc.UnreflectableTableError( - "Table or view named %s could not be " - "reflected: %s" % (full_name, e) + "Table or view named %s could not be reflected: %s" + % (full_name, e) ) from e else: @@ -3556,7 +3838,7 @@ class _DecodingRow: # sets.Set(['value']) (seriously) but thankfully that doesn't # seem to come up in DDL queries. - _encoding_compat = { + _encoding_compat: dict[str, str] = { "koi8r": "koi8_r", "koi8u": "koi8_u", "utf16": "utf-16-be", # MySQL's uft16 is always bigendian @@ -3566,24 +3848,23 @@ class _DecodingRow: "eucjpms": "ujis", } - def __init__(self, rowproxy, charset): + def __init__(self, rowproxy: Row[Unpack[_Ts]], charset: Optional[str]): self.rowproxy = rowproxy - self.charset = self._encoding_compat.get(charset, charset) + self.charset = ( + self._encoding_compat.get(charset, charset) + if charset is not None + else None + ) - def __getitem__(self, index): + def __getitem__(self, index: int) -> Any: item = self.rowproxy[index] - if isinstance(item, _array): - item = item.tostring() - if self.charset and isinstance(item, bytes): return item.decode(self.charset) else: return item - def __getattr__(self, attr): + def __getattr__(self, attr: str) -> Any: item = getattr(self.rowproxy, attr) - if isinstance(item, _array): - item = item.tostring() if self.charset and isinstance(item, bytes): return item.decode(self.charset) else: diff --git a/lib/sqlalchemy/dialects/mysql/cymysql.py b/lib/sqlalchemy/dialects/mysql/cymysql.py index 5c00ada9f94..325d3321e37 100644 --- a/lib/sqlalchemy/dialects/mysql/cymysql.py +++ b/lib/sqlalchemy/dialects/mysql/cymysql.py @@ -1,10 +1,9 @@ # dialects/mysql/cymysql.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors r""" @@ -21,18 +20,36 @@ dialects are mysqlclient and PyMySQL. """ # noqa +from __future__ import annotations + +from typing import Any +from typing import Iterable +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union -from .base import BIT from .base import MySQLDialect from .mysqldb import MySQLDialect_mysqldb +from .types import BIT from ... import util +if TYPE_CHECKING: + from ...engine.base import Connection + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import DBAPICursor + from ...engine.interfaces import DBAPIModule + from ...engine.interfaces import Dialect + from ...engine.interfaces import PoolProxiedConnection + from ...sql.type_api import _ResultProcessorType + class _cymysqlBIT(BIT): - def result_processor(self, dialect, coltype): + def result_processor( + self, dialect: Dialect, coltype: object + ) -> Optional[_ResultProcessorType[Any]]: """Convert MySQL's 64 bit, variable length binary string to a long.""" - def process(value): + def process(value: Optional[Iterable[int]]) -> Optional[int]: if value is not None: v = 0 for i in iter(value): @@ -55,17 +72,22 @@ class MySQLDialect_cymysql(MySQLDialect_mysqldb): colspecs = util.update_copy(MySQLDialect.colspecs, {BIT: _cymysqlBIT}) @classmethod - def import_dbapi(cls): + def import_dbapi(cls) -> DBAPIModule: return __import__("cymysql") - def _detect_charset(self, connection): - return connection.connection.charset + def _detect_charset(self, connection: Connection) -> str: + return connection.connection.charset # type: ignore[no-any-return] - def _extract_error_code(self, exception): - return exception.errno + def _extract_error_code(self, exception: DBAPIModule.Error) -> int: + return exception.errno # type: ignore[no-any-return] - def is_disconnect(self, e, connection, cursor): - if isinstance(e, self.dbapi.OperationalError): + def is_disconnect( + self, + e: DBAPIModule.Error, + connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]], + cursor: Optional[DBAPICursor], + ) -> bool: + if isinstance(e, self.loaded_dbapi.OperationalError): return self._extract_error_code(e) in ( 2006, 2013, @@ -73,7 +95,7 @@ def is_disconnect(self, e, connection, cursor): 2045, 2055, ) - elif isinstance(e, self.dbapi.InterfaceError): + elif isinstance(e, self.loaded_dbapi.InterfaceError): # if underlying connection is closed, # this is the error you get return True diff --git a/lib/sqlalchemy/dialects/mysql/dml.py b/lib/sqlalchemy/dialects/mysql/dml.py index 61476af0229..bc2d288fb56 100644 --- a/lib/sqlalchemy/dialects/mysql/dml.py +++ b/lib/sqlalchemy/dialects/mysql/dml.py @@ -1,5 +1,5 @@ # dialects/mysql/dml.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -110,8 +110,6 @@ class Insert(StandardInsert): The :class:`~.mysql.Insert` object is created using the :func:`sqlalchemy.dialects.mysql.insert` function. - .. versionadded:: 1.2 - """ stringify_dialect = "mysql" @@ -198,13 +196,6 @@ def on_duplicate_key_update(self, *args: _UpdateArg, **kw: Any) -> Self: ] ) - .. versionchanged:: 1.3 parameters can be specified as a dictionary - or list of 2-tuples; the latter form provides for parameter - ordering. - - - .. versionadded:: 1.2 - .. seealso:: :ref:`mysql_insert_on_duplicate_key_update` diff --git a/lib/sqlalchemy/dialects/mysql/enumerated.py b/lib/sqlalchemy/dialects/mysql/enumerated.py index 6745cae55e7..0caffc1edfd 100644 --- a/lib/sqlalchemy/dialects/mysql/enumerated.py +++ b/lib/sqlalchemy/dialects/mysql/enumerated.py @@ -1,29 +1,44 @@ # dialects/mysql/enumerated.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors +from __future__ import annotations +import enum import re +from typing import Any +from typing import Optional +from typing import Type +from typing import TYPE_CHECKING +from typing import Union from .types import _StringType from ... import exc from ... import sql from ... import util from ...sql import sqltypes +from ...sql import type_api +if TYPE_CHECKING: + from ...engine.interfaces import Dialect + from ...sql.elements import ColumnElement + from ...sql.type_api import _BindProcessorType + from ...sql.type_api import _ResultProcessorType + from ...sql.type_api import TypeEngine + from ...sql.type_api import TypeEngineMixin -class ENUM(sqltypes.NativeForEmulated, sqltypes.Enum, _StringType): + +class ENUM(type_api.NativeForEmulated, sqltypes.Enum, _StringType): """MySQL ENUM type.""" __visit_name__ = "ENUM" native_enum = True - def __init__(self, *enums, **kw): + def __init__(self, *enums: Union[str, Type[enum.Enum]], **kw: Any) -> None: """Construct an ENUM. E.g.:: @@ -35,9 +50,6 @@ def __init__(self, *enums, **kw): quotes when generating the schema. This object may also be a PEP-435-compliant enumerated type. - .. versionadded: 1.1 added support for PEP-435-compliant enumerated - types. - :param strict: This flag has no effect. .. versionchanged:: The MySQL ENUM type as well as the base Enum @@ -62,21 +74,27 @@ def __init__(self, *enums, **kw): """ kw.pop("strict", None) - self._enum_init(enums, kw) + self._enum_init(enums, kw) # type: ignore[arg-type] _StringType.__init__(self, length=self.length, **kw) @classmethod - def adapt_emulated_to_native(cls, impl, **kw): + def adapt_emulated_to_native( + cls, + impl: Union[TypeEngine[Any], TypeEngineMixin], + **kw: Any, + ) -> ENUM: """Produce a MySQL native :class:`.mysql.ENUM` from plain :class:`.Enum`. """ + if TYPE_CHECKING: + assert isinstance(impl, ENUM) kw.setdefault("validate_strings", impl.validate_strings) kw.setdefault("values_callable", impl.values_callable) kw.setdefault("omit_aliases", impl._omit_aliases) return cls(**kw) - def _object_value_for_elem(self, elem): + def _object_value_for_elem(self, elem: str) -> Union[str, enum.Enum]: # mysql sends back a blank string for any value that # was persisted that was not in the enums; that is, it does no # validation on the incoming data, it "truncates" it to be @@ -86,18 +104,22 @@ def _object_value_for_elem(self, elem): else: return super()._object_value_for_elem(elem) - def __repr__(self): - return util.generic_repr( + def repr_struct(self) -> util.GenericRepr: + return util.GenericRepr( self, to_inspect=[ENUM, _StringType, sqltypes.Enum] ) +# TODO: SET is a string as far as configuration but does not act like +# a string at the python level. We either need to make a py-type agnostic +# version of String as a base to be used for this, make this some kind of +# TypeDecorator, or just vendor it out as its own type. class SET(_StringType): """MySQL SET type.""" __visit_name__ = "SET" - def __init__(self, *values, **kw): + def __init__(self, *values: str, **kw: Any): """Construct a SET. E.g.:: @@ -150,17 +172,19 @@ def __init__(self, *values, **kw): "setting retrieve_as_bitwise=True" ) if self.retrieve_as_bitwise: - self._bitmap = { + self._inversed_bitmap: dict[str, int] = { value: 2**idx for idx, value in enumerate(self.values) } - self._bitmap.update( - (2**idx, value) for idx, value in enumerate(self.values) - ) + self._bitmap: dict[int, str] = { + 2**idx: value for idx, value in enumerate(self.values) + } length = max([len(v) for v in values] + [0]) kw.setdefault("length", length) super().__init__(**kw) - def column_expression(self, colexpr): + def column_expression( + self, colexpr: ColumnElement[Any] + ) -> ColumnElement[Any]: if self.retrieve_as_bitwise: return sql.type_coerce( sql.type_coerce(colexpr, sqltypes.Integer) + 0, self @@ -168,10 +192,12 @@ def column_expression(self, colexpr): else: return colexpr - def result_processor(self, dialect, coltype): + def result_processor( + self, dialect: Dialect, coltype: Any + ) -> Optional[_ResultProcessorType[Any]]: if self.retrieve_as_bitwise: - def process(value): + def process(value: Union[str, int, None]) -> Optional[set[str]]: if value is not None: value = int(value) @@ -182,11 +208,14 @@ def process(value): else: super_convert = super().result_processor(dialect, coltype) - def process(value): + def process(value: Union[str, set[str], None]) -> Optional[set[str]]: # type: ignore[misc] # noqa: E501 if isinstance(value, str): # MySQLdb returns a string, let's parse if super_convert: value = super_convert(value) + assert value is not None + if TYPE_CHECKING: + assert isinstance(value, str) return set(re.findall(r"[^,]+", value)) else: # mysql-connector-python does a naive @@ -197,44 +226,49 @@ def process(value): return process - def bind_processor(self, dialect): + def bind_processor( + self, dialect: Dialect + ) -> _BindProcessorType[Union[str, int]]: super_convert = super().bind_processor(dialect) if self.retrieve_as_bitwise: - def process(value): + def process( + value: Union[str, int, set[str], None], + ) -> Union[str, int, None]: if value is None: return None elif isinstance(value, (int, str)): if super_convert: - return super_convert(value) + return super_convert(value) # type: ignore[arg-type, no-any-return] # noqa: E501 else: return value else: int_value = 0 for v in value: - int_value |= self._bitmap[v] + int_value |= self._inversed_bitmap[v] return int_value else: - def process(value): + def process( + value: Union[str, int, set[str], None], + ) -> Union[str, int, None]: # accept strings and int (actually bitflag) values directly if value is not None and not isinstance(value, (int, str)): value = ",".join(value) - if super_convert: - return super_convert(value) + return super_convert(value) # type: ignore else: return value return process - def adapt(self, impltype, **kw): + def adapt(self, cls: type, **kw: Any) -> Any: kw["retrieve_as_bitwise"] = self.retrieve_as_bitwise - return util.constructor_copy(self, impltype, *self.values, **kw) + return util.constructor_copy(self, cls, *self.values, **kw) - def __repr__(self): - return util.generic_repr( + def repr_struct(self) -> util.GenericRepr: + return util.GenericRepr( self, to_inspect=[SET, _StringType], additional_kw=[ diff --git a/lib/sqlalchemy/dialects/mysql/expression.py b/lib/sqlalchemy/dialects/mysql/expression.py index b60a0888517..914207e9ee8 100644 --- a/lib/sqlalchemy/dialects/mysql/expression.py +++ b/lib/sqlalchemy/dialects/mysql/expression.py @@ -1,11 +1,13 @@ # dialects/mysql/expression.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors +from __future__ import annotations + +from typing import Any from ... import exc from ... import util @@ -18,7 +20,7 @@ from ...util.typing import Self -class match(Generative, elements.BinaryExpression): +class match(Generative, elements.BinaryExpression[Any]): """Produce a ``MATCH (X, Y) AGAINST ('TEXT')`` clause. E.g.:: @@ -73,8 +75,9 @@ class match(Generative, elements.BinaryExpression): __visit_name__ = "mysql_match" inherit_cache = True + modifiers: util.immutabledict[str, Any] - def __init__(self, *cols, **kw): + def __init__(self, *cols: elements.ColumnElement[Any], **kw: Any): if not cols: raise exc.ArgumentError("columns are required") diff --git a/lib/sqlalchemy/dialects/mysql/json.py b/lib/sqlalchemy/dialects/mysql/json.py index 8912af36631..5c564d73b9e 100644 --- a/lib/sqlalchemy/dialects/mysql/json.py +++ b/lib/sqlalchemy/dialects/mysql/json.py @@ -1,15 +1,24 @@ # dialects/mysql/json.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors +from __future__ import annotations + +from typing import Any +from typing import TYPE_CHECKING from ... import types as sqltypes +from ...sql.sqltypes import _T_JSON + +if TYPE_CHECKING: + from ...engine.interfaces import Dialect + from ...sql.type_api import _BindProcessorType + from ...sql.type_api import _LiteralProcessorType -class JSON(sqltypes.JSON): +class JSON(sqltypes.JSON[_T_JSON]): """MySQL JSON type. MySQL supports JSON as of version 5.7. @@ -34,13 +43,13 @@ class JSON(sqltypes.JSON): class _FormatTypeMixin: - def _format_value(self, value): + def _format_value(self, value: Any) -> str: raise NotImplementedError() - def bind_processor(self, dialect): - super_proc = self.string_bind_processor(dialect) + def bind_processor(self, dialect: Dialect) -> _BindProcessorType[Any]: + super_proc = self.string_bind_processor(dialect) # type: ignore[attr-defined] # noqa: E501 - def process(value): + def process(value: Any) -> Any: value = self._format_value(value) if super_proc: value = super_proc(value) @@ -48,29 +57,31 @@ def process(value): return process - def literal_processor(self, dialect): - super_proc = self.string_literal_processor(dialect) + def literal_processor( + self, dialect: Dialect + ) -> _LiteralProcessorType[Any]: + super_proc = self.string_literal_processor(dialect) # type: ignore[attr-defined] # noqa: E501 - def process(value): + def process(value: Any) -> str: value = self._format_value(value) if super_proc: value = super_proc(value) - return value + return value # type: ignore[no-any-return] return process class JSONIndexType(_FormatTypeMixin, sqltypes.JSON.JSONIndexType): - def _format_value(self, value): + def _format_value(self, value: Any) -> str: if isinstance(value, int): - value = "$[%s]" % value + formatted_value = "$[%s]" % value else: - value = '$."%s"' % value - return value + formatted_value = '$."%s"' % value + return formatted_value class JSONPathType(_FormatTypeMixin, sqltypes.JSON.JSONPathType): - def _format_value(self, value): + def _format_value(self, value: Any) -> str: return "$%s" % ( "".join( [ diff --git a/lib/sqlalchemy/dialects/mysql/mariadb.py b/lib/sqlalchemy/dialects/mysql/mariadb.py index ff5214798f2..cdb1e2ffeed 100644 --- a/lib/sqlalchemy/dialects/mysql/mariadb.py +++ b/lib/sqlalchemy/dialects/mysql/mariadb.py @@ -1,17 +1,16 @@ # dialects/mysql/mariadb.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors -from .base import MariaDBIdentifierPreparer + +from __future__ import annotations + +from typing import Any + from .base import MySQLDialect -from .base import MySQLTypeCompiler -from ... import util from ...sql import sqltypes -from ...sql.sqltypes import UUID -from ...sql.sqltypes import Uuid class INET4(sqltypes.TypeEngine[str]): @@ -32,40 +31,6 @@ class INET6(sqltypes.TypeEngine[str]): __visit_name__ = "INET6" -class _MariaDBUUID(UUID): - def __init__(self, as_uuid: bool = True, native_uuid: bool = True): - self.as_uuid = as_uuid - - # the _MariaDBUUID internal type is only invoked for a Uuid() with - # native_uuid=True. for non-native uuid type, the plain Uuid - # returns itself due to the workings of the Emulated superclass. - assert native_uuid - - # for internal type, force string conversion for result_processor() as - # current drivers are returning a string, not a Python UUID object - self.native_uuid = False - - @property - def native(self): - # override to return True, this is a native type, just turning - # off native_uuid for internal data handling - return True - - def bind_processor(self, dialect): - if not dialect.supports_native_uuid or not dialect._allows_uuid_binds: - return super().bind_processor(dialect) - else: - return None - - -class MariaDBTypeCompiler(MySQLTypeCompiler): - def visit_INET4(self, type_, **kwargs) -> str: - return "INET4" - - def visit_INET6(self, type_, **kwargs) -> str: - return "INET6" - - class MariaDBDialect(MySQLDialect): is_mariadb = True supports_statement_cache = True @@ -74,21 +39,13 @@ class MariaDBDialect(MySQLDialect): _allows_uuid_binds = True name = "mariadb" - preparer = MariaDBIdentifierPreparer - type_compiler_cls = MariaDBTypeCompiler - colspecs = util.update_copy(MySQLDialect.colspecs, {Uuid: _MariaDBUUID}) - - def initialize(self, connection): - super().initialize(connection) - - self.supports_native_uuid = ( - self.server_version_info is not None - and self.server_version_info >= (10, 7) - ) + def __init__(self, **kw: Any) -> None: + kw["is_mariadb"] = True + super().__init__(**kw) -def loader(driver): +def loader(driver: str) -> type[MariaDBDialect]: dialect_mod = __import__( "sqlalchemy.dialects.mysql.%s" % driver ).dialects.mysql @@ -96,7 +53,7 @@ def loader(driver): driver_mod = getattr(dialect_mod, driver) if hasattr(driver_mod, "mariadb_dialect"): driver_cls = driver_mod.mariadb_dialect - return driver_cls + return driver_cls # type: ignore[no-any-return] else: driver_cls = driver_mod.dialect diff --git a/lib/sqlalchemy/dialects/mysql/mariadbconnector.py b/lib/sqlalchemy/dialects/mysql/mariadbconnector.py index fbc60037971..fc26974b822 100644 --- a/lib/sqlalchemy/dialects/mysql/mariadbconnector.py +++ b/lib/sqlalchemy/dialects/mysql/mariadbconnector.py @@ -1,11 +1,9 @@ # dialects/mysql/mariadbconnector.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors - """ @@ -29,7 +27,14 @@ .. mariadb: https://github.com/mariadb-corporation/mariadb-connector-python """ # noqa +from __future__ import annotations + import re +from typing import Any +from typing import Optional +from typing import Sequence +from typing import TYPE_CHECKING +from typing import Union from uuid import UUID as _python_UUID from .base import MySQLCompiler @@ -40,6 +45,19 @@ from ... import util from ...sql import sqltypes +if TYPE_CHECKING: + from ...engine.base import Connection + from ...engine.interfaces import ConnectArgsType + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import DBAPICursor + from ...engine.interfaces import DBAPIModule + from ...engine.interfaces import Dialect + from ...engine.interfaces import IsolationLevel + from ...engine.interfaces import PoolProxiedConnection + from ...engine.url import URL + from ...sql.compiler import SQLCompiler + from ...sql.type_api import _ResultProcessorType + mariadb_cpy_minimum_version = (1, 0, 1) @@ -48,10 +66,12 @@ class _MariaDBUUID(sqltypes.UUID[sqltypes._UUID_RETURN]): # work around JIRA issue # https://jira.mariadb.org/browse/CONPY-270. When that issue is fixed, # this type can be removed. - def result_processor(self, dialect, coltype): + def result_processor( + self, dialect: Dialect, coltype: object + ) -> Optional[_ResultProcessorType[Any]]: if self.as_uuid: - def process(value): + def process(value: Any) -> Any: if value is not None: if hasattr(value, "decode"): value = value.decode("ascii") @@ -61,7 +81,7 @@ def process(value): return process else: - def process(value): + def process(value: Any) -> Any: if value is not None: if hasattr(value, "decode"): value = value.decode("ascii") @@ -72,23 +92,27 @@ def process(value): class MySQLExecutionContext_mariadbconnector(MySQLExecutionContext): - _lastrowid = None + _lastrowid: Optional[int] = None - def create_server_side_cursor(self): + def create_server_side_cursor(self) -> DBAPICursor: return self._dbapi_connection.cursor(buffered=False) - def create_default_cursor(self): + def create_default_cursor(self) -> DBAPICursor: return self._dbapi_connection.cursor(buffered=True) - def post_exec(self): + def post_exec(self) -> None: super().post_exec() self._rowcount = self.cursor.rowcount + if TYPE_CHECKING: + assert isinstance(self.compiled, SQLCompiler) if self.isinsert and self.compiled.postfetch_lastrowid: self._lastrowid = self.cursor.lastrowid - def get_lastrowid(self): + def get_lastrowid(self) -> int: + if TYPE_CHECKING: + assert self._lastrowid is not None return self._lastrowid @@ -127,7 +151,7 @@ class MySQLDialect_mariadbconnector(MySQLDialect): ) @util.memoized_property - def _dbapi_version(self): + def _dbapi_version(self) -> tuple[int, ...]: if self.dbapi and hasattr(self.dbapi, "__version__"): return tuple( [ @@ -140,7 +164,7 @@ def _dbapi_version(self): else: return (99, 99, 99) - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.paramstyle = "qmark" if self.dbapi is not None: @@ -152,19 +176,24 @@ def __init__(self, **kwargs): ) @classmethod - def import_dbapi(cls): + def import_dbapi(cls) -> DBAPIModule: return __import__("mariadb") - def is_disconnect(self, e, connection, cursor): + def is_disconnect( + self, + e: DBAPIModule.Error, + connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]], + cursor: Optional[DBAPICursor], + ) -> bool: if super().is_disconnect(e, connection, cursor): return True - elif isinstance(e, self.dbapi.Error): + elif isinstance(e, self.loaded_dbapi.Error): str_e = str(e).lower() return "not connected" in str_e or "isn't valid" in str_e else: return False - def create_connect_args(self, url): + def create_connect_args(self, url: URL) -> ConnectArgsType: opts = url.translate_connect_args() opts.update(url.query) @@ -201,19 +230,21 @@ def create_connect_args(self, url): except (AttributeError, ImportError): self.supports_sane_rowcount = False opts["client_flag"] = client_flag - return [[], opts] + return [], opts - def _extract_error_code(self, exception): + def _extract_error_code(self, exception: DBAPIModule.Error) -> int: try: - rc = exception.errno + rc: int = exception.errno except: rc = -1 return rc - def _detect_charset(self, connection): + def _detect_charset(self, connection: Connection) -> str: return "utf8mb4" - def get_isolation_level_values(self, dbapi_connection): + def get_isolation_level_values( + self, dbapi_conn: DBAPIConnection + ) -> Sequence[IsolationLevel]: return ( "SERIALIZABLE", "READ UNCOMMITTED", @@ -222,21 +253,26 @@ def get_isolation_level_values(self, dbapi_connection): "AUTOCOMMIT", ) - def set_isolation_level(self, connection, level): + def detect_autocommit_setting(self, dbapi_conn: DBAPIConnection) -> bool: + return bool(dbapi_conn.autocommit) + + def set_isolation_level( + self, dbapi_connection: DBAPIConnection, level: IsolationLevel + ) -> None: if level == "AUTOCOMMIT": - connection.autocommit = True + dbapi_connection.autocommit = True else: - connection.autocommit = False - super().set_isolation_level(connection, level) + dbapi_connection.autocommit = False + super().set_isolation_level(dbapi_connection, level) - def do_begin_twophase(self, connection, xid): + def do_begin_twophase(self, connection: Connection, xid: Any) -> None: connection.execute( sql.text("XA BEGIN :xid").bindparams( sql.bindparam("xid", xid, literal_execute=True) ) ) - def do_prepare_twophase(self, connection, xid): + def do_prepare_twophase(self, connection: Connection, xid: Any) -> None: connection.execute( sql.text("XA END :xid").bindparams( sql.bindparam("xid", xid, literal_execute=True) @@ -249,8 +285,12 @@ def do_prepare_twophase(self, connection, xid): ) def do_rollback_twophase( - self, connection, xid, is_prepared=True, recover=False - ): + self, + connection: Connection, + xid: Any, + is_prepared: bool = True, + recover: bool = False, + ) -> None: if not is_prepared: connection.execute( sql.text("XA END :xid").bindparams( @@ -264,8 +304,12 @@ def do_rollback_twophase( ) def do_commit_twophase( - self, connection, xid, is_prepared=True, recover=False - ): + self, + connection: Connection, + xid: Any, + is_prepared: bool = True, + recover: bool = False, + ) -> None: if not is_prepared: self.do_prepare_twophase(connection, xid) connection.execute( diff --git a/lib/sqlalchemy/dialects/mysql/mysqlconnector.py b/lib/sqlalchemy/dialects/mysql/mysqlconnector.py index 71ac58601c1..6ace9bb3030 100644 --- a/lib/sqlalchemy/dialects/mysql/mysqlconnector.py +++ b/lib/sqlalchemy/dialects/mysql/mysqlconnector.py @@ -1,10 +1,9 @@ # dialects/mysql/mysqlconnector.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors r""" @@ -22,11 +21,19 @@ with features such as server side cursors which remain disabled until upstream issues are repaired. +.. warning:: The MySQL Connector/Python driver published by Oracle is subject + to frequent, major regressions of essential functionality such as being able + to correctly persist simple binary strings which indicate it is not well + tested. The SQLAlchemy project is not able to maintain this dialect fully as + regressions in the driver prevent it from being included in continuous + integration. + .. versionchanged:: 2.0.39 The MySQL Connector/Python dialect has been updated to support the latest version of this DBAPI. Previously, MySQL Connector/Python - was not fully supported. + was not fully supported. However, support remains limited due to ongoing + regressions introduced in this driver. Connecting to MariaDB with MySQL Connector/Python -------------------------------------------------- @@ -38,29 +45,53 @@ """ # noqa +from __future__ import annotations import re +from typing import Any +from typing import cast +from typing import Optional +from typing import Sequence +from typing import TYPE_CHECKING +from typing import Union -from .base import BIT -from .base import MariaDBIdentifierPreparer from .base import MySQLCompiler from .base import MySQLDialect from .base import MySQLExecutionContext from .base import MySQLIdentifierPreparer from .mariadb import MariaDBDialect +from .types import BIT from ... import util +if TYPE_CHECKING: + + from ...engine.base import Connection + from ...engine.cursor import CursorResult + from ...engine.interfaces import ConnectArgsType + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import DBAPICursor + from ...engine.interfaces import DBAPIModule + from ...engine.interfaces import IsolationLevel + from ...engine.interfaces import PoolProxiedConnection + from ...engine.row import Row + from ...engine.url import URL + from ...sql.elements import BinaryExpression + from ...util.typing import TupleAny + from ...util.typing import Unpack + class MySQLExecutionContext_mysqlconnector(MySQLExecutionContext): - def create_server_side_cursor(self): + def create_server_side_cursor(self) -> DBAPICursor: return self._dbapi_connection.cursor(buffered=False) - def create_default_cursor(self): + def create_default_cursor(self) -> DBAPICursor: return self._dbapi_connection.cursor(buffered=True) class MySQLCompiler_mysqlconnector(MySQLCompiler): - def visit_mod_binary(self, binary, operator, **kw): + def visit_mod_binary( + self, binary: BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: return ( self.process(binary.left, **kw) + " % " @@ -70,15 +101,18 @@ def visit_mod_binary(self, binary, operator, **kw): class IdentifierPreparerCommon_mysqlconnector: @property - def _double_percents(self): + def _double_percents(self) -> bool: return False @_double_percents.setter - def _double_percents(self, value): + def _double_percents(self, value: Any) -> None: pass - def _escape_identifier(self, value): - value = value.replace(self.escape_quote, self.escape_to_quote) + def _escape_identifier(self, value: str) -> str: + value = value.replace( + self.escape_quote, # type:ignore[attr-defined] + self.escape_to_quote, # type:ignore[attr-defined] + ) return value @@ -88,14 +122,8 @@ class MySQLIdentifierPreparer_mysqlconnector( pass -class MariaDBIdentifierPreparer_mysqlconnector( - IdentifierPreparerCommon_mysqlconnector, MariaDBIdentifierPreparer -): - pass - - class _myconnpyBIT(BIT): - def result_processor(self, dialect, coltype): + def result_processor(self, dialect: Any, coltype: Any) -> None: """MySQL-connector already converts mysql bits, so.""" return None @@ -120,21 +148,21 @@ class MySQLDialect_mysqlconnector(MySQLDialect): execution_ctx_cls = MySQLExecutionContext_mysqlconnector - preparer = MySQLIdentifierPreparer_mysqlconnector + preparer: type[MySQLIdentifierPreparer] = ( + MySQLIdentifierPreparer_mysqlconnector + ) colspecs = util.update_copy(MySQLDialect.colspecs, {BIT: _myconnpyBIT}) @classmethod - def import_dbapi(cls): - from mysql import connector + def import_dbapi(cls) -> DBAPIModule: + return cast("DBAPIModule", __import__("mysql.connector").connector) - return connector - - def do_ping(self, dbapi_connection): + def do_ping(self, dbapi_connection: DBAPIConnection) -> bool: dbapi_connection.ping(False) return True - def create_connect_args(self, url): + def create_connect_args(self, url: URL) -> ConnectArgsType: opts = url.translate_connect_args(username="user") opts.update(url.query) @@ -169,7 +197,9 @@ def create_connect_args(self, url): # supports_sane_rowcount. if self.dbapi is not None: try: - from mysql.connector.constants import ClientFlag + from mysql.connector import constants # type: ignore + + ClientFlag = constants.ClientFlag client_flags = opts.get( "client_flags", ClientFlag.get_default() @@ -179,27 +209,33 @@ def create_connect_args(self, url): except Exception: pass - return [[], opts] + return [], opts @util.memoized_property - def _mysqlconnector_version_info(self): + def _mysqlconnector_version_info(self) -> Optional[tuple[int, ...]]: if self.dbapi and hasattr(self.dbapi, "__version__"): m = re.match(r"(\d+)\.(\d+)(?:\.(\d+))?", self.dbapi.__version__) if m: return tuple(int(x) for x in m.group(1, 2, 3) if x is not None) + return None - def _detect_charset(self, connection): - return connection.connection.charset + def _detect_charset(self, connection: Connection) -> str: + return connection.connection.charset # type: ignore - def _extract_error_code(self, exception): - return exception.errno + def _extract_error_code(self, exception: BaseException) -> int: + return exception.errno # type: ignore - def is_disconnect(self, e, connection, cursor): + def is_disconnect( + self, + e: Exception, + connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]], + cursor: Optional[DBAPICursor], + ) -> bool: errnos = (2006, 2013, 2014, 2045, 2055, 2048) exceptions = ( - self.dbapi.OperationalError, - self.dbapi.InterfaceError, - self.dbapi.ProgrammingError, + self.loaded_dbapi.OperationalError, # + self.loaded_dbapi.InterfaceError, + self.loaded_dbapi.ProgrammingError, ) if isinstance(e, exceptions): return ( @@ -210,13 +246,23 @@ def is_disconnect(self, e, connection, cursor): else: return False - def _compat_fetchall(self, rp, charset=None): + def _compat_fetchall( + self, + rp: CursorResult[Unpack[TupleAny]], + charset: Optional[str] = None, + ) -> Sequence[Row[Unpack[TupleAny]]]: return rp.fetchall() - def _compat_fetchone(self, rp, charset=None): + def _compat_fetchone( + self, + rp: CursorResult[Unpack[TupleAny]], + charset: Optional[str] = None, + ) -> Optional[Row[Unpack[TupleAny]]]: return rp.fetchone() - def get_isolation_level_values(self, dbapi_connection): + def get_isolation_level_values( + self, dbapi_conn: DBAPIConnection + ) -> Sequence[IsolationLevel]: return ( "SERIALIZABLE", "READ UNCOMMITTED", @@ -225,12 +271,17 @@ def get_isolation_level_values(self, dbapi_connection): "AUTOCOMMIT", ) - def set_isolation_level(self, connection, level): + def detect_autocommit_setting(self, dbapi_conn: DBAPIConnection) -> bool: + return bool(dbapi_conn.autocommit) + + def set_isolation_level( + self, dbapi_connection: DBAPIConnection, level: IsolationLevel + ) -> None: if level == "AUTOCOMMIT": - connection.autocommit = True + dbapi_connection.autocommit = True else: - connection.autocommit = False - super().set_isolation_level(connection, level) + dbapi_connection.autocommit = False + super().set_isolation_level(dbapi_connection, level) class MariaDBDialect_mysqlconnector( @@ -238,7 +289,7 @@ class MariaDBDialect_mysqlconnector( ): supports_statement_cache = True _allows_uuid_binds = False - preparer = MariaDBIdentifierPreparer_mysqlconnector + preparer = MySQLIdentifierPreparer_mysqlconnector dialect = MySQLDialect_mysqlconnector diff --git a/lib/sqlalchemy/dialects/mysql/mysqldb.py b/lib/sqlalchemy/dialects/mysql/mysqldb.py index 3cf56c1fd09..bdcc5c010d5 100644 --- a/lib/sqlalchemy/dialects/mysql/mysqldb.py +++ b/lib/sqlalchemy/dialects/mysql/mysqldb.py @@ -1,11 +1,9 @@ # dialects/mysql/mysqldb.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors - """ @@ -86,17 +84,34 @@ The mysqldb dialect supports server-side cursors. See :ref:`mysql_ss_cursors`. """ +from __future__ import annotations import re +from typing import Any +from typing import Callable +from typing import cast +from typing import Literal +from typing import Optional +from typing import TYPE_CHECKING from .base import MySQLCompiler from .base import MySQLDialect from .base import MySQLExecutionContext from .base import MySQLIdentifierPreparer -from .base import TEXT -from ... import sql from ... import util +if TYPE_CHECKING: + + from ...engine.base import Connection + from ...engine.interfaces import _DBAPIMultiExecuteParams + from ...engine.interfaces import ConnectArgsType + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import DBAPICursor + from ...engine.interfaces import DBAPIModule + from ...engine.interfaces import ExecutionContext + from ...engine.interfaces import IsolationLevel + from ...engine.url import URL + class MySQLExecutionContext_mysqldb(MySQLExecutionContext): pass @@ -119,8 +134,9 @@ class MySQLDialect_mysqldb(MySQLDialect): execution_ctx_cls = MySQLExecutionContext_mysqldb statement_compiler = MySQLCompiler_mysqldb preparer = MySQLIdentifierPreparer + server_version_info: tuple[int, ...] - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any): super().__init__(**kwargs) self._mysql_dbapi_version = ( self._parse_dbapi_version(self.dbapi.__version__) @@ -128,7 +144,7 @@ def __init__(self, **kwargs): else (0, 0, 0) ) - def _parse_dbapi_version(self, version): + def _parse_dbapi_version(self, version: str) -> tuple[int, ...]: m = re.match(r"(\d+)\.(\d+)(?:\.(\d+))?", version) if m: return tuple(int(x) for x in m.group(1, 2, 3) if x is not None) @@ -136,7 +152,7 @@ def _parse_dbapi_version(self, version): return (0, 0, 0) @util.langhelpers.memoized_property - def supports_server_side_cursors(self): + def supports_server_side_cursors(self) -> bool: try: cursors = __import__("MySQLdb.cursors").cursors self._sscursor = cursors.SSCursor @@ -145,13 +161,13 @@ def supports_server_side_cursors(self): return False @classmethod - def import_dbapi(cls): + def import_dbapi(cls) -> DBAPIModule: return __import__("MySQLdb") - def on_connect(self): + def on_connect(self) -> Callable[[DBAPIConnection], None]: super_ = super().on_connect() - def on_connect(conn): + def on_connect(conn: DBAPIConnection) -> None: if super_ is not None: super_(conn) @@ -164,43 +180,24 @@ def on_connect(conn): return on_connect - def do_ping(self, dbapi_connection): + def do_ping(self, dbapi_connection: DBAPIConnection) -> Literal[True]: dbapi_connection.ping() return True - def do_executemany(self, cursor, statement, parameters, context=None): + def do_executemany( + self, + cursor: DBAPICursor, + statement: str, + parameters: _DBAPIMultiExecuteParams, + context: Optional[ExecutionContext] = None, + ) -> None: rowcount = cursor.executemany(statement, parameters) if context is not None: - context._rowcount = rowcount - - def _check_unicode_returns(self, connection): - # work around issue fixed in - # https://github.com/farcepest/MySQLdb1/commit/cd44524fef63bd3fcb71947392326e9742d520e8 - # specific issue w/ the utf8mb4_bin collation and unicode returns - - collation = connection.exec_driver_sql( - "show collation where %s = 'utf8mb4' and %s = 'utf8mb4_bin'" - % ( - self.identifier_preparer.quote("Charset"), - self.identifier_preparer.quote("Collation"), - ) - ).scalar() - has_utf8mb4_bin = self.server_version_info > (5,) and collation - if has_utf8mb4_bin: - additional_tests = [ - sql.collate( - sql.cast( - sql.literal_column("'test collated returns'"), - TEXT(charset="utf8mb4"), - ), - "utf8mb4_bin", - ) - ] - else: - additional_tests = [] - return super()._check_unicode_returns(connection, additional_tests) + cast(MySQLExecutionContext, context)._rowcount = rowcount - def create_connect_args(self, url, _translate_args=None): + def create_connect_args( + self, url: URL, _translate_args: Optional[dict[str, Any]] = None + ) -> ConnectArgsType: if _translate_args is None: _translate_args = dict( database="db", username="user", password="passwd" @@ -249,9 +246,9 @@ def create_connect_args(self, url, _translate_args=None): if client_flag_found_rows is not None: client_flag |= client_flag_found_rows opts["client_flag"] = client_flag - return [[], opts] + return [], opts - def _found_rows_client_flag(self): + def _found_rows_client_flag(self) -> Optional[int]: if self.dbapi is not None: try: CLIENT_FLAGS = __import__( @@ -260,20 +257,23 @@ def _found_rows_client_flag(self): except (AttributeError, ImportError): return None else: - return CLIENT_FLAGS.FOUND_ROWS + return CLIENT_FLAGS.FOUND_ROWS # type: ignore else: return None - def _extract_error_code(self, exception): - return exception.args[0] + def _extract_error_code(self, exception: DBAPIModule.Error) -> int: + return exception.args[0] # type: ignore[no-any-return] - def _detect_charset(self, connection): + def _detect_charset(self, connection: Connection) -> str: """Sniff out the character set in use for connection results.""" try: # note: the SQL here would be # "SHOW VARIABLES LIKE 'character_set%%'" - cset_name = connection.connection.character_set_name + + cset_name: Callable[[], str] = ( + connection.connection.character_set_name + ) except AttributeError: util.warn( "No 'character_set_name' can be detected with " @@ -285,7 +285,9 @@ def _detect_charset(self, connection): else: return cset_name() - def get_isolation_level_values(self, dbapi_connection): + def get_isolation_level_values( + self, dbapi_conn: DBAPIConnection + ) -> tuple[IsolationLevel, ...]: return ( "SERIALIZABLE", "READ UNCOMMITTED", @@ -294,7 +296,12 @@ def get_isolation_level_values(self, dbapi_connection): "AUTOCOMMIT", ) - def set_isolation_level(self, dbapi_connection, level): + def detect_autocommit_setting(self, dbapi_conn: DBAPIConnection) -> bool: + return dbapi_conn.get_autocommit() # type: ignore[no-any-return] + + def set_isolation_level( + self, dbapi_connection: DBAPIConnection, level: IsolationLevel + ) -> None: if level == "AUTOCOMMIT": dbapi_connection.autocommit(True) else: diff --git a/lib/sqlalchemy/dialects/mysql/provision.py b/lib/sqlalchemy/dialects/mysql/provision.py index 46070848cb1..46cd8abf7de 100644 --- a/lib/sqlalchemy/dialects/mysql/provision.py +++ b/lib/sqlalchemy/dialects/mysql/provision.py @@ -1,14 +1,18 @@ # dialects/mysql/provision.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php # mypy: ignore-errors +import contextlib +from ... import event from ... import exc +from ...testing.provision import allow_stale_update_impl from ...testing.provision import configure_follower from ...testing.provision import create_db +from ...testing.provision import delete_from_all_tables from ...testing.provision import drop_db from ...testing.provision import generate_driver_url from ...testing.provision import temp_table_keyword_args @@ -96,7 +100,13 @@ def _mysql_temp_table_keyword_args(cfg, eng): @upsert.for_db("mariadb") def _upsert( - cfg, table, returning, *, set_lambda=None, sort_by_parameter_order=False + cfg, + table, + returning, + *, + set_lambda=None, + sort_by_parameter_order=False, + index_elements=None, ): from sqlalchemy.dialects.mysql import insert @@ -112,3 +122,32 @@ def _upsert( *returning, sort_by_parameter_order=sort_by_parameter_order ) return stmt + + +@delete_from_all_tables.for_db("mysql", "mariadb") +def _delete_from_all_tables(connection, cfg, metadata): + connection.exec_driver_sql("SET foreign_key_checks = 0") + try: + delete_from_all_tables.call_original(connection, cfg, metadata) + finally: + connection.exec_driver_sql("SET foreign_key_checks = 1") + + +@allow_stale_update_impl.for_db("mariadb") +def _allow_stale_update_impl(cfg): + @contextlib.contextmanager + def go(): + @event.listens_for(cfg.db, "engine_connect") + def turn_off_snapshot_isolation(conn): + conn.exec_driver_sql("SET innodb_snapshot_isolation = 'OFF'") + conn.rollback() + + try: + yield + finally: + event.remove(cfg.db, "engine_connect", turn_off_snapshot_isolation) + + # dispose the pool; quick way to just have those reset + cfg.db.dispose() + + return go() diff --git a/lib/sqlalchemy/dialects/mysql/pymysql.py b/lib/sqlalchemy/dialects/mysql/pymysql.py index 67cb4cdd766..a9be4ed4140 100644 --- a/lib/sqlalchemy/dialects/mysql/pymysql.py +++ b/lib/sqlalchemy/dialects/mysql/pymysql.py @@ -1,11 +1,9 @@ # dialects/mysql/pymysql.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors - r""" @@ -49,10 +47,26 @@ to the pymysql driver as well. """ # noqa +from __future__ import annotations + +from typing import Any +from typing import Literal +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union from .mysqldb import MySQLDialect_mysqldb from ...util import langhelpers +if TYPE_CHECKING: + + from ...engine.interfaces import ConnectArgsType + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import DBAPICursor + from ...engine.interfaces import DBAPIModule + from ...engine.interfaces import PoolProxiedConnection + from ...engine.url import URL + class MySQLDialect_pymysql(MySQLDialect_mysqldb): driver = "pymysql" @@ -61,7 +75,7 @@ class MySQLDialect_pymysql(MySQLDialect_mysqldb): description_encoding = None @langhelpers.memoized_property - def supports_server_side_cursors(self): + def supports_server_side_cursors(self) -> bool: try: cursors = __import__("pymysql.cursors").cursors self._sscursor = cursors.SSCursor @@ -70,11 +84,11 @@ def supports_server_side_cursors(self): return False @classmethod - def import_dbapi(cls): + def import_dbapi(cls) -> DBAPIModule: return __import__("pymysql") @langhelpers.memoized_property - def _send_false_to_ping(self): + def _send_false_to_ping(self) -> bool: """determine if pymysql has deprecated, changed the default of, or removed the 'reconnect' argument of connection.ping(). @@ -101,7 +115,7 @@ def _send_false_to_ping(self): not insp.defaults or insp.defaults[0] is not False ) - def do_ping(self, dbapi_connection): + def do_ping(self, dbapi_connection: DBAPIConnection) -> Literal[True]: if self._send_false_to_ping: dbapi_connection.ping(False) else: @@ -109,17 +123,24 @@ def do_ping(self, dbapi_connection): return True - def create_connect_args(self, url, _translate_args=None): + def create_connect_args( + self, url: URL, _translate_args: Optional[dict[str, Any]] = None + ) -> ConnectArgsType: if _translate_args is None: _translate_args = dict(username="user") return super().create_connect_args( url, _translate_args=_translate_args ) - def is_disconnect(self, e, connection, cursor): + def is_disconnect( + self, + e: DBAPIModule.Error, + connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]], + cursor: Optional[DBAPICursor], + ) -> bool: if super().is_disconnect(e, connection, cursor): return True - elif isinstance(e, self.dbapi.Error): + elif isinstance(e, self.loaded_dbapi.Error): str_e = str(e).lower() return ( "already closed" in str_e or "connection was killed" in str_e @@ -127,7 +148,7 @@ def is_disconnect(self, e, connection, cursor): else: return False - def _extract_error_code(self, exception): + def _extract_error_code(self, exception: BaseException) -> Any: if isinstance(exception.args[0], Exception): exception = exception.args[0] return exception.args[0] diff --git a/lib/sqlalchemy/dialects/mysql/pyodbc.py b/lib/sqlalchemy/dialects/mysql/pyodbc.py index 6d44bd38370..8198f8c7da2 100644 --- a/lib/sqlalchemy/dialects/mysql/pyodbc.py +++ b/lib/sqlalchemy/dialects/mysql/pyodbc.py @@ -1,15 +1,13 @@ # dialects/mysql/pyodbc.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors r""" - .. dialect:: mysql+pyodbc :name: PyODBC :dbapi: pyodbc @@ -44,8 +42,15 @@ connection_uri = "mysql+pyodbc:///?odbc_connect=%s" % params """ # noqa +from __future__ import annotations +import datetime import re +from typing import Any +from typing import Callable +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union from .base import MySQLDialect from .base import MySQLExecutionContext @@ -55,23 +60,31 @@ from ...connectors.pyodbc import PyODBCConnector from ...sql.sqltypes import Time +if TYPE_CHECKING: + from ...engine import Connection + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import Dialect + from ...sql.type_api import _ResultProcessorType + class _pyodbcTIME(TIME): - def result_processor(self, dialect, coltype): - def process(value): + def result_processor( + self, dialect: Dialect, coltype: object + ) -> _ResultProcessorType[datetime.time]: + def process(value: Any) -> Union[datetime.time, None]: # pyodbc returns a datetime.time object; no need to convert - return value + return value # type: ignore[no-any-return] return process class MySQLExecutionContext_pyodbc(MySQLExecutionContext): - def get_lastrowid(self): + def get_lastrowid(self) -> int: cursor = self.create_cursor() cursor.execute("SELECT LAST_INSERT_ID()") - lastrowid = cursor.fetchone()[0] + lastrowid = cursor.fetchone()[0] # type: ignore[index] cursor.close() - return lastrowid + return lastrowid # type: ignore[no-any-return] class MySQLDialect_pyodbc(PyODBCConnector, MySQLDialect): @@ -82,7 +95,7 @@ class MySQLDialect_pyodbc(PyODBCConnector, MySQLDialect): pyodbc_driver_name = "MySQL" - def _detect_charset(self, connection): + def _detect_charset(self, connection: Connection) -> str: """Sniff out the character set in use for connection results.""" # Prefer 'character_set_results' for the current connection over the @@ -107,21 +120,25 @@ def _detect_charset(self, connection): ) return "latin1" - def _get_server_version_info(self, connection): + def _get_server_version_info( + self, connection: Connection + ) -> tuple[int, ...]: return MySQLDialect._get_server_version_info(self, connection) - def _extract_error_code(self, exception): + def _extract_error_code(self, exception: BaseException) -> Optional[int]: m = re.compile(r"\((\d+)\)").search(str(exception.args)) - c = m.group(1) + if m is None: + return None + c: Optional[str] = m.group(1) if c: return int(c) else: return None - def on_connect(self): + def on_connect(self) -> Callable[[DBAPIConnection], None]: super_ = super().on_connect() - def on_connect(conn): + def on_connect(conn: DBAPIConnection) -> None: if super_ is not None: super_(conn) diff --git a/lib/sqlalchemy/dialects/mysql/reflection.py b/lib/sqlalchemy/dialects/mysql/reflection.py index 3998be977d9..1cc643e840f 100644 --- a/lib/sqlalchemy/dialects/mysql/reflection.py +++ b/lib/sqlalchemy/dialects/mysql/reflection.py @@ -1,46 +1,62 @@ # dialects/mysql/reflection.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors - +from __future__ import annotations import re +from typing import Any +from typing import Callable +from typing import Literal +from typing import Optional +from typing import overload +from typing import Sequence +from typing import TYPE_CHECKING +from typing import Union from .enumerated import ENUM from .enumerated import SET from .types import DATETIME from .types import TIME from .types import TIMESTAMP -from ... import log from ... import types as sqltypes from ... import util +if TYPE_CHECKING: + from .base import MySQLDialect + from .base import MySQLIdentifierPreparer + from ...engine.interfaces import ReflectedColumn + class ReflectedState: """Stores raw information about a SHOW CREATE TABLE statement.""" - def __init__(self): - self.columns = [] - self.table_options = {} - self.table_name = None - self.keys = [] - self.fk_constraints = [] - self.ck_constraints = [] + charset: Optional[str] + + def __init__(self) -> None: + self.columns: list[ReflectedColumn] = [] + self.table_options: dict[str, str] = {} + self.table_name: Optional[str] = None + self.keys: list[dict[str, Any]] = [] + self.fk_constraints: list[dict[str, Any]] = [] + self.ck_constraints: list[dict[str, Any]] = [] -@log.class_logger class MySQLTableDefinitionParser: """Parses the results of a SHOW CREATE TABLE statement.""" - def __init__(self, dialect, preparer): + def __init__( + self, dialect: MySQLDialect, preparer: MySQLIdentifierPreparer + ): self.dialect = dialect self.preparer = preparer self._prep_regexes() - def parse(self, show_create, charset): + def parse( + self, show_create: str, charset: Optional[str] + ) -> ReflectedState: state = ReflectedState() state.charset = charset for line in re.split(r"\r?\n", show_create): @@ -65,11 +81,11 @@ def parse(self, show_create, charset): if type_ is None: util.warn("Unknown schema content: %r" % line) elif type_ == "key": - state.keys.append(spec) + state.keys.append(spec) # type: ignore[arg-type] elif type_ == "fk_constraint": - state.fk_constraints.append(spec) + state.fk_constraints.append(spec) # type: ignore[arg-type] elif type_ == "ck_constraint": - state.ck_constraints.append(spec) + state.ck_constraints.append(spec) # type: ignore[arg-type] else: pass return state @@ -77,7 +93,13 @@ def parse(self, show_create, charset): def _check_view(self, sql: str) -> bool: return bool(self._re_is_view.match(sql)) - def _parse_constraints(self, line): + def _parse_constraints(self, line: str) -> Union[ + tuple[None, str], + tuple[Literal["partition"], str], + tuple[ + Literal["ck_constraint", "fk_constraint", "key"], dict[str, str] + ], + ]: """Parse a KEY or CONSTRAINT line. :param line: A line of SHOW CREATE TABLE output @@ -127,7 +149,7 @@ def _parse_constraints(self, line): # No match. return (None, line) - def _parse_table_name(self, line, state): + def _parse_table_name(self, line: str, state: ReflectedState) -> None: """Extract the table name. :param line: The first line of SHOW CREATE TABLE @@ -138,7 +160,7 @@ def _parse_table_name(self, line, state): if m: state.table_name = cleanup(m.group("name")) - def _parse_table_options(self, line, state): + def _parse_table_options(self, line: str, state: ReflectedState) -> None: """Build a dictionary of all reflected table-level options. :param line: The final line of SHOW CREATE TABLE output. @@ -164,7 +186,9 @@ def _parse_table_options(self, line, state): for opt, val in options.items(): state.table_options["%s_%s" % (self.dialect.name, opt)] = val - def _parse_partition_options(self, line, state): + def _parse_partition_options( + self, line: str, state: ReflectedState + ) -> None: options = {} new_line = line[:] @@ -220,7 +244,7 @@ def _parse_partition_options(self, line, state): else: state.table_options["%s_%s" % (self.dialect.name, opt)] = val - def _parse_column(self, line, state): + def _parse_column(self, line: str, state: ReflectedState) -> None: """Extract column details. Falls back to a 'minimal support' variant if full parse fails. @@ -283,7 +307,7 @@ def _parse_column(self, line, state): type_instance = col_type(*type_args, **type_kw) - col_kw = {} + col_kw: dict[str, Any] = {} # NOT NULL col_kw["nullable"] = True @@ -324,9 +348,13 @@ def _parse_column(self, line, state): name=name, type=type_instance, default=default, comment=comment ) col_d.update(col_kw) - state.columns.append(col_d) + state.columns.append(col_d) # type: ignore[arg-type] - def _describe_to_create(self, table_name, columns): + def _describe_to_create( + self, + table_name: str, + columns: Sequence[tuple[str, str, str, str, str, str]], + ) -> str: """Re-format DESCRIBE output as a SHOW CREATE TABLE string. DESCRIBE is a much simpler reflection and is sufficient for @@ -379,7 +407,9 @@ def _describe_to_create(self, table_name, columns): ] ) - def _parse_keyexprs(self, identifiers): + def _parse_keyexprs( + self, identifiers: str + ) -> list[tuple[str, Optional[int], str]]: """Unpack '"col"(2),"col" ASC'-ish strings into components.""" return [ @@ -389,11 +419,12 @@ def _parse_keyexprs(self, identifiers): ) ] - def _prep_regexes(self): + def _prep_regexes(self) -> None: """Pre-compile regular expressions.""" - self._re_columns = [] - self._pr_options = [] + self._pr_options: list[ + tuple[re.Pattern[Any], Optional[Callable[[str], str]]] + ] = [] _final = self.preparer.final_quote @@ -451,7 +482,7 @@ def _prep_regexes(self): r"(?: +COLLATE +(?P[\w_]+))?" r"(?: +(?P(?:NOT )?NULL))?" r"(?: +DEFAULT +(?P" - r"(?:NULL|'(?:''|[^'])*'|[\-\w\.\(\)]+" + r"(?:NULL|'(?:''|[^'])*'|\(.+?\)|[\-\w\.\(\)]+" r"(?: +ON UPDATE [\-\w\.\(\)]+)?)" r"))?" r"(?: +(?:GENERATED ALWAYS)? ?AS +(?P\(" @@ -582,21 +613,21 @@ def _prep_regexes(self): _optional_equals = r"(?:\s*(?:=\s*)|\s+)" - def _add_option_string(self, directive): + def _add_option_string(self, directive: str) -> None: regex = r"(?P%s)%s" r"'(?P(?:[^']|'')*?)'(?!')" % ( re.escape(directive), self._optional_equals, ) self._pr_options.append(_pr_compile(regex, cleanup_text)) - def _add_option_word(self, directive): + def _add_option_word(self, directive: str) -> None: regex = r"(?P%s)%s" r"(?P\w+)" % ( re.escape(directive), self._optional_equals, ) self._pr_options.append(_pr_compile(regex)) - def _add_partition_option_word(self, directive): + def _add_partition_option_word(self, directive: str) -> None: if directive == "PARTITION BY" or directive == "SUBPARTITION BY": regex = r"(?%s)%s" r"(?P\w+.*)" % ( re.escape(directive), @@ -611,7 +642,7 @@ def _add_partition_option_word(self, directive): regex = r"(?%s)(?!\S)" % (re.escape(directive),) self._pr_options.append(_pr_compile(regex)) - def _add_option_regex(self, directive, regex): + def _add_option_regex(self, directive: str, regex: str) -> None: regex = r"(?P%s)%s" r"(?P%s)" % ( re.escape(directive), self._optional_equals, @@ -629,21 +660,35 @@ def _add_option_regex(self, directive, regex): ) -def _pr_compile(regex, cleanup=None): +@overload +def _pr_compile( + regex: str, cleanup: Callable[[str], str] +) -> tuple[re.Pattern[Any], Callable[[str], str]]: ... + + +@overload +def _pr_compile( + regex: str, cleanup: None = None +) -> tuple[re.Pattern[Any], None]: ... + + +def _pr_compile( + regex: str, cleanup: Optional[Callable[[str], str]] = None +) -> tuple[re.Pattern[Any], Optional[Callable[[str], str]]]: """Prepare a 2-tuple of compiled regex and callable.""" return (_re_compile(regex), cleanup) -def _re_compile(regex): +def _re_compile(regex: str) -> re.Pattern[Any]: """Compile a string to regex, I and UNICODE.""" return re.compile(regex, re.I | re.UNICODE) -def _strip_values(values): +def _strip_values(values: Sequence[str]) -> list[str]: "Strip reflected values quotes" - strip_values = [] + strip_values: list[str] = [] for a in values: if a[0:1] == '"' or a[0:1] == "'": # strip enclosing quotes and unquote interior @@ -655,7 +700,9 @@ def _strip_values(values): def cleanup_text(raw_text: str) -> str: if "\\" in raw_text: raw_text = re.sub( - _control_char_regexp, lambda s: _control_char_map[s[0]], raw_text + _control_char_regexp, + lambda s: _control_char_map[s[0]], # type: ignore[index] + raw_text, ) return raw_text.replace("''", "'") diff --git a/lib/sqlalchemy/dialects/mysql/reserved_words.py b/lib/sqlalchemy/dialects/mysql/reserved_words.py index 34fecf42724..5f9e4573ee4 100644 --- a/lib/sqlalchemy/dialects/mysql/reserved_words.py +++ b/lib/sqlalchemy/dialects/mysql/reserved_words.py @@ -1,5 +1,5 @@ # dialects/mysql/reserved_words.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -11,7 +11,6 @@ # https://mariadb.com/kb/en/reserved-words/ # includes: Reserved Words, Oracle Mode (separate set unioned) # excludes: Exceptions, Function Names -# mypy: ignore-errors RESERVED_WORDS_MARIADB = { "accessible", diff --git a/lib/sqlalchemy/dialects/mysql/types.py b/lib/sqlalchemy/dialects/mysql/types.py index 015d51a1058..76376081c8d 100644 --- a/lib/sqlalchemy/dialects/mysql/types.py +++ b/lib/sqlalchemy/dialects/mysql/types.py @@ -1,18 +1,30 @@ # dialects/mysql/types.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors - +from __future__ import annotations import datetime +import decimal +from typing import Any +from typing import Iterable +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union from ... import exc from ... import util from ...sql import sqltypes +if TYPE_CHECKING: + from .base import MySQLDialect + from ...engine.interfaces import Dialect + from ...sql.type_api import _BindProcessorType + from ...sql.type_api import _ResultProcessorType + from ...sql.type_api import TypeEngine + class _NumericCommonType: """Base for MySQL numeric types. @@ -22,24 +34,36 @@ class _NumericCommonType: """ - def __init__(self, unsigned=False, zerofill=False, **kw): + def __init__( + self, unsigned: bool = False, zerofill: bool = False, **kw: Any + ): self.unsigned = unsigned self.zerofill = zerofill super().__init__(**kw) -class _NumericType(_NumericCommonType, sqltypes.Numeric): +class _NumericType( + _NumericCommonType, sqltypes.Numeric[Union[decimal.Decimal, float]] +): - def __repr__(self): - return util.generic_repr( + def repr_struct(self) -> util.GenericRepr: + return util.GenericRepr( self, to_inspect=[_NumericType, _NumericCommonType, sqltypes.Numeric], ) -class _FloatType(_NumericCommonType, sqltypes.Float): +class _FloatType( + _NumericCommonType, sqltypes.Float[Union[decimal.Decimal, float]] +): - def __init__(self, precision=None, scale=None, asdecimal=True, **kw): + def __init__( + self, + precision: Optional[int] = None, + scale: Optional[int] = None, + asdecimal: bool = True, + **kw: Any, + ): if isinstance(self, (REAL, DOUBLE)) and ( (precision is None and scale is not None) or (precision is not None and scale is None) @@ -51,19 +75,19 @@ def __init__(self, precision=None, scale=None, asdecimal=True, **kw): super().__init__(precision=precision, asdecimal=asdecimal, **kw) self.scale = scale - def __repr__(self): - return util.generic_repr( + def repr_struct(self) -> util.GenericRepr: + return util.GenericRepr( self, to_inspect=[_FloatType, _NumericCommonType, sqltypes.Float] ) class _IntegerType(_NumericCommonType, sqltypes.Integer): - def __init__(self, display_width=None, **kw): + def __init__(self, display_width: Optional[int] = None, **kw: Any): self.display_width = display_width super().__init__(**kw) - def __repr__(self): - return util.generic_repr( + def repr_struct(self) -> util.GenericRepr: + return util.GenericRepr( self, to_inspect=[_IntegerType, _NumericCommonType, sqltypes.Integer], ) @@ -74,13 +98,13 @@ class _StringType(sqltypes.String): def __init__( self, - charset=None, - collation=None, - ascii=False, # noqa - binary=False, - unicode=False, - national=False, - **kw, + charset: Optional[str] = None, + collation: Optional[str] = None, + ascii: bool = False, # noqa + binary: bool = False, + unicode: bool = False, + national: bool = False, + **kw: Any, ): self.charset = charset @@ -93,25 +117,33 @@ def __init__( self.national = national super().__init__(**kw) - def __repr__(self): - return util.generic_repr( + def repr_struct(self) -> util.GenericRepr: + return util.GenericRepr( self, to_inspect=[_StringType, sqltypes.String] ) -class _MatchType(sqltypes.Float, sqltypes.MatchType): - def __init__(self, **kw): +class _MatchType( + sqltypes.Float[Union[decimal.Decimal, float]], sqltypes.MatchType +): + def __init__(self, **kw: Any): # TODO: float arguments? - sqltypes.Float.__init__(self) + sqltypes.Float.__init__(self) # type: ignore[arg-type] sqltypes.MatchType.__init__(self) -class NUMERIC(_NumericType, sqltypes.NUMERIC): +class NUMERIC(_NumericType, sqltypes.NUMERIC[Union[decimal.Decimal, float]]): """MySQL NUMERIC type.""" __visit_name__ = "NUMERIC" - def __init__(self, precision=None, scale=None, asdecimal=True, **kw): + def __init__( + self, + precision: Optional[int] = None, + scale: Optional[int] = None, + asdecimal: bool = True, + **kw: Any, + ): """Construct a NUMERIC. :param precision: Total digits in this number. If scale and precision @@ -132,12 +164,18 @@ def __init__(self, precision=None, scale=None, asdecimal=True, **kw): ) -class DECIMAL(_NumericType, sqltypes.DECIMAL): +class DECIMAL(_NumericType, sqltypes.DECIMAL[Union[decimal.Decimal, float]]): """MySQL DECIMAL type.""" __visit_name__ = "DECIMAL" - def __init__(self, precision=None, scale=None, asdecimal=True, **kw): + def __init__( + self, + precision: Optional[int] = None, + scale: Optional[int] = None, + asdecimal: bool = True, + **kw: Any, + ): """Construct a DECIMAL. :param precision: Total digits in this number. If scale and precision @@ -158,12 +196,18 @@ def __init__(self, precision=None, scale=None, asdecimal=True, **kw): ) -class DOUBLE(_FloatType, sqltypes.DOUBLE): +class DOUBLE(_FloatType, sqltypes.DOUBLE[Union[decimal.Decimal, float]]): """MySQL DOUBLE type.""" __visit_name__ = "DOUBLE" - def __init__(self, precision=None, scale=None, asdecimal=True, **kw): + def __init__( + self, + precision: Optional[int] = None, + scale: Optional[int] = None, + asdecimal: bool = True, + **kw: Any, + ): """Construct a DOUBLE. .. note:: @@ -192,12 +236,18 @@ def __init__(self, precision=None, scale=None, asdecimal=True, **kw): ) -class REAL(_FloatType, sqltypes.REAL): +class REAL(_FloatType, sqltypes.REAL[Union[decimal.Decimal, float]]): """MySQL REAL type.""" __visit_name__ = "REAL" - def __init__(self, precision=None, scale=None, asdecimal=True, **kw): + def __init__( + self, + precision: Optional[int] = None, + scale: Optional[int] = None, + asdecimal: bool = True, + **kw: Any, + ): """Construct a REAL. .. note:: @@ -226,12 +276,18 @@ def __init__(self, precision=None, scale=None, asdecimal=True, **kw): ) -class FLOAT(_FloatType, sqltypes.FLOAT): +class FLOAT(_FloatType, sqltypes.FLOAT[Union[decimal.Decimal, float]]): """MySQL FLOAT type.""" __visit_name__ = "FLOAT" - def __init__(self, precision=None, scale=None, asdecimal=False, **kw): + def __init__( + self, + precision: Optional[int] = None, + scale: Optional[int] = None, + asdecimal: bool = False, + **kw: Any, + ): """Construct a FLOAT. :param precision: Total digits in this number. If scale and precision @@ -251,7 +307,9 @@ def __init__(self, precision=None, scale=None, asdecimal=False, **kw): precision=precision, scale=scale, asdecimal=asdecimal, **kw ) - def bind_processor(self, dialect): + def bind_processor( + self, dialect: Dialect + ) -> Optional[_BindProcessorType[Union[decimal.Decimal, float]]]: return None @@ -260,7 +318,7 @@ class INTEGER(_IntegerType, sqltypes.INTEGER): __visit_name__ = "INTEGER" - def __init__(self, display_width=None, **kw): + def __init__(self, display_width: Optional[int] = None, **kw: Any): """Construct an INTEGER. :param display_width: Optional, maximum display width for this number. @@ -281,7 +339,7 @@ class BIGINT(_IntegerType, sqltypes.BIGINT): __visit_name__ = "BIGINT" - def __init__(self, display_width=None, **kw): + def __init__(self, display_width: Optional[int] = None, **kw: Any): """Construct a BIGINTEGER. :param display_width: Optional, maximum display width for this number. @@ -302,7 +360,7 @@ class MEDIUMINT(_IntegerType): __visit_name__ = "MEDIUMINT" - def __init__(self, display_width=None, **kw): + def __init__(self, display_width: Optional[int] = None, **kw: Any): """Construct a MEDIUMINTEGER :param display_width: Optional, maximum display width for this number. @@ -323,7 +381,7 @@ class TINYINT(_IntegerType): __visit_name__ = "TINYINT" - def __init__(self, display_width=None, **kw): + def __init__(self, display_width: Optional[int] = None, **kw: Any): """Construct a TINYINT. :param display_width: Optional, maximum display width for this number. @@ -338,13 +396,19 @@ def __init__(self, display_width=None, **kw): """ super().__init__(display_width=display_width, **kw) + def _compare_type_affinity(self, other: TypeEngine[Any]) -> bool: + return ( + self._type_affinity is other._type_affinity + or other._type_affinity is sqltypes.Boolean + ) + class SMALLINT(_IntegerType, sqltypes.SMALLINT): """MySQL SMALLINTEGER type.""" __visit_name__ = "SMALLINT" - def __init__(self, display_width=None, **kw): + def __init__(self, display_width: Optional[int] = None, **kw: Any): """Construct a SMALLINTEGER. :param display_width: Optional, maximum display width for this number. @@ -360,7 +424,7 @@ def __init__(self, display_width=None, **kw): super().__init__(display_width=display_width, **kw) -class BIT(sqltypes.TypeEngine): +class BIT(sqltypes.TypeEngine[Any]): """MySQL BIT type. This type is for MySQL 5.0.3 or greater for MyISAM, and 5.0.5 or greater @@ -371,7 +435,7 @@ class BIT(sqltypes.TypeEngine): __visit_name__ = "BIT" - def __init__(self, length=None): + def __init__(self, length: Optional[int] = None): """Construct a BIT. :param length: Optional, number of bits. @@ -379,19 +443,19 @@ def __init__(self, length=None): """ self.length = length - def result_processor(self, dialect, coltype): + def result_processor( + self, dialect: MySQLDialect, coltype: object # type: ignore[override] + ) -> Optional[_ResultProcessorType[Any]]: """Convert a MySQL's 64 bit, variable length binary string to a long.""" if dialect.supports_native_bit: return None - def process(value): + def process(value: Optional[Iterable[int]]) -> Optional[int]: if value is not None: v = 0 for i in value: - if not isinstance(i, int): - i = ord(i) # convert byte to int on Python 2 v = v << 8 | i return v return value @@ -404,7 +468,7 @@ class TIME(sqltypes.TIME): __visit_name__ = "TIME" - def __init__(self, timezone=False, fsp=None): + def __init__(self, timezone: bool = False, fsp: Optional[int] = None): """Construct a MySQL TIME type. :param timezone: not used by the MySQL dialect. @@ -423,10 +487,12 @@ def __init__(self, timezone=False, fsp=None): super().__init__(timezone=timezone) self.fsp = fsp - def result_processor(self, dialect, coltype): + def result_processor( + self, dialect: Dialect, coltype: object + ) -> _ResultProcessorType[datetime.time]: time = datetime.time - def process(value): + def process(value: Any) -> Optional[datetime.time]: # convert from a timedelta value if value is not None: microseconds = value.microseconds @@ -449,7 +515,7 @@ class TIMESTAMP(sqltypes.TIMESTAMP): __visit_name__ = "TIMESTAMP" - def __init__(self, timezone=False, fsp=None): + def __init__(self, timezone: bool = False, fsp: Optional[int] = None): """Construct a MySQL TIMESTAMP type. :param timezone: not used by the MySQL dialect. @@ -474,7 +540,7 @@ class DATETIME(sqltypes.DATETIME): __visit_name__ = "DATETIME" - def __init__(self, timezone=False, fsp=None): + def __init__(self, timezone: bool = False, fsp: Optional[int] = None): """Construct a MySQL DATETIME type. :param timezone: not used by the MySQL dialect. @@ -494,12 +560,12 @@ def __init__(self, timezone=False, fsp=None): self.fsp = fsp -class YEAR(sqltypes.TypeEngine): +class YEAR(sqltypes.TypeEngine[Any]): """MySQL YEAR type, for single byte storage of years 1901-2155.""" __visit_name__ = "YEAR" - def __init__(self, display_width=None): + def __init__(self, display_width: Optional[int] = None): self.display_width = display_width @@ -508,7 +574,7 @@ class TEXT(_StringType, sqltypes.TEXT): __visit_name__ = "TEXT" - def __init__(self, length=None, **kw): + def __init__(self, length: Optional[int] = None, **kw: Any): """Construct a TEXT. :param length: Optional, if provided the server may optimize storage @@ -544,7 +610,7 @@ class TINYTEXT(_StringType): __visit_name__ = "TINYTEXT" - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any): """Construct a TINYTEXT. :param charset: Optional, a column-level character set for this string @@ -577,7 +643,7 @@ class MEDIUMTEXT(_StringType): __visit_name__ = "MEDIUMTEXT" - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any): """Construct a MEDIUMTEXT. :param charset: Optional, a column-level character set for this string @@ -609,7 +675,7 @@ class LONGTEXT(_StringType): __visit_name__ = "LONGTEXT" - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any): """Construct a LONGTEXT. :param charset: Optional, a column-level character set for this string @@ -641,7 +707,7 @@ class VARCHAR(_StringType, sqltypes.VARCHAR): __visit_name__ = "VARCHAR" - def __init__(self, length=None, **kwargs): + def __init__(self, length: Optional[int] = None, **kwargs: Any) -> None: """Construct a VARCHAR. :param charset: Optional, a column-level character set for this string @@ -673,7 +739,7 @@ class CHAR(_StringType, sqltypes.CHAR): __visit_name__ = "CHAR" - def __init__(self, length=None, **kwargs): + def __init__(self, length: Optional[int] = None, **kwargs: Any): """Construct a CHAR. :param length: Maximum data length, in characters. @@ -689,7 +755,7 @@ def __init__(self, length=None, **kwargs): super().__init__(length=length, **kwargs) @classmethod - def _adapt_string_for_cast(cls, type_): + def _adapt_string_for_cast(cls, type_: sqltypes.String) -> sqltypes.CHAR: # copy the given string type into a CHAR # for the purposes of rendering a CAST expression type_ = sqltypes.to_instance(type_) @@ -718,7 +784,7 @@ class NVARCHAR(_StringType, sqltypes.NVARCHAR): __visit_name__ = "NVARCHAR" - def __init__(self, length=None, **kwargs): + def __init__(self, length: Optional[int] = None, **kwargs: Any): """Construct an NVARCHAR. :param length: Maximum data length, in characters. @@ -744,7 +810,7 @@ class NCHAR(_StringType, sqltypes.NCHAR): __visit_name__ = "NCHAR" - def __init__(self, length=None, **kwargs): + def __init__(self, length: Optional[int] = None, **kwargs: Any): """Construct an NCHAR. :param length: Maximum data length, in characters. diff --git a/lib/sqlalchemy/dialects/oracle/__init__.py b/lib/sqlalchemy/dialects/oracle/__init__.py index 7ceb743d616..0f0b0a5c737 100644 --- a/lib/sqlalchemy/dialects/oracle/__init__.py +++ b/lib/sqlalchemy/dialects/oracle/__init__.py @@ -1,5 +1,5 @@ # dialects/oracle/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -14,6 +14,7 @@ from .base import BINARY_DOUBLE from .base import BINARY_FLOAT from .base import BLOB +from .base import BOOLEAN from .base import CHAR from .base import CLOB from .base import DATE @@ -32,13 +33,21 @@ from .base import TIMESTAMP from .base import VARCHAR from .base import VARCHAR2 +from .base import VECTOR +from .base import VectorIndexConfig +from .base import VectorIndexType +from .json import JSON +from .vector import SparseVector +from .vector import VectorDistanceType +from .vector import VectorStorageFormat +from .vector import VectorStorageType # Alias oracledb also as oracledb_async oracledb_async = type( "oracledb_async", (ModuleType,), {"dialect": oracledb.dialect_async} ) -base.dialect = dialect = cx_oracle.dialect +base.dialect = dialect = oracledb.dialect __all__ = ( "VARCHAR", @@ -64,4 +73,13 @@ "NVARCHAR2", "ROWID", "REAL", + "BOOLEAN", + "VECTOR", + "VectorDistanceType", + "VectorIndexType", + "VectorIndexConfig", + "VectorStorageFormat", + "VectorStorageType", + "SparseVector", + "JSON", ) diff --git a/lib/sqlalchemy/dialects/oracle/base.py b/lib/sqlalchemy/dialects/oracle/base.py index 3d3ff9d5170..a27306e067a 100644 --- a/lib/sqlalchemy/dialects/oracle/base.py +++ b/lib/sqlalchemy/dialects/oracle/base.py @@ -1,5 +1,5 @@ # dialects/oracle/base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -146,17 +146,6 @@ warning is emitted for this initial first-connect condition as it is expected to be a common restriction on Oracle databases. -.. versionadded:: 1.3.16 added support for AUTOCOMMIT to the cx_Oracle dialect - as well as the notion of a default isolation level - -.. versionadded:: 1.3.21 Added support for SERIALIZABLE as well as live - reading of the isolation level. - -.. versionchanged:: 1.3.22 In the event that the default isolation - level cannot be read due to permissions on the v$transaction view as - is common in Oracle installations, the default isolation level is hardcoded - to "READ COMMITTED" which was the behavior prior to 1.3.21. - .. seealso:: :ref:`dbapi_autocommit` @@ -553,9 +542,6 @@ :meth:`_reflection.Inspector.get_check_constraints`, and :meth:`_reflection.Inspector.get_indexes`. -.. versionchanged:: 1.2 The Oracle Database dialect can now reflect UNIQUE and - CHECK constraints. - When using reflection at the :class:`_schema.Table` level, the :class:`_schema.Table` will also include these constraints. @@ -654,6 +640,49 @@ ), ) +.. _oracle_boolean_support: + +Boolean Support +--------------- + +.. versionadded:: 2.1 + +Oracle Database 23ai introduced native support for the ``BOOLEAN`` datatype. +The Oracle dialect automatically detects the database version and uses the +native ``BOOLEAN`` type when available, or falls back to emulation using +``SMALLINT`` on older Oracle versions. + +The standard :class:`_types.Boolean` type can be used in table definitions:: + + from sqlalchemy import Boolean, Column, Integer, Table, MetaData + + metadata = MetaData() + + my_table = Table( + "my_table", + metadata, + Column("id", Integer, primary_key=True), + Column("flag", Boolean), + ) + +On Oracle 23ai and later, this will generate DDL using the native ``BOOLEAN`` type: + +.. code-block:: sql + + CREATE TABLE my_table ( + id INTEGER NOT NULL, + flag BOOLEAN, + PRIMARY KEY (id) + ) + +On earlier Oracle versions, it will use ``SMALLINT`` for storage with appropriate +constraints and conversions. + +The :class:`_types.Boolean` type is also available as ``BOOLEAN`` from the Oracle +dialect for consistency with other type names:: + + from sqlalchemy.dialects.oracle import BOOLEAN + DateTime Compatibility ---------------------- @@ -744,21 +773,247 @@ number of prefix columns to compress, or ``True`` to use the default (all columns for non-unique indexes, all but the last column for unique indexes). +.. _oracle_vector_datatype: + +VECTOR Datatype +--------------- + +Oracle Database 23ai introduced a new VECTOR datatype for artificial intelligence +and machine learning search operations. The VECTOR datatype is a homogeneous array +of 8-bit signed integers, 8-bit unsigned integers (binary), 32-bit floating-point +numbers, or 64-bit floating-point numbers. + +A vector's storage type can be either DENSE or SPARSE. A dense vector contains +meaningful values in most or all of its dimensions. In contrast, a sparse vector +has non-zero values in only a few dimensions, with the majority being zero. + +Sparse vectors are represented by the total number of vector dimensions, an array +of indices, and an array of values where each value’s location in the vector is +indicated by the corresponding indices array position. All other vector values are +treated as zero. + +The storage formats that can be used with sparse vectors are float32, float64, and +int8. Note that the binary storage format cannot be used with sparse vectors. + +Sparse vectors are supported when you are using Oracle Database 23.7 or later. + +.. seealso:: + + `Using VECTOR Data + `_ - in the documentation + for the :ref:`oracledb` driver. + +.. versionadded:: 2.0.41 - Added VECTOR datatype + +.. versionadded:: 2.0.43 - Added DENSE/SPARSE support + +CREATE TABLE support for VECTOR +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +With the :class:`.VECTOR` datatype, you can specify the number of dimensions, +the storage format, and the storage type for the data. Valid values for the +storage format are enum members of :class:`.VectorStorageFormat`. Valid values +for the storage type are enum members of :class:`.VectorStorageType`. If +storage type is not specified, a DENSE vector is created by default. + +To create a table that includes a :class:`.VECTOR` column:: + + from sqlalchemy.dialects.oracle import ( + VECTOR, + VectorStorageFormat, + VectorStorageType, + ) + + t = Table( + "t1", + metadata, + Column("id", Integer, primary_key=True), + Column( + "embedding", + VECTOR( + dim=3, + storage_format=VectorStorageFormat.FLOAT32, + storage_type=VectorStorageType.SPARSE, + ), + ), + Column(...), + ..., + ) + +Vectors can also be defined with an arbitrary number of dimensions and formats. +This allows you to specify vectors of different dimensions with the various +storage formats mentioned below. + +**Examples** + +* In this case, the storage format is flexible, allowing any vector type data to be + inserted, such as INT8 or BINARY etc:: + + vector_col: Mapped[array.array] = mapped_column(VECTOR(dim=3)) + +* The dimension is flexible in this case, meaning that any dimension vector can + be used:: + + vector_col: Mapped[array.array] = mapped_column( + VECTOR(storage_format=VectorStorageType.INT8) + ) + +* Both the dimensions and the storage format are flexible. It creates a DENSE vector:: + + vector_col: Mapped[array.array] = mapped_column(VECTOR) + +* To create a SPARSE vector with both dimensions and the storage format as flexible, + use the :attr:`.VectorStorageType.SPARSE` storage type:: + + vector_col: Mapped[array.array] = mapped_column( + VECTOR(storage_type=VectorStorageType.SPARSE) + ) + +Python Datatypes for VECTOR +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +VECTOR data can be inserted using Python list or Python ``array.array()`` objects. +Python arrays of type FLOAT (32-bit), DOUBLE (64-bit), INT (8-bit signed integers), +or BINARY (8-bit unsigned integers) are used as bind values when inserting +VECTOR columns:: + + from sqlalchemy import insert, select + + with engine.begin() as conn: + conn.execute( + insert(t1), + {"id": 1, "embedding": [1, 2, 3]}, + ) + +Data can be inserted into a sparse vector using the :class:`_oracle.SparseVector` +class, creating an object consisting of the number of dimensions, an array of indices, and a +corresponding array of values:: + + from sqlalchemy import insert, select + from sqlalchemy.dialects.oracle import SparseVector + + sparse_val = SparseVector(10, [1, 2], array.array("d", [23.45, 221.22])) + + with engine.begin() as conn: + conn.execute( + insert(t1), + {"id": 1, "embedding": sparse_val}, + ) + +VECTOR Indexes +~~~~~~~~~~~~~~ + +The VECTOR feature supports an Oracle-specific parameter ``oracle_vector`` +on the :class:`.Index` construct, which allows the construction of VECTOR +indexes. + +SPARSE vectors cannot be used in the creation of vector indexes. + +To utilize VECTOR indexing, set the ``oracle_vector`` parameter to True to use +the default values provided by Oracle. HNSW is the default indexing method:: + + from sqlalchemy import Index + + Index( + "vector_index", + t1.c.embedding, + oracle_vector=True, + ) + +The full range of parameters for vector indexes are available by using the +:class:`.VectorIndexConfig` dataclass in place of a boolean; this dataclass +allows full configuration of the index:: + + Index( + "hnsw_vector_index", + t1.c.embedding, + oracle_vector=VectorIndexConfig( + index_type=VectorIndexType.HNSW, + distance=VectorDistanceType.COSINE, + accuracy=90, + hnsw_neighbors=5, + hnsw_efconstruction=20, + parallel=10, + ), + ) + + Index( + "ivf_vector_index", + t1.c.embedding, + oracle_vector=VectorIndexConfig( + index_type=VectorIndexType.IVF, + distance=VectorDistanceType.DOT, + accuracy=90, + ivf_neighbor_partitions=5, + ), + ) + +For complete explanation of these parameters, see the Oracle documentation linked +below. + +.. seealso:: + + `CREATE VECTOR INDEX `_ - in the Oracle documentation + + + +Similarity Searching +~~~~~~~~~~~~~~~~~~~~ + +When using the :class:`_oracle.VECTOR` datatype with a :class:`.Column` or similar +ORM mapped construct, additional comparison functions are available, including: + +* ``l2_distance`` +* ``cosine_distance`` +* ``inner_product`` + +Example Usage:: + + result_vector = connection.scalars( + select(t1).order_by(t1.embedding.l2_distance([2, 3, 4])).limit(3) + ) + + for user in vector: + print(user.id, user.embedding) + +FETCH APPROXIMATE support +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Approximate vector search can only be performed when all syntax and semantic +rules are satisfied, the corresponding vector index is available, and the +query optimizer determines to perform it. If any of these conditions are +unmet, then an approximate search is not performed. In this case the query +returns exact results. + +To enable approximate searching during similarity searches on VECTORS, the +``oracle_fetch_approximate`` parameter may be used with the :meth:`.Select.fetch` +clause to add ``FETCH APPROX`` to the SELECT statement:: + + select(users_table).fetch(5, oracle_fetch_approximate=True) + """ # noqa from __future__ import annotations from collections import defaultdict +from dataclasses import fields from functools import lru_cache from functools import wraps import re +from typing import Any +from typing import Callable +from typing import TYPE_CHECKING from . import dictionary +from .json import JSON +from .json import JSONIndexType +from .json import JSONPathType from .types import _OracleBoolean from .types import _OracleDate from .types import BFILE from .types import BINARY_DOUBLE from .types import BINARY_FLOAT +from .types import BOOLEAN from .types import DATE from .types import FLOAT from .types import INTERVAL @@ -771,6 +1026,9 @@ from .types import ROWID # noqa from .types import TIMESTAMP from .types import VARCHAR2 # noqa +from .vector import VECTOR +from .vector import VectorIndexConfig +from .vector import VectorIndexType from ... import Computed from ... import exc from ... import schema as sa_schema @@ -784,14 +1042,18 @@ from ...sql import and_ from ...sql import bindparam from ...sql import compiler +from ...sql import elements from ...sql import expression from ...sql import func from ...sql import null from ...sql import or_ from ...sql import select +from ...sql import selectable as sa_selectable from ...sql import sqltypes from ...sql import util as sql_util from ...sql import visitors +from ...sql.base import NO_ARG +from ...sql.compiler import AggregateOrderByStyle from ...sql.visitors import InternalTraversal from ...types import BLOB from ...types import CHAR @@ -803,6 +1065,9 @@ from ...types import REAL from ...types import VARCHAR +if TYPE_CHECKING: + from ...sql.sqltypes import _JSON_VALUE + RESERVED_WORDS = set( "SHARE RAW DROP BETWEEN FROM DESC OPTION PRIOR LONG THEN " "DEFAULT ALTER IS INTO MINUS INTEGER NUMBER GRANT IDENTIFIED " @@ -825,6 +1090,9 @@ sqltypes.Interval: INTERVAL, sqltypes.DateTime: DATE, sqltypes.Date: _OracleDate, + sqltypes.JSON: JSON, + sqltypes.JSON.JSONIndexType: JSONIndexType, + sqltypes.JSON.JSONPathType: JSONPathType, } ischema_names = { @@ -850,6 +1118,9 @@ "BINARY_DOUBLE": BINARY_DOUBLE, "BINARY_FLOAT": BINARY_FLOAT, "ROWID": ROWID, + "BOOLEAN": BOOLEAN, + "VECTOR": VECTOR, + "JSON": JSON, } @@ -996,7 +1267,10 @@ def visit_big_integer(self, type_, **kw): return self.visit_NUMBER(type_, precision=19, **kw) def visit_boolean(self, type_, **kw): - return self.visit_SMALLINT(type_, **kw) + if self.dialect.supports_native_boolean: + return self.visit_BOOLEAN(type_, **kw) + else: + return self.visit_SMALLINT(type_, **kw) def visit_RAW(self, type_, **kw): if type_.length: @@ -1007,6 +1281,30 @@ def visit_RAW(self, type_, **kw): def visit_ROWID(self, type_, **kw): return "ROWID" + def visit_VECTOR(self, type_, **kw): + dim = type_.dim if type_.dim is not None else "*" + storage_format = ( + type_.storage_format.value + if type_.storage_format is not None + else "*" + ) + storage_type = ( + type_.storage_type.value if type_.storage_type is not None else "*" + ) + return f"VECTOR({dim},{storage_format},{storage_type})" + + def visit_JSON(self, type_: JSON, **kw: Any) -> str: + use_blob = ( + not self.dialect._supports_oracle_json + if getattr(type_, "use_blob", NO_ARG) is NO_ARG + else type_.use_blob + ) + + if use_blob: + return "BLOB" + else: + return "JSON" + class OracleCompiler(compiler.SQLCompiler): """Oracle compiler modifies the lexical structure of Select @@ -1035,6 +1333,9 @@ def visit_now_func(self, fn, **kw): def visit_char_length_func(self, fn, **kw): return "LENGTH" + self.function_argspec(fn, **kw) + def visit_pow_func(self, fn, **kw): + return f"POWER{self.function_argspec(fn)}" + def visit_match_op_binary(self, binary, operator, **kw): return "CONTAINS (%s, %s)" % ( self.process(binary.left), @@ -1047,6 +1348,23 @@ def visit_true(self, expr, **kw): def visit_false(self, expr, **kw): return "0" + def visit_cast(self, cast, **kwargs): + # Oracle requires VARCHAR2 to have a length in CAST expressions + # Adapt String types to VARCHAR2 with appropriate length + type_ = cast.typeclause.type + if isinstance(type_, sqltypes.String) and not isinstance( + type_, (sqltypes.Text, sqltypes.CLOB) + ): + adapted = VARCHAR2._adapt_string_for_cast(type_) + type_clause = self.dialect.type_compiler_instance.process(adapted) + else: + type_clause = cast.typeclause._compiler_dispatch(self, **kwargs) + + return "CAST(%s AS %s)" % ( + cast.clause._compiler_dispatch(self, **kwargs), + type_clause, + ) + def get_cte_preamble(self, recursive): return "WITH" @@ -1245,6 +1563,29 @@ def _get_limit_or_fetch(self, select): else: return select._fetch_clause + def fetch_clause( + self, + select, + fetch_clause=None, + require_offset=False, + use_literal_execute_for_simple_int=False, + **kw, + ): + text = super().fetch_clause( + select, + fetch_clause=fetch_clause, + require_offset=require_offset, + use_literal_execute_for_simple_int=( + use_literal_execute_for_simple_int + ), + **kw, + ) + + if select.dialect_options["oracle"]["fetch_approximate"]: + text = re.sub("FETCH FIRST", "FETCH APPROX FIRST", text) + + return text + def translate_select_structure(self, select_stmt, **kwargs): select = select_stmt @@ -1464,7 +1805,9 @@ def visit_regexp_replace_op_binary(self, binary, operator, **kw): ) def visit_aggregate_strings_func(self, fn, **kw): - return "LISTAGG%s" % self.function_argspec(fn, **kw) + return super().visit_aggregate_strings_func( + fn, use_function_name="LISTAGG", **kw + ) def _visit_bitwise(self, binary, fn_name, custom_right=None, **kw): left = self.process(binary.left, **kw) @@ -1491,8 +1834,101 @@ def visit_bitwise_lshift_op_binary(self, binary, operator, **kw): def visit_bitwise_not_op_unary_operator(self, element, operator, **kw): raise exc.CompileError("Cannot compile bitwise_not in oracle") + def _render_json_extract_from_binary(self, binary, operator, **kw): + literal_kw = kw.copy() + literal_kw["literal_binds"] = True + + left = self.process(binary.left, **kw) + right = self.process(binary.right, **literal_kw) + + if binary.type._type_affinity is sqltypes.Boolean: + # RETURNING clause doesn't handle true/false to 1/0 + # mapping, so use CASE expression for boolean + return ( + f"CASE JSON_VALUE({left}, {right})" + f" WHEN 'true' THEN 1" + f" WHEN 'false' THEN 0" + f" ELSE CAST(JSON_VALUE({left}, {right})" + f" AS NUMBER(1)) END" + ) + elif binary.type._type_affinity is sqltypes.Integer: + json_value_returning = "INTEGER" + elif binary.type._type_affinity in ( + sqltypes.Numeric, + sqltypes.Float, + ): + if isinstance(binary.type, sqltypes.Float): + json_value_returning = "FLOAT" + else: + json_value_returning = ( + f"NUMBER({binary.type.precision}, {binary.type.scale})" + ) + elif binary.type._type_affinity is sqltypes.String: + json_value_returning = "VARCHAR2(4000)" + else: + # binary.type._type_affinity is sqltypes.JSON + # or other + return f"JSON_QUERY({left}, {right})" + + return ( + f"JSON_VALUE({left}, {right}" + f" RETURNING {json_value_returning} ERROR ON ERROR)" + ) + + def visit_json_getitem_op_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: + return self._render_json_extract_from_binary(binary, operator, **kw) + + def visit_json_path_getitem_op_binary( + self, binary: elements.BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: + return self._render_json_extract_from_binary(binary, operator, **kw) + class OracleDDLCompiler(compiler.DDLCompiler): + + def _build_vector_index_config( + self, vector_index_config: VectorIndexConfig + ) -> str: + parts = [] + sql_param_name = { + "hnsw_neighbors": "neighbors", + "hnsw_efconstruction": "efconstruction", + "ivf_neighbor_partitions": "neighbor partitions", + "ivf_sample_per_partition": "sample_per_partition", + "ivf_min_vectors_per_partition": "min_vectors_per_partition", + } + if vector_index_config.index_type == VectorIndexType.HNSW: + parts.append("ORGANIZATION INMEMORY NEIGHBOR GRAPH") + elif vector_index_config.index_type == VectorIndexType.IVF: + parts.append("ORGANIZATION NEIGHBOR PARTITIONS") + if vector_index_config.distance is not None: + parts.append(f"DISTANCE {vector_index_config.distance.value}") + + if vector_index_config.accuracy is not None: + parts.append( + f"WITH TARGET ACCURACY {vector_index_config.accuracy}" + ) + + parameters_str = [f"type {vector_index_config.index_type.name}"] + prefix = vector_index_config.index_type.name.lower() + "_" + + for field in fields(vector_index_config): + if field.name.startswith(prefix): + key = sql_param_name.get(field.name) + value = getattr(vector_index_config, field.name) + if value is not None: + parameters_str.append(f"{key} {value}") + + parameters_str = ", ".join(parameters_str) + parts.append(f"PARAMETERS ({parameters_str})") + + if vector_index_config.parallel is not None: + parts.append(f"PARALLEL {vector_index_config.parallel}") + + return " ".join(parts) + def define_constraint_cascades(self, constraint): text = "" if constraint.ondelete is not None: @@ -1525,6 +1961,9 @@ def visit_create_index(self, create, **kw): text += "UNIQUE " if index.dialect_options["oracle"]["bitmap"]: text += "BITMAP " + vector_options = index.dialect_options["oracle"]["vector"] + if vector_options: + text += "VECTOR " text += "INDEX %s ON %s (%s)" % ( self._prepared_index_name(index, include_schema=True), preparer.format_table(index.table, use_schema=True), @@ -1542,6 +1981,11 @@ def visit_create_index(self, create, **kw): text += " COMPRESS %d" % ( index.dialect_options["oracle"]["compress"] ) + if vector_options: + if vector_options is True: + vector_options = VectorIndexConfig() + + text += " " + self._build_vector_index_config(vector_options) return text def post_create_table(self, table): @@ -1657,6 +2101,7 @@ class OracleDialect(default.DefaultDialect): cte_follows_insert = True returns_native_bytes = True + supports_native_boolean = True supports_sequences = True sequences_optional = False postfetch_lastrowid = False @@ -1673,6 +2118,10 @@ class OracleDialect(default.DefaultDialect): supports_empty_insert = False supports_identity_columns = True + _supports_oracle_json = True + + aggregate_order_by_style = AggregateOrderByStyle.WITHIN_GROUP + statement_compiler = OracleCompiler ddl_compiler = OracleDDLCompiler type_compiler_cls = OracleTypeCompiler @@ -1693,9 +2142,18 @@ class OracleDialect(default.DefaultDialect): "tablespace": None, }, ), - (sa_schema.Index, {"bitmap": False, "compress": False}), + ( + sa_schema.Index, + { + "bitmap": False, + "compress": False, + "vector": False, + }, + ), (sa_schema.Sequence, {"order": None}), (sa_schema.Identity, {"order": None, "on_null": None}), + (sa_selectable.Select, {"fetch_approximate": False}), + (sa_selectable.CompoundSelect, {"fetch_approximate": False}), ] @util.deprecated_params( @@ -1716,6 +2174,8 @@ def __init__( use_nchar_for_unicode=False, exclude_tablespaces=("SYSTEM", "SYSAUX"), enable_offset_fetch=True, + json_serializer: Callable[[_JSON_VALUE], str] | None = None, + json_deserializer: Callable[[str], _JSON_VALUE] | None = None, **kwargs, ): default.DefaultDialect.__init__(self, **kwargs) @@ -1726,6 +2186,8 @@ def __init__( self.enable_offset_fetch = self._supports_offset_fetch = ( enable_offset_fetch ) + self._json_serializer = json_serializer + self._json_deserializer = json_deserializer def initialize(self, connection): super().initialize(connection) @@ -1741,6 +2203,8 @@ def initialize(self, connection): self.colspecs.pop(sqltypes.Interval) self.use_ansi = False + self._supports_oracle_json = self.server_version_info >= (21,) + self.supports_native_boolean = self.server_version_info >= (23,) self.supports_identity_columns = self.server_version_info >= (12,) self._supports_offset_fetch = ( self.enable_offset_fetch and self.server_version_info >= (12,) @@ -2325,7 +2789,7 @@ def _table_options_query( and ObjectKind.TABLE in kind and ObjectKind.MATERIALIZED_VIEW not in kind ): - # cant use EXCEPT ALL / MINUS here because we don't have an + # can't use EXCEPT ALL / MINUS here because we don't have an # excludable row vs. the query above # outerjoin + where null works better on oracle 21 but 11 does # not like it at all. this is the next best thing @@ -2485,6 +2949,7 @@ def _column_query(self, owner): all_cols.c.column_name, all_cols.c.data_type, all_cols.c.char_length, + all_cols.c.data_length, all_cols.c.data_precision, all_cols.c.data_scale, all_cols.c.nullable, @@ -2603,6 +3068,9 @@ def maybe_int(value): elif coltype in ("VARCHAR2", "NVARCHAR2", "CHAR", "NCHAR"): char_length = maybe_int(row_dict["char_length"]) coltype = self.ischema_names.get(coltype)(char_length) + elif coltype == "RAW": + data_length = maybe_int(row_dict["data_length"]) + coltype = RAW(data_length) elif "WITH TIME ZONE" in coltype: coltype = TIMESTAMP(timezone=True) elif "WITH LOCAL TIME ZONE" in coltype: diff --git a/lib/sqlalchemy/dialects/oracle/cx_oracle.py b/lib/sqlalchemy/dialects/oracle/cx_oracle.py index a0ebea44028..6a1c86cb893 100644 --- a/lib/sqlalchemy/dialects/oracle/cx_oracle.py +++ b/lib/sqlalchemy/dialects/oracle/cx_oracle.py @@ -1,5 +1,5 @@ # dialects/oracle/cx_oracle.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -117,12 +117,6 @@ "oracle+cx_oracle://user:pass@dsn?encoding=UTF-8&nencoding=UTF-8&mode=SYSDBA&events=true" ) -.. versionchanged:: 1.3 the cx_Oracle dialect now accepts all argument names - within the URL string itself, to be passed to the cx_Oracle DBAPI. As - was the case earlier but not correctly documented, the - :paramref:`_sa.create_engine.connect_args` parameter also accepts all - cx_Oracle DBAPI connect arguments. - To pass arguments directly to ``.connect()`` without using the query string, use the :paramref:`_sa.create_engine.connect_args` dictionary. Any cx_Oracle parameter value and/or constant may be passed, such as:: @@ -323,12 +317,6 @@ def creator(): the SQLAlchemy dialect to use NCHAR/NCLOB for the :class:`.Unicode` / :class:`.UnicodeText` datatypes instead of VARCHAR/CLOB. -.. versionchanged:: 1.3 The :class:`.Unicode` and :class:`.UnicodeText` - datatypes now correspond to the ``VARCHAR2`` and ``CLOB`` Oracle Database - datatypes unless the ``use_nchar_for_unicode=True`` is passed to the dialect - when :func:`_sa.create_engine` is called. - - .. _cx_oracle_unicode_encoding_errors: Encoding Errors @@ -343,9 +331,6 @@ def creator(): ``Cursor.var()``, as well as SQLAlchemy's own decoding function, as the cx_Oracle dialect makes use of both under different circumstances. -.. versionadded:: 1.3.11 - - .. _cx_oracle_setinputsizes: Fine grained control over cx_Oracle data binding performance with setinputsizes @@ -372,9 +357,6 @@ def creator(): well as to fully control how ``setinputsizes()`` is used on a per-statement basis. -.. versionadded:: 1.2.9 Added :meth:`.DialectEvents.setinputsizes` - - Example 1 - logging all setinputsizes calls ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -484,14 +466,11 @@ def _remove_clob(inputsizes, cursor, statement, parameters, context): SQL statements that are not otherwise associated with a :class:`.Numeric` SQLAlchemy type (or a subclass of such). -.. versionchanged:: 1.2 The numeric handling system for cx_Oracle has been - reworked to take advantage of newer cx_Oracle features as well - as better integration of outputtypehandlers. - """ # noqa from __future__ import annotations import decimal +import json import random import re @@ -499,6 +478,7 @@ def _remove_clob(inputsizes, cursor, statement, parameters, context): from .base import OracleCompiler from .base import OracleDialect from .base import OracleExecutionContext +from .json import JSON from .types import _OracleDateLiteralRender from ... import exc from ... import util @@ -507,11 +487,71 @@ def _remove_clob(inputsizes, cursor, statement, parameters, context): from ...engine import processors from ...sql import sqltypes from ...sql._typing import is_sql_compiler +from ...sql.base import NO_ARG +from ...sql.sqltypes import Boolean # source: # https://github.com/oracle/python-cx_Oracle/issues/596#issuecomment-999243649 _CX_ORACLE_MAGIC_LOB_SIZE = 131072 +# largest JSON we can deserialize if we are not using +# DB_TYPE_JSON +_CX_ORACLE_MAX_JSON_CONVERTED = 32767 + + +class _OracleJson(JSON): + def get_dbapi_type(self, dbapi): + return dbapi.DB_TYPE_JSON + + def _should_use_blob(self, dialect): + use_blob = ( + not dialect._supports_oracle_json + if self.use_blob is NO_ARG + else self.use_blob + ) + + return use_blob + + def bind_processor(self, dialect): + + if self._should_use_blob(dialect): + + DBAPIBinary = dialect.dbapi.Binary + + def string_process(value): + if value is not None: + # utf-8 is standard for oracledb + # https://python-oracledb.readthedocs.io/en/latest/user_guide/globalization.html#setting-the-client-character-set # noqa: E501 + return DBAPIBinary(value.encode("utf-8")) + else: + return None + + else: + string_process = None + + json_serializer = dialect._json_serializer or json.dumps + + return self._make_bind_processor(string_process, json_serializer) + + def result_processor(self, dialect, coltype): + if self._should_use_blob(dialect): + # for plain BLOB, use traditional binary decode + json.loads() + string_process = self._str_impl.result_processor(dialect, coltype) + json_deserializer = dialect._json_deserializer or json.loads + + def process(value): + if value is None: + return None + if string_process: + value = string_process(value) + return json_deserializer(value) + + return process + + else: + # for JSON, json decoder is set as an outputtypehandler + return None + class _OracleInteger(sqltypes.Integer): def get_dbapi_type(self, dbapi): @@ -903,7 +943,17 @@ def _generate_out_parameter_vars(self): decimal.Decimal, arraysize=len_params, ) - + elif isinstance(type_impl, Boolean): + if self.dialect.supports_native_boolean: + out_parameters[name] = self.cursor.var( + cx_Oracle.BOOLEAN, arraysize=len_params + ) + else: + out_parameters[name] = self.cursor.var( + cx_Oracle.NUMBER, + arraysize=len_params, + outconverter=bool, + ) else: out_parameters[name] = self.cursor.var( dbtype, arraysize=len_params @@ -1052,6 +1102,10 @@ class OracleDialect_cx_oracle(OracleDialect): update_executemany_returning = True delete_executemany_returning = True + supports_native_json_serialization = False + supports_native_json_deserialization = False + dialect_injects_custom_json_deserializer = True + bind_typing = interfaces.BindTyping.SETINPUTSIZES driver = "cx_oracle" @@ -1064,6 +1118,7 @@ class OracleDialect_cx_oracle(OracleDialect): sqltypes.Float: _OracleFloat, oracle.BINARY_FLOAT: _OracleBINARY_FLOAT, oracle.BINARY_DOUBLE: _OracleBINARY_DOUBLE, + sqltypes.JSON: _OracleJson, sqltypes.Integer: _OracleInteger, oracle.NUMBER: _OracleNUMBER, sqltypes.Date: _CXOracleDate, @@ -1089,28 +1144,14 @@ class OracleDialect_cx_oracle(OracleDialect): execute_sequence_format = list - _cx_oracle_threaded = None - _cursor_var_unicode_kwargs = util.immutabledict() - @util.deprecated_params( - threaded=( - "1.3", - "The 'threaded' parameter to the cx_oracle/oracledb dialect " - "is deprecated as a dialect-level argument, and will be removed " - "in a future release. As of version 1.3, it defaults to False " - "rather than True. The 'threaded' option can be passed to " - "cx_Oracle directly in the URL query string passed to " - ":func:`_sa.create_engine`.", - ) - ) def __init__( self, auto_convert_lobs=True, coerce_to_decimal=True, arraysize=None, encoding_errors=None, - threaded=None, **kwargs, ): OracleDialect.__init__(self, **kwargs) @@ -1120,8 +1161,6 @@ def __init__( self._cursor_var_unicode_kwargs = { "encodingErrors": encoding_errors } - if threaded is not None: - self._cx_oracle_threaded = threaded self.auto_convert_lobs = auto_convert_lobs self.coerce_to_decimal = coerce_to_decimal if self._use_nchar_for_unicode: @@ -1150,6 +1189,9 @@ def __init__( dbapi_module.FIXED_NCHAR, dbapi_module.FIXED_CHAR, dbapi_module.TIMESTAMP, + # we dont make use of Oracle's JSON serialization; does not + # handle "none as null" + # dbapi_module.DB_TYPE_JSON, int, # _OracleInteger, # _OracleBINARY_FLOAT, _OracleBINARY_DOUBLE, dbapi_module.NATIVE_FLOAT, @@ -1242,6 +1284,9 @@ def set_isolation_level(self, dbapi_connection, level): with dbapi_connection.cursor() as cursor: cursor.execute(f"ALTER SESSION SET ISOLATION_LEVEL={level}") + def detect_autocommit_setting(self, dbapi_conn) -> bool: + return bool(dbapi_conn.autocommit) + def _detect_decimal_char(self, connection): # we have the option to change this setting upon connect, # or just look at what it is upon connect and convert. @@ -1381,6 +1426,16 @@ def output_type_handler( _CX_ORACLE_MAGIC_LOB_SIZE, cursor.arraysize, ) + elif ( + default_type is cx_Oracle.DB_TYPE_JSON + and dialect._json_deserializer is not None + ): + return cursor.var( + cx_Oracle.DB_TYPE_VARCHAR, + _CX_ORACLE_MAX_JSON_CONVERTED, + cursor.arraysize, + outconverter=dialect._json_deserializer, + ) return output_type_handler @@ -1395,17 +1450,6 @@ def on_connect(conn): def create_connect_args(self, url): opts = dict(url.query) - for opt in ("use_ansi", "auto_convert_lobs"): - if opt in opts: - util.warn_deprecated( - f"{self.driver} dialect option {opt!r} should only be " - "passed to create_engine directly, not within the URL " - "string", - version="1.3", - ) - util.coerce_kw_type(opts, opt, bool) - setattr(self, opt, opts.pop(opt)) - database = url.database service_name = opts.pop("service_name", None) if database or service_name: @@ -1438,9 +1482,6 @@ def create_connect_args(self, url): if url.username is not None: opts["user"] = url.username - if self._cx_oracle_threaded is not None: - opts.setdefault("threaded", self._cx_oracle_threaded) - def convert_cx_oracle_constant(value): if isinstance(value, str): try: diff --git a/lib/sqlalchemy/dialects/oracle/dictionary.py b/lib/sqlalchemy/dialects/oracle/dictionary.py index f785a66ef71..5f36041b437 100644 --- a/lib/sqlalchemy/dialects/oracle/dictionary.py +++ b/lib/sqlalchemy/dialects/oracle/dictionary.py @@ -1,5 +1,5 @@ # dialects/oracle/dictionary.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/dialects/oracle/json.py b/lib/sqlalchemy/dialects/oracle/json.py new file mode 100644 index 00000000000..51e9ba872ea --- /dev/null +++ b/lib/sqlalchemy/dialects/oracle/json.py @@ -0,0 +1,158 @@ +# dialects/oracle/json.py +# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# +# +# This module is part of SQLAlchemy and is released under +# the MIT License: https://www.opensource.org/licenses/mit-license.php +# mypy: ignore-errors + +from __future__ import annotations + +from typing import Any +from typing import TYPE_CHECKING +from typing import TypeVar + +from ... import types as sqltypes +from ...sql.base import _NoArg +from ...sql.base import NO_ARG +from ...sql.sqltypes import _T_JSON + + +if TYPE_CHECKING: + from ...engine.interfaces import Dialect + from ...sql.type_api import _BindProcessorType + from ...sql.type_api import _LiteralProcessorType + +_T = TypeVar("_T", bound=Any) + + +class JSON(sqltypes.JSON[_T_JSON]): + """Oracle JSON type. + + .. versionadded:: 2.1 + + Oracle Database supports JSON storage and querying for character and BLOB + datatypes in Oracle 12c, and supports a dedicated JSON data type as of + Oracle 21c. SQLAlchemy supports both of these scenarios when using the + oracledb DBAPI. This type is used implicitly whenever the base + :class:`_types.JSON` datatype is used against an Oracle backend, or may be + constructed directly for access to Oracle-specific parameters such as + :paramref:`_oracle.JSON.use_blob`. + + Index operations are adapted to render using the ``JSON_QUERY`` and + ``JSON_VALUE`` functions at the database level. + + **Platform Support** - When using Oracle Database versions prior to 21c, + BLOB is used as the storage format. In 21c or later, the native JSON + datatype is used. This can be overridden using the + :paramref:`_oracle.JSON.use_blob` parameter. + + **Serialization / Deserialization** - JSON serialization of bound + parameters uses Python ``json.dumps()`` by default rather than oracledb's + native serializer, in order to support the + :paramref:`_sqltypes.JSON.none_as_null` feature. The default serializer + does **not** accept Python ``Decimal`` objects; to use a custom serializer, + pass :paramref:`_sa.create_engine.json_serializer` to + :func:`_sa.create_engine`. + + When using the native JSON datatype (21c+), deserialization uses oracledb's + native deserializer by default, which is required for JSON values larger + than 32767 bytes. However, this deserializer returns all numeric values as + ``Decimal`` since Oracle Database stores JSON numbers using its internal + NUMBER type. To receive standard Python numeric types, pass + ``json_deserializer=json.loads`` via + :paramref:`_sa.create_engine.json_deserializer`; note that this limits + maximum JSON value size to 32767 bytes. When using BLOB storage, + SQLAlchemy deserializes using ``json.loads()`` directly rather than the + oracledb deserializer. + + **CHECK Constraint with BLOB** - When using BLOB storage, either on Oracle + Database versions prior to 21c or via the :paramref:`_oracle.JSON.use_blob` + parameter, the oracledb driver documentation recommends adding a + `` IS JSON`` check constraint to indicate to the driver that the + column stores JSON data. This constraint is **not** automatically + generated by :class:`_oracle.JSON` and is not required by SQLAlchemy's + implementation in order to read JSON data from the column. If desired, it + can be added explicitly using :class:`_schema.CheckConstraint`. + + .. seealso:: + + :class:`_types.JSON` - main documentation for the generic + cross-platform JSON datatype. + + """ + + use_blob: bool | _NoArg + + def __init__( + self, none_as_null: bool = False, use_blob: bool | _NoArg = NO_ARG + ): + """Construct a :class:`_oracle.JSON` type. + + :param none_as_null=False: if True, persist the value ``None`` as a SQL + NULL value, not the JSON encoding of ``null``. See the notes at + :paramref:`_sqltypes.JSON.none_as_null` for complete background on + this option. + + :param use_blob: A boolean parameter indicating if the type should be + rendered in DDL using BLOB instead of JSON. Normally, JSON or BLOB + is chosen automatically based on the version of Oracle in use + (21c or greater for JSON). If the parameter is left at its default + value of the ``NO_ARG`` constant, this automatic selection is used. + However when ``True``, the BLOB datatype will be used unconditionally, + and if ``False``, JSON will be used unconditionally (including on + backends older than 21c, which will raise an error by the server. + This may be used to assert that only JSON-supporting backends + should be used). + + """ + + super().__init__(none_as_null=none_as_null) + self.use_blob = use_blob + + +class _FormatTypeMixin: + def _format_value(self, value: Any) -> str: + raise NotImplementedError() + + def bind_processor(self, dialect: Dialect) -> _BindProcessorType[Any]: + super_proc = self.string_bind_processor(dialect) # type: ignore[attr-defined] # noqa: E501 + + def process(value: Any) -> Any: + value = self._format_value(value) + if super_proc: + value = super_proc(value) + return value + + return process + + def literal_processor( + self, dialect: Dialect + ) -> _LiteralProcessorType[Any]: + super_proc = self.string_literal_processor(dialect) # type: ignore[attr-defined] # noqa: E501 + + def process(value: Any) -> str: + value = self._format_value(value) + if super_proc: + value = super_proc(value) + return value # type: ignore[no-any-return] + + return process + + +class JSONIndexType(_FormatTypeMixin, sqltypes.JSON.JSONIndexType): + def _format_value(self, value: Any) -> str: + if isinstance(value, int): + return f"$[{value}]" + else: + return f'$."{value}"' + + +class JSONPathType(_FormatTypeMixin, sqltypes.JSON.JSONPathType): + def _format_value(self, value: Any) -> str: + return "$%s" % ( + "".join( + f"[{elem}]" if isinstance(elem, int) else f'."{elem}"' + for elem in value + ) + ) diff --git a/lib/sqlalchemy/dialects/oracle/oracledb.py b/lib/sqlalchemy/dialects/oracle/oracledb.py index 8105608837f..5ba4e0ae6cf 100644 --- a/lib/sqlalchemy/dialects/oracle/oracledb.py +++ b/lib/sqlalchemy/dialects/oracle/oracledb.py @@ -1,5 +1,5 @@ # dialects/oracle/oracledb.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -112,7 +112,7 @@ For example to use an `Easy Connect string `_ with a timeout to prevent connection establishment from hanging if the network -transport to the database cannot be establishd in 30 seconds, and also setting +transport to the database cannot be established in 30 seconds, and also setting a keep-alive time of 60 seconds to stop idle network connections from being terminated by a firewall:: @@ -236,12 +236,8 @@ is where the Oracle Autonomous Database wallet zip file was extracted. Note this directory should be protected. -Connection Pooling ------------------- - -Applications with multiple concurrent users should use connection pooling. A -minimal sized connection pool is also beneficial for long-running, single-user -applications that do not frequently use a connection. +Using python-oracledb Connection Pooling +---------------------------------------- The python-oracledb driver provides its own connection pool implementation that may be used in place of SQLAlchemy's pooling functionality. The driver pool @@ -416,12 +412,6 @@ def creator(): the SQLAlchemy dialect to use NCHAR/NCLOB for the :class:`.Unicode` / :class:`.UnicodeText` datatypes instead of VARCHAR/CLOB. -.. versionchanged:: 1.3 The :class:`.Unicode` and :class:`.UnicodeText` - datatypes now correspond to the ``VARCHAR2`` and ``CLOB`` Oracle Database - datatypes unless the ``use_nchar_for_unicode=True`` is passed to the dialect - when :func:`_sa.create_engine` is called. - - .. _oracledb_unicode_encoding_errors: Encoding Errors @@ -436,9 +426,6 @@ def creator(): ``Cursor.var()``, as well as SQLAlchemy's own decoding function, as the python-oracledb dialect makes use of both under different circumstances. -.. versionadded:: 1.3.11 - - .. _oracledb_setinputsizes: Fine grained control over python-oracledb data binding with setinputsizes @@ -465,9 +452,6 @@ def creator(): well as to fully control how ``setinputsizes()`` is used on a per-statement basis. -.. versionadded:: 1.2.9 Added :meth:`.DialectEvents.setinputsizes` - - Example 1 - logging all setinputsizes calls ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -585,12 +569,23 @@ def _remove_clob(inputsizes, cursor, statement, parameters, context): SQL statements that are not otherwise associated with a :class:`.Numeric` SQLAlchemy type (or a subclass of such). -.. versionchanged:: 1.2 The numeric handling system for the oracle dialects has - been reworked to take advantage of newer driver features as well as better - integration of outputtypehandlers. - .. versionadded:: 2.0.0 added support for the python-oracledb driver. +.. _oracledb_json: + +JSON Support +------------ + +Oracle Database supports a native JSON datatype as of version 21c, as well as +support for JSON functions on character and BLOB columns as of version 12c. The +SQLAlchemy :class:`_sqltypes.JSON` datatype may be used with the oracledb +backend in the same way it works with any other backend, with some slight +behavioral changes particularly when using the native JSON datatype. See +:class:`_oracle.JSON` for platform-specific notes. + +.. versionadded:: 2.1 added JSON support for the Oracle backend. + + """ # noqa from __future__ import annotations @@ -603,6 +598,7 @@ def _remove_clob(inputsizes, cursor, statement, parameters, context): from ... import exc from ...connectors.asyncio import AsyncAdapt_dbapi_connection from ...connectors.asyncio import AsyncAdapt_dbapi_cursor +from ...connectors.asyncio import AsyncAdapt_dbapi_module from ...connectors.asyncio import AsyncAdapt_dbapi_ss_cursor from ...engine import default from ...util import await_ @@ -732,6 +728,8 @@ def _check_max_identifier_length(self, connection): class AsyncAdapt_oracledb_cursor(AsyncAdapt_dbapi_cursor): _cursor: AsyncCursor + _awaitable_cursor_close: bool = False + __slots__ = () @property @@ -745,10 +743,6 @@ def outputtypehandler(self, value): def var(self, *args, **kwargs): return self._cursor.var(*args, **kwargs) - def close(self): - self._rows.clear() - self._cursor.close() - def setinputsizes(self, *args: Any, **kwargs: Any) -> Any: return self._cursor.setinputsizes(*args, **kwargs) @@ -856,8 +850,9 @@ def tpc_rollback(self, *args: Any, **kwargs: Any) -> Any: return await_(self._connection.tpc_rollback(*args, **kwargs)) -class OracledbAdaptDBAPI: +class OracledbAdaptDBAPI(AsyncAdapt_dbapi_module): def __init__(self, oracledb) -> None: + super().__init__(oracledb) self.oracledb = oracledb for k, v in self.oracledb.__dict__.items(): @@ -866,8 +861,8 @@ def __init__(self, oracledb) -> None: def connect(self, *arg, **kw): creator_fn = kw.pop("async_creator_fn", self.oracledb.connect_async) - return AsyncAdapt_oracledb_connection( - self, await_(creator_fn(*arg, **kw)) + return await_( + AsyncAdapt_oracledb_connection.create(self, creator_fn(*arg, **kw)) ) diff --git a/lib/sqlalchemy/dialects/oracle/provision.py b/lib/sqlalchemy/dialects/oracle/provision.py index 3587de9d011..db92c5da255 100644 --- a/lib/sqlalchemy/dialects/oracle/provision.py +++ b/lib/sqlalchemy/dialects/oracle/provision.py @@ -1,11 +1,13 @@ # dialects/oracle/provision.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php # mypy: ignore-errors +import time + from ... import create_engine from ... import exc from ... import inspect @@ -16,13 +18,49 @@ from ...testing.provision import drop_all_schema_objects_pre_tables from ...testing.provision import drop_db from ...testing.provision import follower_url_from_main +from ...testing.provision import generate_driver_url from ...testing.provision import log from ...testing.provision import post_configure_engine +from ...testing.provision import post_configure_testing_engine from ...testing.provision import run_reap_dbs from ...testing.provision import set_default_schema_on_connection from ...testing.provision import stop_test_class_outside_fixtures from ...testing.provision import temp_table_keyword_args from ...testing.provision import update_db_opts +from ...testing.warnings import warn_test_suite + + +@generate_driver_url.for_db("oracle") +def _oracle_generate_driver_url(url, driver, query_str): + + backend = url.get_backend_name() + + new_url = url.set( + drivername="%s+%s" % (backend, driver), + ) + + # use oracledb's retry feature, which is essential for oracle 23c + # which otherwise frequently rejects connections under load + # for cx_oracle we have a connect event instead + if driver in ("oracledb", "oracledb_async"): + # oracledb is even nice enough to convert from string to int + # for these opts, apparently + new_url = new_url.update_query_pairs( + [("retry_count", "5"), ("retry_delay", "2")] + ) + else: + # remove these params for cx_oracle if we received an + # already-modified URL + new_url = new_url.difference_update_query( + ["retry_count", "retry_delay"] + ) + + try: + new_url.get_dialect() + except exc.NoSuchModuleError: + return None + else: + return new_url @create_db.for_db("oracle") @@ -103,20 +141,6 @@ def _ora_stop_test_class_outside_fixtures(config, db, cls): except exc.DatabaseError as err: log.warning("purge recyclebin command failed: %s", err) - # clear statement cache on all connections that were used - # https://github.com/oracle/python-cx_Oracle/issues/519 - - for cx_oracle_conn in _all_conns: - try: - sc = cx_oracle_conn.stmtcachesize - except db.dialect.dbapi.InterfaceError: - # connection closed - pass - else: - cx_oracle_conn.stmtcachesize = 0 - cx_oracle_conn.stmtcachesize = sc - _all_conns.clear() - def _purge_recyclebin(eng, schema=None): with eng.begin() as conn: @@ -134,24 +158,68 @@ def _purge_recyclebin(eng, schema=None): conn.exec_driver_sql(f'purge {type_} {owner}."{object_name}"') -_all_conns = set() +def _connect_with_retry(dialect, conn_rec, cargs, cparams): + assert dialect.driver == "cx_oracle" + + def _is_couldnt_connect(err): + return "DPY-6005" in str(err) or "ORA-12516" in str(err) + + err_ = None + for _ in range(5): + try: + return dialect.loaded_dbapi.connect(*cargs, **cparams) + except ( + dialect.loaded_dbapi.DatabaseError, + dialect.loaded_dbapi.OperationalError, + ) as err: + err_ = err + if _is_couldnt_connect(err): + warn_test_suite("Oracle database reconnecting...") + time.sleep(2) + continue + else: + raise + if err_ is not None: + raise Exception("connect failed after five attempts") from err_ + + +@post_configure_testing_engine.for_db("oracle") +def _oracle_post_configure_testing_engine(url, engine, options, scope): + from ... import event + + if engine.dialect.driver == "cx_oracle": + event.listen(engine, "do_connect", _connect_with_retry) @post_configure_engine.for_db("oracle") def _oracle_post_configure_engine(url, engine, follower_ident): - from sqlalchemy import event - @event.listens_for(engine, "checkout") - def checkout(dbapi_con, con_record, con_proxy): - _all_conns.add(dbapi_con) + from ... import event @event.listens_for(engine, "checkin") def checkin(dbapi_connection, connection_record): - # work around cx_Oracle issue: + # this was meant to work around this issue: # https://github.com/oracle/python-cx_Oracle/issues/530 # invalidate oracle connections that had 2pc set up - if "cx_oracle_xid" in connection_record.info: - connection_record.invalidate() + # however things are too complex with some of the 2pc tests, + # so just block cx_oracle from being used in 2pc tests (use oracledb + # instead) + # if "cx_oracle_xid" in connection_record.info: + # connection_record.invalidate() + + # clear statement cache on all connections that were used + # https://github.com/oracle/python-cx_Oracle/issues/519 + # TODO: oracledb claims to have this feature built in somehow, + # see if that's in use and/or if it needs to be enabled + # (or if this doesn't even apply to the newer oracle's we're using) + try: + sc = dbapi_connection.stmtcachesize + except: + # connection closed + pass + else: + dbapi_connection.stmtcachesize = 0 + dbapi_connection.stmtcachesize = sc @run_reap_dbs.for_db("oracle") diff --git a/lib/sqlalchemy/dialects/oracle/types.py b/lib/sqlalchemy/dialects/oracle/types.py index 06aeaace2f5..1ddd8f7bf6d 100644 --- a/lib/sqlalchemy/dialects/oracle/types.py +++ b/lib/sqlalchemy/dialects/oracle/types.py @@ -1,5 +1,5 @@ # dialects/oracle/types.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING from ... import exc +from ...sql import operators from ...sql import sqltypes from ...types import NVARCHAR from ...types import VARCHAR @@ -22,6 +23,9 @@ from ...sql.type_api import _LiteralProcessorType +BOOLEAN = sqltypes.BOOLEAN + + class RAW(sqltypes._Binary): __visit_name__ = "RAW" @@ -36,6 +40,23 @@ class NCLOB(sqltypes.Text): class VARCHAR2(VARCHAR): __visit_name__ = "VARCHAR2" + @classmethod + def _adapt_string_for_cast(cls, type_: sqltypes.String) -> "VARCHAR2": + """Adapt a String type for use in CAST expressions. + + Oracle requires a length for VARCHAR2 in CAST expressions. + If no length is specified, we default to 4000 (max for VARCHAR2). + """ + type_ = sqltypes.to_instance(type_) + if isinstance(type_, VARCHAR2): + return type_ + elif isinstance(type_, VARCHAR): + return VARCHAR2( + length=type_.length or 4000, collation=type_.collation + ) + else: + return VARCHAR2(length=type_.length or 4000) + NVARCHAR2 = NVARCHAR @@ -309,8 +330,38 @@ class ROWID(sqltypes.TypeEngine): """ __visit_name__ = "ROWID" + operator_classes = operators.OperatorClass.ANY class _OracleBoolean(sqltypes.Boolean): + def get_dbapi_type(self, dbapi): + # this can probably be dbapi.BOOLEAN (including for older versions), + # however sticking with NUMBER to avoid any surprises with older + # versions or non-bool values return dbapi.NUMBER + + def result_processor(self, dialect, coltype): + # we dont need a result processor even if we are not native + # boolean because we use an outputtypehandler + return None + + def _cx_oracle_outputtypehandler(self, dialect): + cx_Oracle = dialect.dbapi + + def handler(cursor, name, default_type, size, precision, scale): + # if native boolean no handler needed + if default_type is cx_Oracle.BOOLEAN: + return None + + # OTOH if we are getting a number back and we are either + # native boolean pulling from a smallint, or non native + # boolean pulling from a smallint that's emulated, use bool + return cursor.var( + cx_Oracle.NUMBER, + 255, + arraysize=cursor.arraysize, + outconverter=bool, + ) + + return handler diff --git a/lib/sqlalchemy/dialects/oracle/vector.py b/lib/sqlalchemy/dialects/oracle/vector.py new file mode 100644 index 00000000000..66e439c7787 --- /dev/null +++ b/lib/sqlalchemy/dialects/oracle/vector.py @@ -0,0 +1,368 @@ +# dialects/oracle/vector.py +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors +# +# +# This module is part of SQLAlchemy and is released under +# the MIT License: https://www.opensource.org/licenses/mit-license.php +# mypy: ignore-errors + + +from __future__ import annotations + +import array +from dataclasses import dataclass +from enum import Enum +from typing import Optional +from typing import Union + +from ... import types +from ...sql.operators import OperatorClass +from ...types import Float + + +class VectorIndexType(Enum): + """Enum representing different types of VECTOR index structures. + + See :ref:`oracle_vector_datatype` for background. + + .. versionadded:: 2.0.41 + + """ + + HNSW = "HNSW" + """ + The HNSW (Hierarchical Navigable Small World) index type. + """ + IVF = "IVF" + """ + The IVF (Inverted File Index) index type + """ + + +class VectorDistanceType(Enum): + """Enum representing different types of vector distance metrics. + + See :ref:`oracle_vector_datatype` for background. + + .. versionadded:: 2.0.41 + + """ + + EUCLIDEAN = "EUCLIDEAN" + """Euclidean distance (L2 norm). + + Measures the straight-line distance between two vectors in space. + """ + DOT = "DOT" + """Dot product similarity. + + Measures the algebraic similarity between two vectors. + """ + COSINE = "COSINE" + """Cosine similarity. + + Measures the cosine of the angle between two vectors. + """ + MANHATTAN = "MANHATTAN" + """Manhattan distance (L1 norm). + + Calculates the sum of absolute differences across dimensions. + """ + + +class VectorStorageFormat(Enum): + """Enum representing the data format used to store vector components. + + See :ref:`oracle_vector_datatype` for background. + + .. versionadded:: 2.0.41 + + """ + + INT8 = "INT8" + """ + 8-bit integer format. + """ + BINARY = "BINARY" + """ + Binary format. + """ + FLOAT32 = "FLOAT32" + """ + 32-bit floating-point format. + """ + FLOAT64 = "FLOAT64" + """ + 64-bit floating-point format. + """ + + +class VectorStorageType(Enum): + """Enum representing the vector type, + + See :ref:`oracle_vector_datatype` for background. + + .. versionadded:: 2.0.43 + + """ + + SPARSE = "SPARSE" + """ + A Sparse vector is a vector which has zero value for + most of its dimensions. + """ + DENSE = "DENSE" + """ + A Dense vector is a vector where most, if not all, elements + hold meaningful values. + """ + + +@dataclass +class VectorIndexConfig: + """Define the configuration for Oracle VECTOR Index. + + See :ref:`oracle_vector_datatype` for background. + + .. versionadded:: 2.0.41 + + :param index_type: Enum value from :class:`.VectorIndexType` + Specifies the indexing method. For HNSW, this must be + :attr:`.VectorIndexType.HNSW`. + + :param distance: Enum value from :class:`.VectorDistanceType` + specifies the metric for calculating distance between VECTORS. + + :param accuracy: integer. Should be in the range 0 to 100 + Specifies the accuracy of the nearest neighbor search during + query execution. + + :param parallel: integer. Specifies degree of parallelism. + + :param hnsw_neighbors: integer. Should be in the range 0 to + 2048. Specifies the number of nearest neighbors considered + during the search. The attribute :attr:`.VectorIndexConfig.hnsw_neighbors` + is HNSW index specific. + + :param hnsw_efconstruction: integer. Should be in the range 0 + to 65535. Controls the trade-off between indexing speed and + recall quality during index construction. The attribute + :attr:`.VectorIndexConfig.hnsw_efconstruction` is HNSW index + specific. + + :param ivf_neighbor_partitions: integer. Should be in the range + 0 to 10,000,000. Specifies the number of partitions used to + divide the dataset. The attribute + :attr:`.VectorIndexConfig.ivf_neighbor_partitions` is IVF index + specific. + + :param ivf_sample_per_partition: integer. Should be between 1 + and ``num_vectors / neighbor partitions``. Specifies the + number of samples used per partition. The attribute + :attr:`.VectorIndexConfig.ivf_sample_per_partition` is IVF index + specific. + + :param ivf_min_vectors_per_partition: integer. From 0 (no trimming) + to the total number of vectors (results in 1 partition). Specifies + the minimum number of vectors per partition. The attribute + :attr:`.VectorIndexConfig.ivf_min_vectors_per_partition` + is IVF index specific. + + """ + + index_type: VectorIndexType = VectorIndexType.HNSW + distance: Optional[VectorDistanceType] = None + accuracy: Optional[int] = None + hnsw_neighbors: Optional[int] = None + hnsw_efconstruction: Optional[int] = None + ivf_neighbor_partitions: Optional[int] = None + ivf_sample_per_partition: Optional[int] = None + ivf_min_vectors_per_partition: Optional[int] = None + parallel: Optional[int] = None + + def __post_init__(self): + self.index_type = VectorIndexType(self.index_type) + for field in [ + "hnsw_neighbors", + "hnsw_efconstruction", + "ivf_neighbor_partitions", + "ivf_sample_per_partition", + "ivf_min_vectors_per_partition", + "parallel", + "accuracy", + ]: + value = getattr(self, field) + if value is not None and not isinstance(value, int): + raise TypeError( + f"{field} must be an integer if" + f"provided, got {type(value).__name__}" + ) + + +class SparseVector: + """ + Lightweight SQLAlchemy-side version of SparseVector. + This mimics oracledb.SparseVector. + + .. versionadded:: 2.0.43 + + """ + + def __init__( + self, + num_dimensions: int, + indices: Union[list, array.array], + values: Union[list, array.array], + ): + if not isinstance(indices, array.array) or indices.typecode != "I": + indices = array.array("I", indices) + if not isinstance(values, array.array): + values = array.array("d", values) + if len(indices) != len(values): + raise TypeError("indices and values must be of the same length!") + + self.num_dimensions = num_dimensions + self.indices = indices + self.values = values + + def __str__(self): + return ( + f"SparseVector(num_dimensions={self.num_dimensions}, " + f"size={len(self.indices)}, typecode={self.values.typecode})" + ) + + +class VECTOR(types.TypeEngine): + """Oracle VECTOR datatype. + + For complete background on using this type, see + :ref:`oracle_vector_datatype`. + + .. versionadded:: 2.0.41 + + """ + + cache_ok = True + + operator_classes = OperatorClass.BASE | OperatorClass.MATH + + __visit_name__ = "VECTOR" + + _typecode_map = { + VectorStorageFormat.INT8: "b", # Signed int + VectorStorageFormat.BINARY: "B", # Unsigned int + VectorStorageFormat.FLOAT32: "f", # Float + VectorStorageFormat.FLOAT64: "d", # Double + } + + def __init__(self, dim=None, storage_format=None, storage_type=None): + """Construct a VECTOR. + + :param dim: integer. The dimension of the VECTOR datatype. This + should be an integer value. + + :param storage_format: VectorStorageFormat. The VECTOR storage + type format. This should be Enum values form + :class:`.VectorStorageFormat` INT8, BINARY, FLOAT32, or FLOAT64. + + :param storage_type: VectorStorageType. The Vector storage type. This + should be Enum values from :class:`.VectorStorageType` SPARSE or + DENSE. + + """ + + if dim is not None and not isinstance(dim, int): + raise TypeError("dim must be an integer") + if storage_format is not None and not isinstance( + storage_format, VectorStorageFormat + ): + raise TypeError( + "storage_format must be an enum of type VectorStorageFormat" + ) + if storage_type is not None and not isinstance( + storage_type, VectorStorageType + ): + raise TypeError( + "storage_type must be an enum of type VectorStorageType" + ) + + self.dim = dim + self.storage_format = storage_format + self.storage_type = storage_type + + def _cached_bind_processor(self, dialect): + """ + Converts a Python-side SparseVector instance into an + oracledb.SparseVectormor a compatible array format before + binding it to the database. + """ + + def process(value): + if value is None or isinstance(value, array.array): + return value + + # Convert list to a array.array + elif isinstance(value, list): + typecode = self._array_typecode(self.storage_format) + value = array.array(typecode, value) + return value + + # Convert SqlAlchemy SparseVector to oracledb SparseVector object + elif isinstance(value, SparseVector): + return dialect.dbapi.SparseVector( + value.num_dimensions, + value.indices, + value.values, + ) + + else: + raise TypeError( + """ + Invalid input for VECTOR: expected a list, an array.array, + or a SparseVector object. + """ + ) + + return process + + def _cached_result_processor(self, dialect, coltype): + """ + Converts database-returned values into Python-native representations. + If the value is an oracledb.SparseVector, it is converted into the + SQLAlchemy-side SparseVector class. + If the value is a array.array, it is converted to a plain Python list. + + """ + + def process(value): + if value is None: + return None + + elif isinstance(value, array.array): + return list(value) + + # Convert Oracledb SparseVector to SqlAlchemy SparseVector object + elif isinstance(value, dialect.dbapi.SparseVector): + return SparseVector( + num_dimensions=value.num_dimensions, + indices=value.indices, + values=value.values, + ) + + return process + + def _array_typecode(self, typecode): + """ + Map storage format to array typecode. + """ + return self._typecode_map.get(typecode, "d") + + class comparator_factory(types.TypeEngine.Comparator): + def l2_distance(self, other): + return self.op("<->", return_type=Float)(other) + + def inner_product(self, other): + return self.op("<#>", return_type=Float)(other) + + def cosine_distance(self, other): + return self.op("<=>", return_type=Float)(other) diff --git a/lib/sqlalchemy/dialects/postgresql/__init__.py b/lib/sqlalchemy/dialects/postgresql/__init__.py index 88935e20245..55dc41400cb 100644 --- a/lib/sqlalchemy/dialects/postgresql/__init__.py +++ b/lib/sqlalchemy/dialects/postgresql/__init__.py @@ -1,5 +1,5 @@ # dialects/postgresql/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -33,10 +33,12 @@ from .base import TEXT from .base import UUID from .base import VARCHAR +from .bitstring import BitString from .dml import Insert from .dml import insert from .ext import aggregate_order_by from .ext import array_agg +from .ext import distinct_on from .ext import ExcludeConstraint from .ext import phraseto_tsquery from .ext import plainto_tsquery @@ -95,7 +97,7 @@ "psycopg_async", (ModuleType,), {"dialect": psycopg.dialect_async} ) -base.dialect = dialect = psycopg2.dialect +base.dialect = dialect = psycopg.dialect __all__ = ( @@ -153,6 +155,7 @@ "JSONPATH", "Any", "All", + "BitString", "DropEnumType", "DropDomainType", "CreateDomainType", @@ -164,4 +167,5 @@ "array_agg", "insert", "Insert", + "distinct_on", ) diff --git a/lib/sqlalchemy/dialects/postgresql/_psycopg_common.py b/lib/sqlalchemy/dialects/postgresql/_psycopg_common.py index e5b39e50040..97b3d30bbcb 100644 --- a/lib/sqlalchemy/dialects/postgresql/_psycopg_common.py +++ b/lib/sqlalchemy/dialects/postgresql/_psycopg_common.py @@ -1,5 +1,5 @@ # dialects/postgresql/_psycopg_common.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -174,8 +174,10 @@ def get_deferrable(self, connection): def _do_autocommit(self, connection, value): connection.autocommit = value + def detect_autocommit_setting(self, dbapi_connection): + return bool(dbapi_connection.autocommit) + def do_ping(self, dbapi_connection): - cursor = None before_autocommit = dbapi_connection.autocommit if not before_autocommit: @@ -189,3 +191,39 @@ def do_ping(self, dbapi_connection): dbapi_connection.autocommit = before_autocommit return True + + def do_begin_twophase(self, connection, xid): + connection.connection.tpc_begin(xid) + + def do_prepare_twophase(self, connection, xid): + connection.connection.tpc_prepare() + + def _do_twophase(self, dbapi_conn, operation, xid, recover=False): + if recover: + if not self._twophase_idle_check(dbapi_conn): + dbapi_conn.rollback() + operation(xid) + else: + operation() + + def _twophase_idle_check(self, dbapi_conn): + raise NotImplementedError + + def do_rollback_twophase( + self, connection, xid, is_prepared=True, recover=False + ): + dbapi_conn = connection.connection.dbapi_connection + self._do_twophase( + dbapi_conn, dbapi_conn.tpc_rollback, xid, recover=recover + ) + + def do_commit_twophase( + self, connection, xid, is_prepared=True, recover=False + ): + dbapi_conn = connection.connection.dbapi_connection + self._do_twophase( + dbapi_conn, dbapi_conn.tpc_commit, xid, recover=recover + ) + + def do_recover_twophase(self, connection): + return [row[1] for row in connection.connection.tpc_recover()] diff --git a/lib/sqlalchemy/dialects/postgresql/array.py b/lib/sqlalchemy/dialects/postgresql/array.py index 7708769cb53..c2f2985b96a 100644 --- a/lib/sqlalchemy/dialects/postgresql/array.py +++ b/lib/sqlalchemy/dialects/postgresql/array.py @@ -1,18 +1,21 @@ # dialects/postgresql/array.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors from __future__ import annotations import re -from typing import Any +from typing import Any as typing_Any +from typing import Iterable from typing import Optional +from typing import Sequence +from typing import TYPE_CHECKING from typing import TypeVar +from typing import Union from .operators import CONTAINED_BY from .operators import CONTAINS @@ -21,28 +24,70 @@ from ... import util from ...sql import expression from ...sql import operators -from ...sql._typing import _TypeEngineArgument - +from ...sql.visitors import InternalTraversal + +if TYPE_CHECKING: + from ...engine.interfaces import Dialect + from ...sql._typing import _ColumnExpressionArgument + from ...sql._typing import _TypeEngineArgument + from ...sql.elements import ColumnElement + from ...sql.elements import Grouping + from ...sql.expression import BindParameter + from ...sql.operators import OperatorType + from ...sql.selectable import _SelectIterable + from ...sql.type_api import _BindProcessorType + from ...sql.type_api import _LiteralProcessorType + from ...sql.type_api import _ResultProcessorType + from ...sql.type_api import TypeEngine + from ...sql.visitors import _TraverseInternalsType + from ...util.typing import Self + + +_T = TypeVar("_T", bound=typing_Any) +_CT = TypeVar("_CT", bound=typing_Any) + + +def Any( + other: typing_Any, + arrexpr: _ColumnExpressionArgument[_T], + operator: OperatorType = operators.eq, +) -> ColumnElement[bool]: + """A synonym for the ARRAY-level :meth:`.ARRAY.Comparator.any` method. + See that method for details. -_T = TypeVar("_T", bound=Any) + .. deprecated:: 2.1 + The :meth:`_types.ARRAY.Comparator.any` and + :meth:`_types.ARRAY.Comparator.all` methods for arrays are deprecated + for removal, along with the PG-specific :func:`_postgresql.Any` and + :func:`_postgresql.All` functions. See :func:`_sql.any_` and + :func:`_sql.all_` functions for modern use. -def Any(other, arrexpr, operator=operators.eq): - """A synonym for the ARRAY-level :meth:`.ARRAY.Comparator.any` method. - See that method for details. """ - return arrexpr.any(other, operator) + return arrexpr.any(other, operator) # type: ignore[no-any-return, union-attr] # noqa: E501 -def All(other, arrexpr, operator=operators.eq): +def All( + other: typing_Any, + arrexpr: _ColumnExpressionArgument[_T], + operator: OperatorType = operators.eq, +) -> ColumnElement[bool]: """A synonym for the ARRAY-level :meth:`.ARRAY.Comparator.all` method. See that method for details. + .. deprecated:: 2.1 + + The :meth:`_types.ARRAY.Comparator.any` and + :meth:`_types.ARRAY.Comparator.all` methods for arrays are deprecated + for removal, along with the PG-specific :func:`_postgresql.Any` and + :func:`_postgresql.All` functions. See :func:`_sql.any_` and + :func:`_sql.all_` functions for modern use. + """ - return arrexpr.all(other, operator) + return arrexpr.all(other, operator) # type: ignore[no-any-return, union-attr] # noqa: E501 class array(expression.ExpressionClauseList[_T]): @@ -66,11 +111,32 @@ class array(expression.ExpressionClauseList[_T]): ARRAY[%(param_3)s, %(param_4)s, %(param_5)s]) AS anon_1 An instance of :class:`.array` will always have the datatype - :class:`_types.ARRAY`. The "inner" type of the array is inferred from - the values present, unless the ``type_`` keyword argument is passed:: + :class:`_types.ARRAY`. The "inner" type of the array is inferred from the + values present, unless the :paramref:`_postgresql.array.type_` keyword + argument is passed:: array(["foo", "bar"], type_=CHAR) + When constructing an empty array, the :paramref:`_postgresql.array.type_` + argument is particularly important as PostgreSQL server typically requires + a cast to be rendered for the inner type in order to render an empty array. + SQLAlchemy's compilation for the empty array will produce this cast so + that:: + + stmt = array([], type_=Integer) + print(stmt.compile(dialect=postgresql.dialect())) + + Produces: + + .. sourcecode:: sql + + ARRAY[]::INTEGER[] + + As required by PostgreSQL for empty arrays. + + .. versionadded:: 2.0.40 added support to render empty PostgreSQL array + literals with a required cast. + Multidimensional arrays are produced by nesting :class:`.array` constructs. The dimensionality of the final :class:`_types.ARRAY` type is calculated by @@ -94,8 +160,6 @@ class array(expression.ExpressionClauseList[_T]): ARRAY[q, x] ] AS anon_1 - .. versionadded:: 1.3.6 added support for multidimensional array literals - .. seealso:: :class:`_postgresql.ARRAY` @@ -105,18 +169,33 @@ class array(expression.ExpressionClauseList[_T]): __visit_name__ = "array" stringify_dialect = "postgresql" - inherit_cache = True - def __init__(self, clauses, **kw): - type_arg = kw.pop("type_", None) - super().__init__(operators.comma_op, *clauses, **kw) + _traverse_internals: _TraverseInternalsType = [ + ("clauses", InternalTraversal.dp_clauseelement_tuple), + ("type", InternalTraversal.dp_type), + ] - self._type_tuple = [arg.type for arg in self.clauses] + def __init__( + self, + clauses: Iterable[_T], + *, + type_: Optional[_TypeEngineArgument[_T]] = None, + **kw: typing_Any, + ): + r"""Construct an ARRAY literal. + + :param clauses: iterable, such as a list, containing elements to be + rendered in the array + :param type\_: optional type. If omitted, the type is inferred + from the contents of the array. + + """ + super().__init__(operators.comma_op, *clauses, **kw) main_type = ( - type_arg - if type_arg is not None - else self._type_tuple[0] if self._type_tuple else sqltypes.NULLTYPE + type_ + if type_ is not None + else self.clauses[0].type if self.clauses else sqltypes.NULLTYPE ) if isinstance(main_type, ARRAY): @@ -127,15 +206,21 @@ def __init__(self, clauses, **kw): if main_type.dimensions is not None else 2 ), - ) + ) # type: ignore[assignment] else: - self.type = ARRAY(main_type) + self.type = ARRAY(main_type) # type: ignore[assignment] @property - def _select_iterable(self): + def _select_iterable(self) -> _SelectIterable: return (self,) - def _bind_param(self, operator, obj, _assume_scalar=False, type_=None): + def _bind_param( + self, + operator: OperatorType, + obj: typing_Any, + type_: Optional[TypeEngine[_T]] = None, + _assume_scalar: bool = False, + ) -> BindParameter[_T]: if _assume_scalar or operator is operators.getitem: return expression.BindParameter( None, @@ -154,16 +239,18 @@ def _bind_param(self, operator, obj, _assume_scalar=False, type_=None): ) for o in obj ] - ) + ) # type: ignore[return-value] - def self_group(self, against=None): + def self_group( + self, against: Optional[OperatorType] = None + ) -> Union[Self, Grouping[_T]]: if against in (operators.any_op, operators.all_op, operators.getitem): return expression.Grouping(self) else: return self -class ARRAY(sqltypes.ARRAY): +class ARRAY(sqltypes.ARRAY[_T]): """PostgreSQL ARRAY type. The :class:`_postgresql.ARRAY` type is constructed in the same way @@ -237,7 +324,7 @@ class SomeOrmClass(Base): def __init__( self, - item_type: _TypeEngineArgument[Any], + item_type: _TypeEngineArgument[_T], as_tuple: bool = False, dimensions: Optional[int] = None, zero_indexes: bool = False, @@ -286,7 +373,7 @@ def __init__( self.dimensions = dimensions self.zero_indexes = zero_indexes - class Comparator(sqltypes.ARRAY.Comparator): + class Comparator(sqltypes.ARRAY.Comparator[_CT]): """Define comparison operations for :class:`_types.ARRAY`. Note that these operations are in addition to those provided @@ -296,7 +383,9 @@ class Comparator(sqltypes.ARRAY.Comparator): """ - def contains(self, other, **kwargs): + def contains( + self, other: typing_Any, **kwargs: typing_Any + ) -> ColumnElement[bool]: """Boolean expression. Test if elements are a superset of the elements of the argument array expression. @@ -305,7 +394,7 @@ def contains(self, other, **kwargs): """ return self.operate(CONTAINS, other, result_type=sqltypes.Boolean) - def contained_by(self, other): + def contained_by(self, other: typing_Any) -> ColumnElement[bool]: """Boolean expression. Test if elements are a proper subset of the elements of the argument array expression. """ @@ -313,7 +402,7 @@ def contained_by(self, other): CONTAINED_BY, other, result_type=sqltypes.Boolean ) - def overlap(self, other): + def overlap(self, other: typing_Any) -> ColumnElement[bool]: """Boolean expression. Test if array has elements in common with an argument array expression. """ @@ -321,35 +410,26 @@ def overlap(self, other): comparator_factory = Comparator - @property - def hashable(self): - return self.as_tuple - - @property - def python_type(self): - return list - - def compare_values(self, x, y): - return x == y - @util.memoized_property - def _against_native_enum(self): + def _against_native_enum(self) -> bool: return ( isinstance(self.item_type, sqltypes.Enum) and self.item_type.native_enum ) - def literal_processor(self, dialect): + def literal_processor( + self, dialect: Dialect + ) -> Optional[_LiteralProcessorType[_T]]: item_proc = self.item_type.dialect_impl(dialect).literal_processor( dialect ) if item_proc is None: return None - def to_str(elements): + def to_str(elements: Iterable[typing_Any]) -> str: return f"ARRAY[{', '.join(elements)}]" - def process(value): + def process(value: Sequence[typing_Any]) -> str: inner = self._apply_item_processor( value, item_proc, self.dimensions, to_str ) @@ -357,12 +437,16 @@ def process(value): return process - def bind_processor(self, dialect): + def bind_processor( + self, dialect: Dialect + ) -> Optional[_BindProcessorType[Sequence[typing_Any]]]: item_proc = self.item_type.dialect_impl(dialect).bind_processor( dialect ) - def process(value): + def process( + value: Optional[Sequence[typing_Any]], + ) -> Optional[list[typing_Any]]: if value is None: return value else: @@ -372,12 +456,16 @@ def process(value): return process - def result_processor(self, dialect, coltype): + def result_processor( + self, dialect: Dialect, coltype: object + ) -> _ResultProcessorType[Sequence[typing_Any]]: item_proc = self.item_type.dialect_impl(dialect).result_processor( dialect, coltype ) - def process(value): + def process( + value: Sequence[typing_Any], + ) -> Optional[Sequence[typing_Any]]: if value is None: return value else: @@ -392,11 +480,13 @@ def process(value): super_rp = process pattern = re.compile(r"^{(.*)}$") - def handle_raw_string(value): - inner = pattern.match(value).group(1) + def handle_raw_string(value: str) -> Sequence[Optional[str]]: + inner = pattern.match(value).group(1) # type: ignore[union-attr] # noqa: E501 return _split_enum_values(inner) - def process(value): + def process( + value: Sequence[typing_Any], + ) -> Optional[Sequence[typing_Any]]: if value is None: return value # isinstance(value, str) is required to handle @@ -411,10 +501,13 @@ def process(value): return process -def _split_enum_values(array_string): +def _split_enum_values(array_string: str) -> Sequence[Optional[str]]: if '"' not in array_string: # no escape char is present so it can just split on the comma - return array_string.split(",") if array_string else [] + return [ + r if r != "NULL" else None + for r in (array_string.split(",") if array_string else []) + ] # handles quoted strings from: # r'abc,"quoted","also\\\\quoted", "quoted, comma", "esc \" quot", qpr' @@ -431,5 +524,11 @@ def _split_enum_values(array_string): elif in_quotes: result.append(tok.replace("_$ESC_QUOTE$_", '"')) else: - result.extend(re.findall(r"([^\s,]+),?", tok)) + # interpret NULL (without quotes!) as None + result.extend( + [ + r if r != "NULL" else None + for r in re.findall(r"([^\s,]+),?", tok) + ] + ) return result diff --git a/lib/sqlalchemy/dialects/postgresql/asyncpg.py b/lib/sqlalchemy/dialects/postgresql/asyncpg.py index 3d6aae91764..33aa9a9de92 100644 --- a/lib/sqlalchemy/dialects/postgresql/asyncpg.py +++ b/lib/sqlalchemy/dialects/postgresql/asyncpg.py @@ -1,5 +1,5 @@ # dialects/postgresql/asyncpg.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under @@ -178,13 +178,15 @@ from __future__ import annotations -import asyncio from collections import deque import decimal import json as _py_json import re import time +from types import NoneType from typing import Any +from typing import Awaitable +from typing import Callable from typing import NoReturn from typing import Optional from typing import Protocol @@ -207,6 +209,7 @@ from .base import PGIdentifierPreparer from .base import REGCLASS from .base import REGCONFIG +from .bitstring import BitString from .types import BIT from .types import BYTEA from .types import CITEXT @@ -214,7 +217,10 @@ from ... import util from ...connectors.asyncio import AsyncAdapt_dbapi_connection from ...connectors.asyncio import AsyncAdapt_dbapi_cursor +from ...connectors.asyncio import AsyncAdapt_dbapi_module from ...connectors.asyncio import AsyncAdapt_dbapi_ss_cursor +from ...connectors.asyncio import AsyncAdapt_Error +from ...connectors.asyncio import AsyncAdapt_terminate from ...engine import processors from ...sql import sqltypes from ...util.concurrency import await_ @@ -242,6 +248,25 @@ class AsyncpgTime(sqltypes.Time): class AsyncpgBit(BIT): render_bind_cast = True + def bind_processor(self, dialect): + asyncpg_BitString = dialect.dbapi.asyncpg.BitString + + def to_bind(value): + if isinstance(value, str): + value = BitString(value) + value = asyncpg_BitString.from_int(int(value), len(value)) + return value + + return to_bind + + def result_processor(self, dialect, coltype): + def to_result(value): + if value is not None: + value = BitString.from_int(value.to_int(), length=len(value)) + return value + + return to_result + class AsyncpgByteA(BYTEA): render_bind_cast = True @@ -283,16 +308,6 @@ class AsyncpgBigInteger(sqltypes.BigInteger): render_bind_cast = True -class AsyncpgJSON(json.JSON): - def result_processor(self, dialect, coltype): - return None - - -class AsyncpgJSONB(json.JSONB): - def result_processor(self, dialect, coltype): - return None - - class AsyncpgJSONIndexType(sqltypes.JSON.JSONIndexType): pass @@ -413,8 +428,6 @@ class _AsyncpgMultiRange(ranges.AbstractMultiRangeImpl): def bind_processor(self, dialect): asyncpg_Range = dialect.dbapi.asyncpg.Range - NoneType = type(None) - def to_range(value): if isinstance(value, (str, NoneType)): return value @@ -490,6 +503,12 @@ class PGIdentifierPreparer_asyncpg(PGIdentifierPreparer): pass +class _AsyncpgTransaction(Protocol): + async def start(self) -> None: ... + async def commit(self) -> None: ... + async def rollback(self) -> None: ... + + class _AsyncpgConnection(Protocol): async def executemany( self, operation: Any, seq_of_parameters: Sequence[Tuple[Any, ...]] @@ -509,11 +528,11 @@ def transaction( isolation: Optional[str] = None, readonly: bool = False, deferrable: bool = False, - ) -> Any: ... + ) -> _AsyncpgTransaction: ... def fetchrow(self, operation: str) -> Any: ... - async def close(self) -> None: ... + async def close(self, timeout: int = ...) -> None: ... def terminate(self) -> None: ... @@ -533,6 +552,7 @@ class AsyncAdapt_asyncpg_cursor(AsyncAdapt_dbapi_cursor): _adapt_connection: AsyncAdapt_asyncpg_connection _connection: _AsyncpgConnection _cursor: Optional[_AsyncpgCursor] + _awaitable_cursor_close: bool = False def __init__(self, adapt_connection: AsyncAdapt_asyncpg_connection): self._adapt_connection = adapt_connection @@ -551,7 +571,7 @@ async def _prepare_and_execute(self, operation, parameters): adapt_connection = self._adapt_connection async with adapt_connection._execute_mutex: - if not adapt_connection._started: + if adapt_connection._transaction is None: await adapt_connection._start_transaction() if parameters is None: @@ -622,7 +642,7 @@ async def _executemany(self, operation, seq_of_parameters): self._invalidate_schema_cache_asof ) - if not adapt_connection._started: + if adapt_connection._transaction is None: await adapt_connection._start_transaction() try: @@ -722,11 +742,14 @@ def executemany(self, operation, seq_of_parameters): ) -class AsyncAdapt_asyncpg_connection(AsyncAdapt_dbapi_connection): +class AsyncAdapt_asyncpg_connection( + AsyncAdapt_terminate, AsyncAdapt_dbapi_connection +): _cursor_cls = AsyncAdapt_asyncpg_cursor _ss_cursor_cls = AsyncAdapt_asyncpg_ss_cursor _connection: _AsyncpgConnection + _transaction: Optional[_AsyncpgTransaction] __slots__ = ( "isolation_level", @@ -734,7 +757,6 @@ class AsyncAdapt_asyncpg_connection(AsyncAdapt_dbapi_connection): "readonly", "deferrable", "_transaction", - "_started", "_prepared_statement_cache", "_prepared_statement_name_func", "_invalidate_schema_cache_asof", @@ -752,7 +774,6 @@ def __init__( self.readonly = False self.deferrable = False self._transaction = None - self._started = False self._invalidate_schema_cache_asof = time.time() if prepared_statement_cache_size: @@ -803,27 +824,27 @@ async def _prepare(self, operation, invalidate_timestamp): return prepared_stmt, attributes - def _handle_exception(self, error: Exception) -> NoReturn: - if self._connection.is_closed(): - self._transaction = None - self._started = False - + @classmethod + def _handle_exception_no_connection( + cls, dbapi: Any, error: Exception + ) -> NoReturn: if not isinstance(error, AsyncAdapt_asyncpg_dbapi.Error): - exception_mapping = self.dbapi._asyncpg_error_translate + exception_mapping = dbapi._asyncpg_error_translate for super_ in type(error).__mro__: if super_ in exception_mapping: + message = error.args[0] translated_error = exception_mapping[super_]( - "%s: %s" % (type(error), error) - ) - translated_error.pgcode = translated_error.sqlstate = ( - getattr(error, "sqlstate", None) + message, error ) raise translated_error from error - else: - super()._handle_exception(error) - else: - super()._handle_exception(error) + super()._handle_exception_no_connection(dbapi, error) + + def _handle_exception(self, error: Exception) -> NoReturn: + if self._connection.is_closed(): + self._transaction = None + + super()._handle_exception(error) @property def autocommit(self): @@ -844,7 +865,7 @@ def ping(self): async def _async_ping(self): if self._transaction is None and self.isolation_level != "autocommit": - # create a tranasction explicitly to support pgbouncer + # create a transaction explicitly to support pgbouncer # transaction mode. See #10226 tr = self._connection.transaction() await tr.start() @@ -856,14 +877,14 @@ async def _async_ping(self): await self._connection.fetchrow(";") def set_isolation_level(self, level): - if self._started: - self.rollback() + self.rollback() self.isolation_level = self._isolation_setting = level async def _start_transaction(self): if self.isolation_level == "autocommit": return + assert self._transaction is None try: self._transaction = self._connection.transaction( isolation=self.isolation_level, @@ -873,46 +894,28 @@ async def _start_transaction(self): await self._transaction.start() except Exception as error: self._handle_exception(error) - else: - self._started = True - async def _rollback_and_discard(self): + async def _call_and_discard(self, fn: Callable[[], Awaitable[Any]]): try: - await self._transaction.rollback() + await fn() finally: - # if asyncpg .rollback() was actually called, then whether or - # not it raised or succeeded, the transation is done, discard it + # if asyncpg fn was actually called, then whether or + # not it raised or succeeded, the transaction is done, discard it self._transaction = None - self._started = False - - async def _commit_and_discard(self): - try: - await self._transaction.commit() - finally: - # if asyncpg .commit() was actually called, then whether or - # not it raised or succeeded, the transation is done, discard it - self._transaction = None - self._started = False def rollback(self): - if self._started: - assert self._transaction is not None + if self._transaction is not None: try: - await_(self._rollback_and_discard()) - self._transaction = None - self._started = False + await_(self._call_and_discard(self._transaction.rollback)) except Exception as error: # don't dereference asyncpg transaction if we didn't # actually try to call rollback() on it self._handle_exception(error) def commit(self): - if self._started: - assert self._transaction is not None + if self._transaction is not None: try: - await_(self._commit_and_discard()) - self._transaction = None - self._started = False + await_(self._call_and_discard(self._transaction.commit)) except Exception as error: # don't dereference asyncpg transaction if we didn't # actually try to call commit() on it @@ -923,38 +926,28 @@ def close(self): await_(self._connection.close()) - def terminate(self): - if util.concurrency.in_greenlet(): - # in a greenlet; this is the connection was invalidated - # case. - try: - # try to gracefully close; see #10717 - # timeout added in asyncpg 0.14.0 December 2017 - await_(asyncio.shield(self._connection.close(timeout=2))) - except ( - asyncio.TimeoutError, - asyncio.CancelledError, - OSError, - self.dbapi.asyncpg.PostgresError, - ): - # in the case where we are recycling an old connection - # that may have already been disconnected, close() will - # fail with the above timeout. in this case, terminate - # the connection without any further waiting. - # see issue #8419 - self._connection.terminate() - else: - # not in a greenlet; this is the gc cleanup case - self._connection.terminate() - self._started = False + def _terminate_handled_exceptions(self): + return super()._terminate_handled_exceptions() + ( + self.dbapi.asyncpg.PostgresError, + ) + + async def _terminate_graceful_close(self) -> None: + # timeout added in asyncpg 0.14.0 December 2017 + await self._connection.close(timeout=2) + self._transaction = None + + def _terminate_force_close(self) -> None: + self._connection.terminate() + self._transaction = None @staticmethod def _default_name_func(): return None -class AsyncAdapt_asyncpg_dbapi: +class AsyncAdapt_asyncpg_dbapi(AsyncAdapt_dbapi_module): def __init__(self, asyncpg): + super().__init__(asyncpg) self.asyncpg = asyncpg self.paramstyle = "numeric_dollar" @@ -967,17 +960,29 @@ def connect(self, *arg, **kw): "prepared_statement_name_func", None ) - return AsyncAdapt_asyncpg_connection( - self, - await_(creator_fn(*arg, **kw)), - prepared_statement_cache_size=prepared_statement_cache_size, - prepared_statement_name_func=prepared_statement_name_func, + return await_( + AsyncAdapt_asyncpg_connection.create( + self, + creator_fn(*arg, **kw), + prepared_statement_cache_size=prepared_statement_cache_size, + prepared_statement_name_func=prepared_statement_name_func, + ) ) - class Error(Exception): - pass + class Error(AsyncAdapt_Error): + + pgcode: str | None + + sqlstate: str | None - class Warning(Exception): # noqa + detail: str | None + + def __init__(self, message, error=None): + super().__init__(message, error) + self.detail = getattr(error, "detail", None) + self.pgcode = self.sqlstate = getattr(error, "sqlstate", None) + + class Warning(AsyncAdapt_Error): # noqa pass class InterfaceError(Error): @@ -998,6 +1003,24 @@ class ProgrammingError(DatabaseError): class IntegrityError(DatabaseError): pass + class RestrictViolationError(IntegrityError): + pass + + class NotNullViolationError(IntegrityError): + pass + + class ForeignKeyViolationError(IntegrityError): + pass + + class UniqueViolationError(IntegrityError): + pass + + class CheckViolationError(IntegrityError): + pass + + class ExclusionViolationError(IntegrityError): + pass + class DataError(DatabaseError): pass @@ -1007,8 +1030,11 @@ class NotSupportedError(DatabaseError): class InternalServerError(InternalError): pass + class InternalClientError(InternalError): + pass + class InvalidCachedStatementError(NotSupportedError): - def __init__(self, message): + def __init__(self, message, error=None): super().__init__( message + " (SQLAlchemy asyncpg dialect will now invalidate " "all prepared caches in response to this exception)", @@ -1031,6 +1057,13 @@ def _asyncpg_error_translate(self): asyncpg.exceptions.InterfaceError: self.InterfaceError, asyncpg.exceptions.InvalidCachedStatementError: self.InvalidCachedStatementError, # noqa: E501 asyncpg.exceptions.InternalServerError: self.InternalServerError, + asyncpg.exceptions.RestrictViolationError: self.RestrictViolationError, # noqa: E501 + asyncpg.exceptions.NotNullViolationError: self.NotNullViolationError, # noqa: E501 + asyncpg.exceptions.ForeignKeyViolationError: self.ForeignKeyViolationError, # noqa: E501 + asyncpg.exceptions.UniqueViolationError: self.UniqueViolationError, + asyncpg.exceptions.CheckViolationError: self.CheckViolationError, + asyncpg.exceptions.ExclusionViolationError: self.ExclusionViolationError, # noqa: E501 + asyncpg.exceptions.InternalClientError: self.InternalClientError, } def Binary(self, value): @@ -1052,6 +1085,10 @@ class PGDialect_asyncpg(PGDialect): statement_compiler = PGCompiler_asyncpg preparer = PGIdentifierPreparer_asyncpg + supports_native_json_serialization = False + supports_native_json_deserialization = True + dialect_injects_custom_json_deserializer = True + colspecs = util.update_copy( PGDialect.colspecs, { @@ -1071,9 +1108,7 @@ class PGDialect_asyncpg(PGDialect): sqltypes.BigInteger: AsyncpgBigInteger, sqltypes.Numeric: AsyncpgNumeric, sqltypes.Float: AsyncpgFloat, - sqltypes.JSON: AsyncpgJSON, sqltypes.LargeBinary: AsyncpgByteA, - json.JSONB: AsyncpgJSONB, sqltypes.JSON.JSONPathType: AsyncpgJSONPathType, sqltypes.JSON.JSONIndexType: AsyncpgJSONIndexType, sqltypes.JSON.JSONIntIndexType: AsyncpgJSONIntIndexType, @@ -1125,6 +1160,9 @@ def get_isolation_level_values(self, dbapi_connection): def set_isolation_level(self, dbapi_connection, level): dbapi_connection.set_isolation_level(self._isolation_lookup[level]) + def detect_autocommit_setting(self, dbapi_conn) -> bool: + return bool(dbapi_conn.autocommit) + def set_readonly(self, connection, value): connection.readonly = value diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 1f00127bfa6..1702bc70c97 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -1,5 +1,5 @@ # dialects/postgresql/base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -114,6 +114,47 @@ def use_identity(element, compiler, **kw): PRIMARY KEY (id) ) +.. _postgresql_monotonic_functions: + +PostgreSQL 18 and above UUID with uuidv7 as a server default +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +PostgreSQL 18's ``uuidv7`` SQL function is available as any other +SQL function using the :data:`_sql.func` namespace:: + + >>> from sqlalchemy import select, func + >>> print(select(func.uuidv7())) + SELECT uuidv7() AS uuidv7_1 + +When using ``func.uuidv7()`` as a default on a :class:`.Column` using either +Core or ORM, an extra directive ``monotonic=True`` may be passed which +indicates this function produces monotonically increasing values; this in turn +allows Core and ORM to use a more efficient batched form of INSERT for large +insert operations:: + + import uuid + + + class MyClass(Base): + __tablename__ = "my_table" + + id: Mapped[uuid.UUID] = mapped_column( + server_default=func.uuidv7(monotonic=True) + ) + +With the above mapping, the ORM will be able to efficiently batch rows when +running bulk insert operations using the :ref:`engine_insertmanyvalues` +feature. + +.. versionadded:: 2.1 + Added ``monotonic=True`` to allow functions like PostgreSQL's + ``uuidv7()`` to work with batched "insertmanyvalues" + +.. seealso:: + + :ref:`engine_insertmanyvalues_monotonic_functions` + + .. _postgresql_ss_cursors: Server Side Cursors @@ -227,7 +268,8 @@ def use_identity(element, compiler, **kw): Note that some DBAPIs such as asyncpg only support "readonly" with SERIALIZABLE isolation. -.. versionadded:: 1.4 added support for the ``postgresql_readonly`` +.. versionadded:: 1.4 + Added support for the ``postgresql_readonly`` and ``postgresql_deferrable`` execution options. .. _postgresql_reset_on_return: @@ -266,7 +308,7 @@ def use_identity(element, compiler, **kw): from sqlalchemy import event postgresql_engine = create_engine( - "postgresql+pyscopg2://scott:tiger@hostname/dbname", + "postgresql+psycopg2://scott:tiger@hostname/dbname", # disable default reset-on-return scheme pool_reset_on_return=None, ) @@ -978,11 +1020,18 @@ def set_search_path(dbapi_connection, connection_record): Several extensions to the :class:`.Index` construct are available, specific to the PostgreSQL dialect. +.. _postgresql_covering_indexes: + Covering Indexes ^^^^^^^^^^^^^^^^ -The ``postgresql_include`` option renders INCLUDE(colname) for the given -string names:: +A covering index includes additional columns that are not part of the index key +but are stored in the index, allowing PostgreSQL to satisfy queries using only +the index without accessing the table (an "index-only scan"). This is +indicated on the index using the ``INCLUDE`` clause. The +``postgresql_include`` option for :class:`.Index` (as well as +:class:`.UniqueConstraint`) renders ``INCLUDE(colname)`` for the given string +names:: Index("my_index", table.c.x, postgresql_include=["y"]) @@ -990,7 +1039,13 @@ def set_search_path(dbapi_connection, connection_record): Note that this feature requires PostgreSQL 11 or later. +.. seealso:: + + :ref:`postgresql_constraint_options_include` - the same feature implemented + for :class:`.UniqueConstraint` + .. versionadded:: 1.4 + Support for covering indexes with :class:`.Index`. .. _postgresql_partial_indexes: @@ -1042,10 +1097,6 @@ def set_search_path(dbapi_connection, connection_record): :paramref:`_postgresql.ExcludeConstraint.ops` parameter. See that parameter for details. -.. versionadded:: 1.3.21 added support for operator classes with - :class:`_postgresql.ExcludeConstraint`. - - Index Types ^^^^^^^^^^^ @@ -1164,20 +1215,44 @@ def set_search_path(dbapi_connection, connection_record): ------------------------ Several options for CREATE TABLE are supported directly by the PostgreSQL -dialect in conjunction with the :class:`_schema.Table` construct: +dialect in conjunction with the :class:`_schema.Table` construct, detailed +in the following sections. + +.. seealso:: -* ``INHERITS``:: + `PostgreSQL CREATE TABLE options + `_ - + in the PostgreSQL documentation. + +``INHERITS`` +^^^^^^^^^^^^ + +Specifies one or more parent tables from which this table inherits columns and +constraints, enabling table inheritance hierarchies in PostgreSQL. + +:: Table("some_table", metadata, ..., postgresql_inherits="some_supertable") Table("some_table", metadata, ..., postgresql_inherits=("t1", "t2", ...)) -* ``ON COMMIT``:: +``ON COMMIT`` +^^^^^^^^^^^^^ + +Controls the behavior of temporary tables at transaction commit, with options +to preserve rows, delete rows, or drop the table. + +:: Table("some_table", metadata, ..., postgresql_on_commit="PRESERVE ROWS") -* - ``PARTITION BY``:: +``PARTITION BY`` +^^^^^^^^^^^^^^^^ + +Declares the table as a partitioned table using the specified partitioning +strategy (RANGE, LIST, or HASH) on the given column(s). + +:: Table( "some_table", @@ -1186,83 +1261,272 @@ def set_search_path(dbapi_connection, connection_record): postgresql_partition_by="LIST (part_column)", ) - .. versionadded:: 1.2.6 +``TABLESPACE`` +^^^^^^^^^^^^^^ + +Specifies the tablespace where the table will be stored, allowing control over +the physical location of table data on disk. -* - ``TABLESPACE``:: +:: Table("some_table", metadata, ..., postgresql_tablespace="some_tablespace") - The above option is also available on the :class:`.Index` construct. +The above option is also available on the :class:`.Index` construct. + +``USING`` +^^^^^^^^^ + +Specifies the table access method to use for storing table data, such as +``heap`` (the default) or other custom access methods. -* - ``USING``:: +:: Table("some_table", metadata, ..., postgresql_using="heap") - .. versionadded:: 2.0.26 +.. versionadded:: 2.0.26 -* ``WITH OIDS``:: +.. _postgresql_table_options_with: - Table("some_table", metadata, ..., postgresql_with_oids=True) +``WITH`` +^^^^^^^^ -* ``WITHOUT OIDS``:: +:: - Table("some_table", metadata, ..., postgresql_with_oids=False) + Table("some_table", metadata, ..., postgresql_with={"fillfactor": 100}) + +The ``postgresql_with`` parameter accepts a dictionary of storage parameters +that will be applied to the table using the ``WITH`` clause in the +``CREATE TABLE`` statement. Storage parameters control various aspects of +table behavior such as the fill factor for pages, autovacuum settings, and +toast table parameters. + +When the table is created, the parameters are rendered as a comma-separated +list within the ``WITH`` clause. For example:: + + Table( + "mytable", + metadata, + Column("id", Integer, primary_key=True), + postgresql_with={ + "fillfactor": 70, + "autovacuum_enabled": "false", + "toast.vacuum_truncate": True, + }, + ) + +This will generate DDL similar to: + +.. sourcecode:: sql + + CREATE TABLE mytable ( + id INTEGER NOT NULL, + PRIMARY KEY (id) + ) WITH (fillfactor = 70, autovacuum_enabled = false, toast.vacuum_truncate = True) + +The values in the dictionary can be integers, strings, or boolean values. +Parameter names can include dots to specify parameters for associated objects +like toast tables (e.g., ``toast.vacuum_truncate``). .. seealso:: - `PostgreSQL CREATE TABLE options - `_ - - in the PostgreSQL documentation. + `PostgreSQL Storage Parameters + `_ - + documentation on available storage parameters. + +.. versionadded:: 2.1 + +``WITH OIDS`` +^^^^^^^^^^^^^ + +Enables the legacy OID (object identifier) system column for the table, which +assigns a unique identifier to each row. + +:: + + Table("some_table", metadata, ..., postgresql_with_oids=True) + +.. note:: Support for tables "with OIDs" was removed in postgresql 12. + +``WITHOUT OIDS`` +^^^^^^^^^^^^^^^^ + +Explicitly disables the OID system column for the table (the default behavior +in modern PostgreSQL versions). + +:: + + Table("some_table", metadata, ..., postgresql_with_oids=False) .. _postgresql_constraint_options: PostgreSQL Constraint Options ----------------------------- -The following option(s) are supported by the PostgreSQL dialect in conjunction -with selected constraint constructs: +The following sections indicate options which are supported by the PostgreSQL +dialect in conjunction with selected constraint constructs. -* ``NOT VALID``: This option applies towards CHECK and FOREIGN KEY constraints - when the constraint is being added to an existing table via ALTER TABLE, - and has the effect that existing rows are not scanned during the ALTER - operation against the constraint being added. - When using a SQL migration tool such as `Alembic `_ - that renders ALTER TABLE constructs, the ``postgresql_not_valid`` argument - may be specified as an additional keyword argument within the operation - that creates the constraint, as in the following Alembic example:: +``NOT VALID`` +^^^^^^^^^^^^^ - def update(): - op.create_foreign_key( - "fk_user_address", - "address", - "user", - ["user_id"], - ["id"], - postgresql_not_valid=True, - ) +Allows a constraint to be added without validating existing rows, improving +performance when adding constraints to large tables. This option applies +towards CHECK and FOREIGN KEY constraints when the constraint is being added +to an existing table via ALTER TABLE, and has the effect that existing rows +are not scanned during the ALTER operation against the constraint being added. - The keyword is ultimately accepted directly by the - :class:`_schema.CheckConstraint`, :class:`_schema.ForeignKeyConstraint` - and :class:`_schema.ForeignKey` constructs; when using a tool like - Alembic, dialect-specific keyword arguments are passed through to - these constructs from the migration operation directives:: +When using a SQL migration tool such as `Alembic `_ +that renders ALTER TABLE constructs, the ``postgresql_not_valid`` argument +may be specified as an additional keyword argument within the operation +that creates the constraint, as in the following Alembic example:: - CheckConstraint("some_field IS NOT NULL", postgresql_not_valid=True) + def update(): + op.create_foreign_key( + "fk_user_address", + "address", + "user", + ["user_id"], + ["id"], + postgresql_not_valid=True, + ) - ForeignKeyConstraint( - ["some_id"], ["some_table.some_id"], postgresql_not_valid=True - ) +The keyword is ultimately accepted directly by the +:class:`_schema.CheckConstraint`, :class:`_schema.ForeignKeyConstraint` +and :class:`_schema.ForeignKey` constructs; when using a tool like +Alembic, dialect-specific keyword arguments are passed through to +these constructs from the migration operation directives:: + + CheckConstraint("some_field IS NOT NULL", postgresql_not_valid=True) + + ForeignKeyConstraint( + ["some_id"], ["some_table.some_id"], postgresql_not_valid=True + ) + +.. versionadded:: 1.4.32 + +.. seealso:: + + `PostgreSQL ALTER TABLE options + `_ - + in the PostgreSQL documentation. + +.. _postgresql_constraint_options_include: + +``INCLUDE`` +^^^^^^^^^^^ + +This keyword is applicable to both a ``UNIQUE`` constraint as well as an +``INDEX``. The ``postgresql_include`` option available for +:class:`.UniqueConstraint` as well as :class:`.Index` creates a covering index +by including additional columns in the underlying index without making them +part of the key constraint. This option adds one or more columns as a "payload" +to the index created automatically by PostgreSQL for the constraint. For +example, the following table definition:: + + Table( + "mytable", + metadata, + Column("id", Integer, nullable=False), + Column("value", Integer, nullable=False), + UniqueConstraint("id", postgresql_include=["value"]), + ) + +would produce the DDL statement + +.. sourcecode:: sql + + CREATE TABLE mytable ( + id INTEGER NOT NULL, + value INTEGER NOT NULL, + UNIQUE (id) INCLUDE (value) + ) + +Note that this feature requires PostgreSQL 11 or later. + +.. versionadded:: 2.0.41 + Added support for ``postgresql_include`` to :class:`.UniqueConstraint`, + to complement the existing feature in :class:`.Index`. + +.. seealso:: + + :ref:`postgresql_covering_indexes` - background on ``postgresql_include`` + for the :class:`.Index` construct. + + +Column list with foreign key ``ON DELETE SET`` actions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Allows selective column updates when a foreign key action is triggered, limiting +which columns are set to NULL or DEFAULT upon deletion of a referenced row. +This applies to :class:`.ForeignKey` and :class:`.ForeignKeyConstraint`, the +:paramref:`.ForeignKey.ondelete` parameter will accept on the PostgreSQL +backend only a string list of column names inside parenthesis, following the +``SET NULL`` or ``SET DEFAULT`` phrases, which will limit the set of columns +that are subject to the action:: + + fktable = Table( + "fktable", + metadata, + Column("tid", Integer), + Column("id", Integer), + Column("fk_id_del_set_null", Integer), + ForeignKeyConstraint( + columns=["tid", "fk_id_del_set_null"], + refcolumns=[pktable.c.tid, pktable.c.id], + ondelete="SET NULL (fk_id_del_set_null)", + ), + ) + +.. versionadded:: 2.0.40 + +.. _postgresql_computed_column_notes: + +Computed Columns (GENERATED ALWAYS AS) +--------------------------------------- + +SQLAlchemy's support for the "GENERATED ALWAYS AS" SQL instruction, which +establishes a dynamic, automatically populated value for a column, is available +using the :ref:`computed_ddl` feature of SQLAlchemy DDL. E.g.:: + + from sqlalchemy import Table, Column, MetaData, Integer, Computed + + metadata_obj = MetaData() + + square = Table( + "square", + metadata_obj, + Column("id", Integer, primary_key=True), + Column("side", Integer), + Column("area", Integer, Computed("side * side")), + Column("perimeter", Integer, Computed("4 * side", persisted=True)), + ) + +There are two general varieties of the "computed" column, ``VIRTUAL`` and ``STORED``. +A ``STORED`` computed column computes and persists its value at INSERT/UPDATE time, +while a ``VIRTUAL`` computed column computes its value on access without persisting it. +This preference is indicated using the :paramref:`.Computed.persisted` parameter, +which defaults to ``None`` to use the database default behavior. + +For PostgreSQL, prior to version 18 only the ``STORED`` variant was supported, +requiring the ``STORED`` keyword to be emitted explicitly. PostgreSQL 18 added +support for ``VIRTUAL`` columns and made ``VIRTUAL`` the default behavior. + +To accommodate this change, SQLAlchemy's behavior when +:paramref:`.Computed.persisted` is not specified depends on the PostgreSQL +version: on PostgreSQL 18 and later, no keyword is rendered, allowing the +database to use its default of ``VIRTUAL``; on PostgreSQL 17 and earlier, +``STORED`` is rendered and a warning is emitted. To ensure consistent +``STORED`` behavior across all PostgreSQL versions, explicitly set +``persisted=True``. + +.. versionchanged:: 2.1 + + PostgreSQL 18+ now defaults to ``VIRTUAL`` when :paramref:`.Computed.persisted` + is not specified. A warning is emitted for older versions of PostgreSQL + when this parameter is not indicated. - .. versionadded:: 1.4.32 - .. seealso:: - `PostgreSQL ALTER TABLE options - `_ - - in the PostgreSQL documentation. .. _postgresql_table_valued_overview: @@ -1482,6 +1746,7 @@ def update(): import re from typing import Any from typing import cast +from typing import Dict from typing import List from typing import Optional from typing import Tuple @@ -1546,6 +1811,7 @@ def update(): from ...sql import compiler from ...sql import elements from ...sql import expression +from ...sql import functions from ...sql import roles from ...sql import sqltypes from ...sql import util as sql_util @@ -1672,6 +1938,7 @@ def update(): "verbose", } + colspecs = { sqltypes.ARRAY: _array.ARRAY, sqltypes.Interval: INTERVAL, @@ -1788,6 +2055,8 @@ def render_bind_cast(self, type_, dbapi_type, sqltext): }""" def visit_array(self, element, **kw): + if not element.clauses and not element.type.item_type._isnull: + return "ARRAY[]::%s" % element.type.compile(self.dialect) return "ARRAY[%s]" % self.visit_clauselist(element, **kw) def visit_slice(self, element, **kw): @@ -1811,9 +2080,27 @@ def visit_json_getitem_op_binary( kw["eager_grouping"] = True - return self._generate_generic_binary( - binary, " -> " if not _cast_applied else " ->> ", **kw - ) + if ( + not _cast_applied + and isinstance(binary.left.type, _json.JSONB) + and self.dialect._supports_jsonb_subscripting + ): + left = binary.left + if isinstance(left, (functions.FunctionElement, elements.Cast)): + left = elements.Grouping(left) + + # for pg14+JSONB use subscript notation: col['key'] instead + # of col -> 'key' + return "%s[%s]" % ( + self.process(left, **kw), + self.process(binary.right, **kw), + ) + else: + # Fall back to arrow notation for older versions or when cast + # is applied + return self._generate_generic_binary( + binary, " -> " if not _cast_applied else " ->> ", **kw + ) def visit_json_path_getitem_op_binary( self, binary, operator, _cast_applied=False, **kw @@ -1830,6 +2117,23 @@ def visit_json_path_getitem_op_binary( binary, " #> " if not _cast_applied else " #>> ", **kw ) + def visit_hstore_getitem_op_binary(self, binary, operator, **kw): + kw["eager_grouping"] = True + + if self.dialect._supports_jsonb_subscripting: + # use subscript notation: col['key'] instead of col -> 'key' + # For function calls, wrap in parentheses: (func())[key] + left_str = self.process(binary.left, **kw) + if isinstance(binary.left, sql.functions.FunctionElement): + left_str = f"({left_str})" + return "%s[%s]" % ( + left_str, + self.process(binary.right, **kw), + ) + else: + # Fall back to arrow notation for older versions + return self._generate_generic_binary(binary, " -> ", **kw) + def visit_getitem_binary(self, binary, operator, **kw): return "%s[%s]" % ( self.process(binary.left, **kw), @@ -1945,7 +2249,12 @@ def render_literal_value(self, value, type_): return value def visit_aggregate_strings_func(self, fn, **kw): - return "string_agg%s" % self.function_argspec(fn) + return super().visit_aggregate_strings_func( + fn, use_function_name="string_agg", **kw + ) + + def visit_pow_func(self, fn, **kw): + return f"power{self.function_argspec(fn)}" def visit_sequence(self, seq, **kw): return "nextval('%s')" % self.preparer.format_sequence(seq) @@ -1985,6 +2294,21 @@ def get_select_precolumns(self, select, **kw): else: return "" + def visit_postgresql_distinct_on(self, element, **kw): + if self.stack[-1]["selectable"]._distinct_on: + raise exc.CompileError( + "Cannot mix ``select.ext(distinct_on(...))`` and " + "``select.distinct(...)``" + ) + + if element._distinct_on: + cols = ", ".join( + self.process(col, **kw) for col in element._distinct_on + ) + return f"ON ({cols})" + else: + return None + def for_update_clause(self, select, **kw): if select._for_update_arg.read: if select._for_update_arg.key_share: @@ -2046,10 +2370,11 @@ def _on_conflict_target(self, clause, **kw): for c in clause.inferred_target_elements ) if clause.inferred_target_whereclause is not None: + whereclause_kw = dict(kw) + whereclause_kw.update(include_table=False, use_schema=False) target_text += " WHERE %s" % self.process( clause.inferred_target_whereclause, - include_table=False, - use_schema=False, + **whereclause_kw, ) else: target_text = "" @@ -2076,6 +2401,8 @@ def visit_on_conflict_do_update(self, on_conflict, **kw): insert_statement = self.stack[-1]["selectable"] cols = insert_statement.table.c + set_kw = dict(kw) + set_kw.update(use_schema=False) for c in cols: col_key = c.key @@ -2092,7 +2419,10 @@ def visit_on_conflict_do_update(self, on_conflict, **kw): and value.type._isnull ): value = value._with_binary_element_type(c.type) - value_text = self.process(value.self_group(), use_schema=False) + + value_text = self.process( + value.self_group(), is_upsert_set=True, **set_kw + ) key_text = self.preparer.quote(c.name) action_set_ops.append("%s = %s" % (key_text, value_text)) @@ -2115,14 +2445,17 @@ def visit_on_conflict_do_update(self, on_conflict, **kw): ) value_text = self.process( coercions.expect(roles.ExpressionElementRole, v), - use_schema=False, + is_upsert_set=True, + **set_kw, ) action_set_ops.append("%s = %s" % (key_text, value_text)) action_text = ", ".join(action_set_ops) if clause.update_whereclause is not None: + where_kw = dict(kw) + where_kw.update(include_table=True, use_schema=False) action_text += " WHERE %s" % self.process( - clause.update_whereclause, include_table=True, use_schema=False + clause.update_whereclause, **where_kw ) return "ON CONFLICT %s DO UPDATE SET %s" % (target_text, action_text) @@ -2227,6 +2560,18 @@ def _define_constraint_validity(self, constraint): not_valid = constraint.dialect_options["postgresql"]["not_valid"] return " NOT VALID" if not_valid else "" + def _define_include(self, obj): + includeclause = obj.dialect_options["postgresql"]["include"] + if not includeclause: + return "" + inclusions = [ + obj.table.c[col] if isinstance(col, str) else col + for col in includeclause + ] + return " INCLUDE (%s)" % ", ".join( + [self.preparer.quote(c.name) for c in inclusions] + ) + def visit_check_constraint(self, constraint, **kw): if constraint._type_bound: typ = list(constraint.columns)[0].type @@ -2250,6 +2595,35 @@ def visit_foreign_key_constraint(self, constraint, **kw): text += self._define_constraint_validity(constraint) return text + def visit_primary_key_constraint(self, constraint, **kw): + text = self.define_constraint_preamble(constraint, **kw) + text += self.define_primary_key_body(constraint, **kw) + text += self._define_include(constraint) + text += self.define_constraint_deferrability(constraint) + return text + + def visit_unique_constraint(self, constraint, **kw): + if len(constraint) == 0: + return "" + text = self.define_constraint_preamble(constraint, **kw) + text += self.define_unique_body(constraint, **kw) + text += self._define_include(constraint) + text += self.define_constraint_deferrability(constraint) + return text + + @util.memoized_property + def _fk_ondelete_pattern(self): + return re.compile( + r"^(?:RESTRICT|CASCADE|SET (?:NULL|DEFAULT)(?:\s*\(.+\))?" + r"|NO ACTION)$", + re.I, + ) + + def define_constraint_ondelete_cascade(self, constraint): + return " ON DELETE %s" % self.preparer.validate_sql_phrase( + constraint.ondelete, self._fk_ondelete_pattern + ) + def visit_create_enum_type(self, create, **kw): type_ = create.element @@ -2351,15 +2725,7 @@ def visit_create_index(self, create, **kw): ) ) - includeclause = index.dialect_options["postgresql"]["include"] - if includeclause: - inclusions = [ - index.table.c[col] if isinstance(col, str) else col - for col in includeclause - ] - text += " INCLUDE (%s)" % ", ".join( - [preparer.quote(c.name) for c in inclusions] - ) + text += self._define_include(index) nulls_not_distinct = index.dialect_options["postgresql"][ "nulls_not_distinct" @@ -2475,6 +2841,10 @@ def post_create_table(self, table): if pg_opts["using"]: table_opts.append("\n USING %s" % pg_opts["using"]) + if pg_opts["with"]: + storage_params = (f"{k} = {v}" for k, v in pg_opts["with"].items()) + table_opts.append(f" WITH ({', '.join(storage_params)})") + if pg_opts["with_oids"] is True: table_opts.append("\n WITH OIDS") elif pg_opts["with_oids"] is False: @@ -2493,12 +2863,24 @@ def post_create_table(self, table): return "".join(table_opts) def visit_computed_column(self, generated, **kw): + if self.dialect.supports_virtual_generated_columns: + return super().visit_computed_column(generated, **kw) if generated.persisted is False: raise exc.CompileError( "PostrgreSQL computed columns do not support 'virtual' " "persistence; set the 'persisted' flag to None or True for " "PostgreSQL support." ) + elif generated.persisted is None: + util.warn( + f"Computed column {generated.column} is being created as " + "'STORED' since the current PostgreSQL version does not " + "support VIRTUAL columns. On PostgreSQL 18+, when " + "'persisted' is not " + "specified, no keyword will be rendered and VIRTUAL will be " + "used by default. Set 'persisted=True' to ensure STORED " + "behavior across all PostgreSQL versions." + ) return "GENERATED ALWAYS AS (%s) STORED" % self.sql_compiler.process( generated.sqltext, include_table=False, literal_binds=True @@ -2753,6 +3135,22 @@ def format_type(self, type_, use_schema=True): name = self.quote(type_.name) effective_schema = self.schema_for_object(type_) + # a built-in type with the same name will obscure this type, so raise + # for that case. this applies really to any visible type with the same + # name in any other visible schema that would not be appropriate for + # us to check against, so this is not a robust check, but + # at least do something for an obvious built-in name conflict + if ( + effective_schema is None + and type_.name in self.dialect.ischema_names + ): + raise exc.CompileError( + f"{type_!r} has name " + f"'{type_.name}' that matches an existing type, and " + "requires an explicit schema name in order to be rendered " + "in DDL." + ) + if ( not self.omit_schema and use_schema @@ -2845,8 +3243,8 @@ def get_domains( * nullable - Indicates if this domain can be ``NULL``. * default - The default value of the domain or ``None`` if the domain has no default. - * constraints - A list of dict wit the constraint defined by this - domain. Each element constaints two keys: ``name`` of the + * constraints - A list of dict with the constraint defined by this + domain. Each element contains two keys: ``name`` of the constraint and ``check`` with the constraint text. :param schema: schema name. If None, the default schema @@ -3016,6 +3414,7 @@ class PGDialect(default.DefaultDialect): supports_native_boolean = True supports_native_uuid = True supports_smallserial = True + supports_virtual_generated_columns = True supports_sequences = True sequences_optional = True @@ -3090,6 +3489,7 @@ class PGDialect(default.DefaultDialect): "tablespace": None, "partition_by": None, "with_oids": None, + "with": None, "on_commit": None, "inherits": None, "using": None, @@ -3107,9 +3507,16 @@ class PGDialect(default.DefaultDialect): "not_valid": False, }, ), + ( + schema.PrimaryKeyConstraint, + {"include": None}, + ), ( schema.UniqueConstraint, - {"nulls_not_distinct": None}, + { + "include": None, + "nulls_not_distinct": None, + }, ), ] @@ -3118,6 +3525,8 @@ class PGDialect(default.DefaultDialect): _backslash_escapes = True _supports_create_index_concurrently = True _supports_drop_index_concurrently = True + _supports_jsonb_subscripting = True + _pg_am_btree_oid = -1 def __init__( self, @@ -3146,6 +3555,12 @@ def initialize(self, connection): ) self.supports_identity_columns = self.server_version_info >= (10,) + self._supports_jsonb_subscripting = self.server_version_info >= (14,) + + self.supports_virtual_generated_columns = self.server_version_info >= ( + 18, + ) + def get_isolation_level_values(self, dbapi_conn): # note the generic dialect doesn't have AUTOCOMMIT, however # all postgresql dialects should include AUTOCOMMIT. @@ -3272,7 +3687,11 @@ def do_begin_twophase(self, connection, xid): self.do_begin(connection.connection) def do_prepare_twophase(self, connection, xid): - connection.exec_driver_sql("PREPARE TRANSACTION '%s'" % xid) + connection.execute( + sql.text("PREPARE TRANSACTION :xid'").bindparams( + sql.bindparam("xid", xid, literal_execute=True) + ) + ) def do_rollback_twophase( self, connection, xid, is_prepared=True, recover=False @@ -3284,7 +3703,11 @@ def do_rollback_twophase( # Must find out a way how to make the dbapi not # open a transaction. connection.exec_driver_sql("ROLLBACK") - connection.exec_driver_sql("ROLLBACK PREPARED '%s'" % xid) + connection.execute( + sql.text("ROLLBACK PREPARED :xid").bindparams( + sql.bindparam("xid", xid, literal_execute=True) + ) + ) connection.exec_driver_sql("BEGIN") self.do_rollback(connection.connection) else: @@ -3296,7 +3719,11 @@ def do_commit_twophase( if is_prepared: if recover: connection.exec_driver_sql("ROLLBACK") - connection.exec_driver_sql("COMMIT PREPARED '%s'" % xid) + connection.execute( + sql.text("COMMIT PREPARED :xid").bindparams( + sql.bindparam("xid", xid, literal_execute=True) + ) + ) connection.exec_driver_sql("BEGIN") self.do_rollback(connection.connection) else: @@ -3556,6 +3983,130 @@ def _kind_to_relkinds(self, kind: ObjectKind) -> Tuple[str, ...]: relkinds += pg_catalog.RELKINDS_MAT_VIEW return relkinds + @reflection.cache + def get_table_options(self, connection, table_name, schema=None, **kw): + data = self.get_multi_table_options( + connection, + schema=schema, + filter_names=[table_name], + scope=ObjectScope.ANY, + kind=ObjectKind.ANY, + **kw, + ) + return self._value_or_raise(data, table_name, schema) + + @lru_cache() + def _table_options_query(self, schema, has_filter_names, scope, kind): + inherits_sq = ( + select( + pg_catalog.pg_inherits.c.inhrelid, + sql.func.array_agg( + aggregate_order_by( + pg_catalog.pg_class.c.relname, + pg_catalog.pg_inherits.c.inhseqno, + ) + ).label("parent_table_names"), + ) + .select_from(pg_catalog.pg_inherits) + .join( + pg_catalog.pg_class, + pg_catalog.pg_inherits.c.inhparent + == pg_catalog.pg_class.c.oid, + ) + .group_by(pg_catalog.pg_inherits.c.inhrelid) + .subquery("inherits") + ) + + if self.server_version_info < (12,): + # this is not in the pg_catalog.pg_class since it was + # removed in PostgreSQL version 12 + has_oids = sql.column("relhasoids", BOOLEAN) + else: + has_oids = sql.null().label("relhasoids") + + relkinds = self._kind_to_relkinds(kind) + query = ( + select( + pg_catalog.pg_class.c.oid, + pg_catalog.pg_class.c.relname, + pg_catalog.pg_class.c.reloptions, + has_oids, + sql.case( + ( + sql.and_( + pg_catalog.pg_am.c.amname.is_not(None), + pg_catalog.pg_am.c.amname + != sql.func.current_setting( + "default_table_access_method" + ), + ), + pg_catalog.pg_am.c.amname, + ), + else_=sql.null(), + ).label("access_method_name"), + pg_catalog.pg_tablespace.c.spcname.label("tablespace_name"), + inherits_sq.c.parent_table_names, + ) + .select_from(pg_catalog.pg_class) + .outerjoin( + # NOTE: on postgresql < 12, this could be avoided + # since relam is always 0 so nothing is joined. + pg_catalog.pg_am, + pg_catalog.pg_class.c.relam == pg_catalog.pg_am.c.oid, + ) + .outerjoin( + inherits_sq, + pg_catalog.pg_class.c.oid == inherits_sq.c.inhrelid, + ) + .outerjoin( + pg_catalog.pg_tablespace, + pg_catalog.pg_tablespace.c.oid + == pg_catalog.pg_class.c.reltablespace, + ) + .where(self._pg_class_relkind_condition(relkinds)) + ) + query = self._pg_class_filter_scope_schema(query, schema, scope=scope) + if has_filter_names: + query = query.where( + pg_catalog.pg_class.c.relname.in_(bindparam("filter_names")) + ) + return query + + def get_multi_table_options( + self, connection, schema, filter_names, scope, kind, **kw + ): + has_filter_names, params = self._prepare_filter_names(filter_names) + query = self._table_options_query( + schema, has_filter_names, scope, kind + ) + rows = connection.execute(query, params).mappings() + table_options = {} + + for row in rows: + current: dict[str, Any] = {} + if row["access_method_name"] is not None: + current["postgresql_using"] = row["access_method_name"] + + if row["parent_table_names"]: + current["postgresql_inherits"] = tuple( + row["parent_table_names"] + ) + + if row["reloptions"]: + current["postgresql_with"] = dict( + option.split("=", 1) for option in row["reloptions"] + ) + + if row["relhasoids"]: + current["postgresql_with_oids"] = True + + if row["tablespace_name"] is not None: + current["postgresql_tablespace"] = row["tablespace_name"] + + table_options[(schema, row["relname"])] = current + + return table_options.items() + @reflection.cache def get_columns(self, connection, table_name, schema=None, **kw): data = self.get_multi_columns( @@ -3579,76 +4130,111 @@ def _columns_query(self, schema, has_filter_names, scope, kind): ) if self.server_version_info >= (10,): # join lateral performs worse (~2x slower) than a scalar_subquery - identity = ( - select( - sql.func.json_build_object( - "always", - pg_catalog.pg_attribute.c.attidentity == "a", - "start", - pg_catalog.pg_sequence.c.seqstart, - "increment", - pg_catalog.pg_sequence.c.seqincrement, - "minvalue", - pg_catalog.pg_sequence.c.seqmin, - "maxvalue", - pg_catalog.pg_sequence.c.seqmax, - "cache", - pg_catalog.pg_sequence.c.seqcache, - "cycle", - pg_catalog.pg_sequence.c.seqcycle, - type_=sqltypes.JSON(), - ) - ) - .select_from(pg_catalog.pg_sequence) - .where( - # attidentity != '' is required or it will reflect also + # also the subquery can be run only if the column is an identity + identity = sql.case( + ( # attidentity != '' is required or it will reflect also # serial columns as identity. pg_catalog.pg_attribute.c.attidentity != "", - pg_catalog.pg_sequence.c.seqrelid - == sql.cast( - sql.cast( - pg_catalog.pg_get_serial_sequence( - sql.cast( + select( + sql.func.json_build_object( + "always", + pg_catalog.pg_attribute.c.attidentity == "a", + "start", + pg_catalog.pg_sequence.c.seqstart, + "increment", + pg_catalog.pg_sequence.c.seqincrement, + "minvalue", + pg_catalog.pg_sequence.c.seqmin, + "maxvalue", + pg_catalog.pg_sequence.c.seqmax, + "cache", + pg_catalog.pg_sequence.c.seqcache, + "cycle", + pg_catalog.pg_sequence.c.seqcycle, + type_=sqltypes.JSON(), + ) + ) + .select_from(pg_catalog.pg_sequence) + .where( + # not needed but pg seems to like it + pg_catalog.pg_attribute.c.attidentity != "", + pg_catalog.pg_sequence.c.seqrelid + == sql.cast( + sql.cast( + pg_catalog.pg_get_serial_sequence( sql.cast( - pg_catalog.pg_attribute.c.attrelid, - REGCLASS, + sql.cast( + pg_catalog.pg_attribute.c.attrelid, + REGCLASS, + ), + TEXT, ), - TEXT, + pg_catalog.pg_attribute.c.attname, ), - pg_catalog.pg_attribute.c.attname, + REGCLASS, ), - REGCLASS, + OID, ), - OID, - ), - ) - .correlate(pg_catalog.pg_attribute) - .scalar_subquery() - .label("identity_options") - ) + ) + .correlate(pg_catalog.pg_attribute) + .scalar_subquery(), + ), + else_=sql.null(), + ).label("identity_options") else: identity = sql.null().label("identity_options") - # join lateral performs the same as scalar_subquery here - default = ( - select( - pg_catalog.pg_get_expr( - pg_catalog.pg_attrdef.c.adbin, - pg_catalog.pg_attrdef.c.adrelid, - ) - ) - .select_from(pg_catalog.pg_attrdef) - .where( - pg_catalog.pg_attrdef.c.adrelid - == pg_catalog.pg_attribute.c.attrelid, - pg_catalog.pg_attrdef.c.adnum - == pg_catalog.pg_attribute.c.attnum, + # join lateral performs the same as scalar_subquery here, also + # the subquery can be run only if the column has a default + default = sql.case( + ( pg_catalog.pg_attribute.c.atthasdef, - ) - .correlate(pg_catalog.pg_attribute) - .scalar_subquery() - .label("default") - ) + select( + pg_catalog.pg_get_expr( + pg_catalog.pg_attrdef.c.adbin, + pg_catalog.pg_attrdef.c.adrelid, + ) + ) + .select_from(pg_catalog.pg_attrdef) + .where( + # not needed but pg seems to like it + pg_catalog.pg_attribute.c.atthasdef, + pg_catalog.pg_attrdef.c.adrelid + == pg_catalog.pg_attribute.c.attrelid, + pg_catalog.pg_attrdef.c.adnum + == pg_catalog.pg_attribute.c.attnum, + ) + .correlate(pg_catalog.pg_attribute) + .scalar_subquery(), + ), + else_=sql.null(), + ).label("default") + + # get the name of the collate when it's different from the default one + collate = sql.case( + ( + sql.and_( + pg_catalog.pg_attribute.c.attcollation != 0, + select(pg_catalog.pg_type.c.typcollation) + .where( + pg_catalog.pg_type.c.oid + == pg_catalog.pg_attribute.c.atttypid, + ) + .correlate(pg_catalog.pg_attribute) + .scalar_subquery() + != pg_catalog.pg_attribute.c.attcollation, + ), + select(pg_catalog.pg_collation.c.collname) + .where( + pg_catalog.pg_collation.c.oid + == pg_catalog.pg_attribute.c.attcollation + ) + .correlate(pg_catalog.pg_attribute) + .scalar_subquery(), + ), + else_=sql.null(), + ).label("collation") + relkinds = self._kind_to_relkinds(kind) query = ( select( @@ -3663,6 +4249,7 @@ def _columns_query(self, schema, has_filter_names, scope, kind): pg_catalog.pg_description.c.description.label("comment"), generated, identity, + collate, ) .select_from(pg_catalog.pg_class) # NOTE: postgresql support table with no user column, meaning @@ -3705,29 +4292,8 @@ def get_multi_columns( query = self._columns_query(schema, has_filter_names, scope, kind) rows = connection.execute(query, params).mappings() - # dictionary with (name, ) if default search path or (schema, name) - # as keys - domains = { - ((d["schema"], d["name"]) if not d["visible"] else (d["name"],)): d - for d in self._load_domains( - connection, schema="*", info_cache=kw.get("info_cache") - ) - } - - # dictionary with (name, ) if default search path or (schema, name) - # as keys - enums = dict( - ( - ((rec["name"],), rec) - if rec["visible"] - else ((rec["schema"], rec["name"]), rec) - ) - for rec in self._load_enums( - connection, schema="*", info_cache=kw.get("info_cache") - ) - ) - - columns = self._get_columns_info(rows, domains, enums, schema) + named_type_loader = _NamedTypeLoader(self, connection, kw) + columns = self._get_columns_info(rows, named_type_loader, schema) return columns.items() @@ -3738,9 +4304,9 @@ def get_multi_columns( def _reflect_type( self, format_type: Optional[str], - domains: dict[str, ReflectedDomain], - enums: dict[str, ReflectedEnum], + named_type_loader: _NamedTypeLoader, type_description: str, + collation: Optional[str], ) -> sqltypes.TypeEngine[Any]: """ Attempts to reconstruct a column type defined in ischema_names based @@ -3809,7 +4375,8 @@ def _reflect_type( charlen = int(attype_args[0]) args = (charlen,) - elif attype.startswith("interval"): + # a domain or enum can start with interval, so be mindful of that. + elif attype == "interval" or attype.startswith("interval "): schema_type = INTERVAL field_match = re.match(r"interval (.+)", attype) @@ -3822,25 +4389,30 @@ def _reflect_type( else: enum_or_domain_key = tuple(util.quoted_token_parser(attype)) - if enum_or_domain_key in enums: + if ( + schema_type is None + and enum_or_domain_key in named_type_loader.enums + ): schema_type = ENUM - enum = enums[enum_or_domain_key] + enum = named_type_loader.enums[enum_or_domain_key] - args = tuple(enum["labels"]) kwargs["name"] = enum["name"] if not enum["visible"]: kwargs["schema"] = enum["schema"] args = tuple(enum["labels"]) - elif enum_or_domain_key in domains: + elif ( + schema_type is None + and enum_or_domain_key in named_type_loader.domains + ): schema_type = DOMAIN - domain = domains[enum_or_domain_key] + domain = named_type_loader.domains[enum_or_domain_key] data_type = self._reflect_type( domain["type"], - domains, - enums, + named_type_loader, type_description="DOMAIN '%s'" % domain["name"], + collation=domain["collation"], ) args = (domain["name"], data_type) @@ -3873,6 +4445,9 @@ def _reflect_type( ) return sqltypes.NULLTYPE + if collation is not None: + kwargs["collation"] = collation + data_type = schema_type(*args, **kwargs) if array_dim >= 1: # postgres does not preserve dimensionality or size of array types. @@ -3880,7 +4455,7 @@ def _reflect_type( return data_type - def _get_columns_info(self, rows, domains, enums, schema): + def _get_columns_info(self, rows, named_type_loader, schema): columns = defaultdict(list) for row_dict in rows: # ensure that each table has an entry, even if it has no columns @@ -3891,11 +4466,13 @@ def _get_columns_info(self, rows, domains, enums, schema): continue table_cols = columns[(schema, row_dict["table_name"])] + collation = row_dict["collation"] + coltype = self._reflect_type( row_dict["format_type"], - domains, - enums, + named_type_loader, type_description="column '%s'" % row_dict["name"], + collation=collation, ) default = row_dict["default"] @@ -3906,7 +4483,7 @@ def _get_columns_info(self, rows, domains, enums, schema): if isinstance(coltype, DOMAIN): if not default: # domain can override the default value but - # cant set it to None + # can't set it to None if coltype.default is not None: default = coltype.default @@ -3916,8 +4493,7 @@ def _get_columns_info(self, rows, domains, enums, schema): # If a zero byte or blank string depending on driver (is also # absent for older PG versions), then not a generated column. - # Otherwise, s = stored. (Other values might be added in the - # future.) + # Otherwise, s = stored, v = virtual. if generated not in (None, "", b"\x00"): computed = dict( sqltext=default, persisted=generated in ("s", b"s") @@ -3991,21 +4567,35 @@ def _get_table_oids( result = connection.execute(oid_q, params) return result.all() - @lru_cache() - def _constraint_query(self, is_unique): + @util.memoized_property + def _constraint_query(self): + if self.server_version_info >= (11, 0): + indnkeyatts = pg_catalog.pg_index.c.indnkeyatts + else: + indnkeyatts = pg_catalog.pg_index.c.indnatts.label("indnkeyatts") + + if self.server_version_info >= (15,): + indnullsnotdistinct = pg_catalog.pg_index.c.indnullsnotdistinct + else: + indnullsnotdistinct = sql.false().label("indnullsnotdistinct") + con_sq = ( select( pg_catalog.pg_constraint.c.conrelid, pg_catalog.pg_constraint.c.conname, - pg_catalog.pg_constraint.c.conindid, - sql.func.unnest(pg_catalog.pg_constraint.c.conkey).label( - "attnum" - ), + sql.func.unnest(pg_catalog.pg_index.c.indkey).label("attnum"), sql.func.generate_subscripts( - pg_catalog.pg_constraint.c.conkey, 1 + pg_catalog.pg_index.c.indkey, 1 ).label("ord"), + indnkeyatts, + indnullsnotdistinct, pg_catalog.pg_description.c.description, ) + .join( + pg_catalog.pg_index, + pg_catalog.pg_constraint.c.conindid + == pg_catalog.pg_index.c.indexrelid, + ) .outerjoin( pg_catalog.pg_description, pg_catalog.pg_description.c.objoid @@ -4014,6 +4604,9 @@ def _constraint_query(self, is_unique): .where( pg_catalog.pg_constraint.c.contype == bindparam("contype"), pg_catalog.pg_constraint.c.conrelid.in_(bindparam("oids")), + # NOTE: filtering also on pg_index.indrelid for oids does + # not seem to have a performance effect, but it may be an + # option if perf problems are reported ) .subquery("con") ) @@ -4022,9 +4615,10 @@ def _constraint_query(self, is_unique): select( con_sq.c.conrelid, con_sq.c.conname, - con_sq.c.conindid, con_sq.c.description, con_sq.c.ord, + con_sq.c.indnkeyatts, + con_sq.c.indnullsnotdistinct, pg_catalog.pg_attribute.c.attname, ) .select_from(pg_catalog.pg_attribute) @@ -4041,13 +4635,13 @@ def _constraint_query(self, is_unique): # a sequential scan of pg_attribute. # The condition in the con_sq subquery is not actually needed # in pg15, but it may be needed in older versions. Keeping it - # does not seems to have any inpact in any case. + # does not seems to have any impact in any case. con_sq.c.conrelid.in_(bindparam("oids")) ) .subquery("attr") ) - constraint_query = ( + return ( select( attr_sq.c.conrelid, sql.func.array_agg( @@ -4059,31 +4653,15 @@ def _constraint_query(self, is_unique): ).label("cols"), attr_sq.c.conname, sql.func.min(attr_sq.c.description).label("description"), + sql.func.min(attr_sq.c.indnkeyatts).label("indnkeyatts"), + sql.func.bool_and(attr_sq.c.indnullsnotdistinct).label( + "indnullsnotdistinct" + ), ) .group_by(attr_sq.c.conrelid, attr_sq.c.conname) .order_by(attr_sq.c.conrelid, attr_sq.c.conname) ) - if is_unique: - if self.server_version_info >= (15,): - constraint_query = constraint_query.join( - pg_catalog.pg_index, - attr_sq.c.conindid == pg_catalog.pg_index.c.indexrelid, - ).add_columns( - sql.func.bool_and( - pg_catalog.pg_index.c.indnullsnotdistinct - ).label("indnullsnotdistinct") - ) - else: - constraint_query = constraint_query.add_columns( - sql.false().label("indnullsnotdistinct") - ) - else: - constraint_query = constraint_query.add_columns( - sql.null().label("extra") - ) - return constraint_query - def _reflect_constraint( self, connection, contype, schema, filter_names, scope, kind, **kw ): @@ -4099,26 +4677,42 @@ def _reflect_constraint( batches[0:3000] = [] result = connection.execute( - self._constraint_query(is_unique), + self._constraint_query, {"oids": [r[0] for r in batch], "contype": contype}, - ) + ).mappings() result_by_oid = defaultdict(list) - for oid, cols, constraint_name, comment, extra in result: - result_by_oid[oid].append( - (cols, constraint_name, comment, extra) - ) + for row_dict in result: + result_by_oid[row_dict["conrelid"]].append(row_dict) for oid, tablename in batch: for_oid = result_by_oid.get(oid, ()) if for_oid: - for cols, constraint, comment, extra in for_oid: - if is_unique: - yield tablename, cols, constraint, comment, { - "nullsnotdistinct": extra - } + for row in for_oid: + # See note in get_multi_indexes + all_cols = row["cols"] + indnkeyatts = row["indnkeyatts"] + if len(all_cols) > indnkeyatts: + inc_cols = all_cols[indnkeyatts:] + cst_cols = all_cols[:indnkeyatts] else: - yield tablename, cols, constraint, comment, None + inc_cols = [] + cst_cols = all_cols + + opts = {} + if self.server_version_info >= (11,): + opts["postgresql_include"] = inc_cols + if is_unique: + opts["postgresql_nulls_not_distinct"] = row[ + "indnullsnotdistinct" + ] + yield ( + tablename, + cst_cols, + row["conname"], + row["description"], + opts, + ) else: yield tablename, None, None, None, None @@ -4144,20 +4738,27 @@ def get_multi_pk_constraint( # only a single pk can be present for each table. Return an entry # even if a table has no primary key default = ReflectionDefaults.pk_constraint + + def pk_constraint(pk_name, cols, comment, opts): + info = { + "constrained_columns": cols, + "name": pk_name, + "comment": comment, + } + if opts: + info["dialect_options"] = opts + return info + return ( ( (schema, table_name), ( - { - "constrained_columns": [] if cols is None else cols, - "name": pk_name, - "comment": comment, - } + pk_constraint(pk_name, cols, comment, opts) if pk_name is not None else default() ), ) - for table_name, cols, pk_name, comment, _ in result + for table_name, cols, pk_name, comment, opts in result ) @reflection.cache @@ -4241,21 +4842,66 @@ def _foreing_key_query(self, schema, has_filter_names, scope, kind): @util.memoized_property def _fk_regex_pattern(self): # optionally quoted token - qtoken = '(?:"[^"]+"|[A-Za-z0-9_]+?)' + qtoken = r'(?:"(?:[^"]|"")+"|[\w]+?)' # https://www.postgresql.org/docs/current/static/sql-createtable.html return re.compile( r"FOREIGN KEY \((.*?)\) " rf"REFERENCES (?:({qtoken})\.)?({qtoken})\(((?:{qtoken}(?: *, *)?)+)\)" # noqa: E501 r"[\s]?(MATCH (FULL|PARTIAL|SIMPLE)+)?" - r"[\s]?(ON UPDATE " - r"(CASCADE|RESTRICT|NO ACTION|SET NULL|SET DEFAULT)+)?" - r"[\s]?(ON DELETE " - r"(CASCADE|RESTRICT|NO ACTION|SET NULL|SET DEFAULT)+)?" + r"[\s]?(?:ON (UPDATE|DELETE) " + r"(CASCADE|RESTRICT|NO ACTION|" + r"SET (?:NULL|DEFAULT)(?:\s\(.+\))?)+)?" + r"[\s]?(?:ON (UPDATE|DELETE) " + r"(CASCADE|RESTRICT|NO ACTION|" + r"SET (?:NULL|DEFAULT)(?:\s\(.+\))?)+)?" r"[\s]?(DEFERRABLE|NOT DEFERRABLE)?" r"[\s]?(INITIALLY (DEFERRED|IMMEDIATE)+)?" ) + def _parse_fk(self, condef): + FK_REGEX = self._fk_regex_pattern + m = re.search(FK_REGEX, condef).groups() + + ( + constrained_columns, + referred_schema, + referred_table, + referred_columns, + _, + match, + upddelkey1, + upddelval1, + upddelkey2, + upddelval2, + deferrable, + _, + initially, + ) = m + + onupdate = ( + upddelval1 + if upddelkey1 == "UPDATE" + else upddelval2 if upddelkey2 == "UPDATE" else None + ) + ondelete = ( + upddelval1 + if upddelkey1 == "DELETE" + else upddelval2 if upddelkey2 == "DELETE" else None + ) + + return ( + constrained_columns, + referred_schema, + referred_table, + referred_columns, + match, + onupdate, + ondelete, + deferrable, + initially, + ) + def get_multi_foreign_keys( self, connection, @@ -4272,8 +4918,6 @@ def get_multi_foreign_keys( query = self._foreing_key_query(schema, has_filter_names, scope, kind) result = connection.execute(query, params) - FK_REGEX = self._fk_regex_pattern - fkeys = defaultdict(list) default = ReflectionDefaults.foreign_keys for table_name, conname, condef, conschema, comment in result: @@ -4283,23 +4927,18 @@ def get_multi_foreign_keys( fkeys[(schema, table_name)] = default() continue table_fks = fkeys[(schema, table_name)] - m = re.search(FK_REGEX, condef).groups() ( constrained_columns, referred_schema, referred_table, referred_columns, - _, match, - _, onupdate, - _, ondelete, deferrable, - _, initially, - ) = m + ) = self._parse_fk(condef) if deferrable is not None: deferrable = True if deferrable == "DEFERRABLE" else False @@ -4367,7 +5006,10 @@ def get_indexes(self, connection, table_name, schema=None, **kw): @util.memoized_property def _index_query(self): - pg_class_index = pg_catalog.pg_class.alias("cls_idx") + # NOTE: pg_index is used as from two times to improve performance, + # since extraing all the index information from `idx_sq` to avoid + # the second pg_index use leads to a worse performing query in + # particular when querying for a single table (as of pg 17) # NOTE: repeating oids clause improve query performance # subquery to get the columns @@ -4376,6 +5018,9 @@ def _index_query(self): pg_catalog.pg_index.c.indexrelid, pg_catalog.pg_index.c.indrelid, sql.func.unnest(pg_catalog.pg_index.c.indkey).label("attnum"), + sql.func.unnest(pg_catalog.pg_index.c.indclass).label( + "att_opclass" + ), sql.func.generate_subscripts( pg_catalog.pg_index.c.indkey, 1 ).label("ord"), @@ -4407,6 +5052,10 @@ def _index_query(self): else_=pg_catalog.pg_attribute.c.attname.cast(TEXT), ).label("element"), (idx_sq.c.attnum == 0).label("is_expr"), + # since it's converted to array cast it to bigint (oid are + # "unsigned four-byte integer") to make it easier for + # dialects to interpret + idx_sq.c.att_opclass.cast(BIGINT), ) .select_from(idx_sq) .outerjoin( @@ -4431,6 +5080,9 @@ def _index_query(self): sql.func.array_agg( aggregate_order_by(attr_sq.c.is_expr, attr_sq.c.ord) ).label("elements_is_expr"), + sql.func.array_agg( + aggregate_order_by(attr_sq.c.att_opclass, attr_sq.c.ord) + ).label("elements_opclass"), ) .group_by(attr_sq.c.indexrelid) .subquery("idx_cols") @@ -4439,7 +5091,7 @@ def _index_query(self): if self.server_version_info >= (11, 0): indnkeyatts = pg_catalog.pg_index.c.indnkeyatts else: - indnkeyatts = sql.null().label("indnkeyatts") + indnkeyatts = pg_catalog.pg_index.c.indnatts.label("indnkeyatts") if self.server_version_info >= (15,): nulls_not_distinct = pg_catalog.pg_index.c.indnullsnotdistinct @@ -4449,14 +5101,15 @@ def _index_query(self): return ( select( pg_catalog.pg_index.c.indrelid, - pg_class_index.c.relname.label("relname_index"), + pg_catalog.pg_class.c.relname, pg_catalog.pg_index.c.indisunique, pg_catalog.pg_constraint.c.conrelid.is_not(None).label( "has_constraint" ), pg_catalog.pg_index.c.indoption, - pg_class_index.c.reloptions, - pg_catalog.pg_am.c.amname, + pg_catalog.pg_class.c.reloptions, + # will get the value using the pg_am cached dict + pg_catalog.pg_class.c.relam, # NOTE: pg_get_expr is very fast so this case has almost no # performance impact sql.case( @@ -4473,6 +5126,8 @@ def _index_query(self): nulls_not_distinct, cols_sq.c.elements, cols_sq.c.elements_is_expr, + # will get the value using the pg_opclass cached dict + cols_sq.c.elements_opclass, ) .select_from(pg_catalog.pg_index) .where( @@ -4480,12 +5135,8 @@ def _index_query(self): ~pg_catalog.pg_index.c.indisprimary, ) .join( - pg_class_index, - pg_catalog.pg_index.c.indexrelid == pg_class_index.c.oid, - ) - .join( - pg_catalog.pg_am, - pg_class_index.c.relam == pg_catalog.pg_am.c.oid, + pg_catalog.pg_class, + pg_catalog.pg_index.c.indexrelid == pg_catalog.pg_class.c.oid, ) .outerjoin( cols_sq, @@ -4502,7 +5153,9 @@ def _index_query(self): == sql.any_(_array.array(("p", "u", "x"))), ), ) - .order_by(pg_catalog.pg_index.c.indrelid, pg_class_index.c.relname) + .order_by( + pg_catalog.pg_index.c.indrelid, pg_catalog.pg_class.c.relname + ) ) def get_multi_indexes( @@ -4512,6 +5165,14 @@ def get_multi_indexes( connection, schema, filter_names, scope, kind, **kw ) + pg_am_btree_oid = self._load_pg_am_btree_oid(connection) + # lazy load only if needed, the assumption is that most indexes + # will use btree so it may not be needed at all + pg_am_dict = None + pg_opclass_dict = self._load_pg_opclass_notdefault_dict( + connection, **kw + ) + indexes = defaultdict(list) default = ReflectionDefaults.indexes @@ -4537,17 +5198,18 @@ def get_multi_indexes( continue for row in result_by_oid[oid]: - index_name = row["relname_index"] + index_name = row["relname"] table_indexes = indexes[(schema, table_name)] all_elements = row["elements"] all_elements_is_expr = row["elements_is_expr"] + all_elements_opclass = row["elements_opclass"] indnkeyatts = row["indnkeyatts"] # "The number of key columns in the index, not counting any # included columns, which are merely stored and do not # participate in the index semantics" - if indnkeyatts and len(all_elements) > indnkeyatts: + if len(all_elements) > indnkeyatts: # this is a "covering index" which has INCLUDE columns # as well as regular index columns inc_cols = all_elements[indnkeyatts:] @@ -4562,10 +5224,14 @@ def get_multi_indexes( not is_expr for is_expr in all_elements_is_expr[indnkeyatts:] ) + idx_elements_opclass = all_elements_opclass[ + :indnkeyatts + ] else: idx_elements = all_elements idx_elements_is_expr = all_elements_is_expr inc_cols = [] + idx_elements_opclass = all_elements_opclass index = {"name": index_name, "unique": row["indisunique"]} if any(idx_elements_is_expr): @@ -4579,6 +5245,20 @@ def get_multi_indexes( else: index["column_names"] = idx_elements + dialect_options = {} + + postgresql_ops = {} + for name, opclass in zip( + idx_elements, idx_elements_opclass + ): + # is not in the dict if the opclass is the default one + opclass_name = pg_opclass_dict.get(opclass) + if opclass_name is not None: + postgresql_ops[name] = opclass_name + + if postgresql_ops: + dialect_options["postgresql_ops"] = postgresql_ops + sorting = {} for col_index, col_flags in enumerate(row["indoption"]): col_sorting = () @@ -4598,7 +5278,6 @@ def get_multi_indexes( if row["has_constraint"]: index["duplicates_constraint"] = index_name - dialect_options = {} if row["reloptions"]: dialect_options["postgresql_with"] = dict( [ @@ -4610,9 +5289,14 @@ def get_multi_indexes( # reflection info. But we don't want an Index object # to have a ``postgresql_using`` in it that is just the # default, so for the moment leaving this out. - amname = row["amname"] - if amname != "btree": - dialect_options["postgresql_using"] = row["amname"] + if row["relam"] != pg_am_btree_oid: + if pg_am_dict is None: + pg_am_dict = self._load_pg_am_dict( + connection, **kw + ) + dialect_options["postgresql_using"] = pg_am_dict[ + row["relam"] + ] if row["filter_definition"]: dialect_options["postgresql_where"] = row[ "filter_definition" @@ -4666,7 +5350,7 @@ def get_multi_unique_constraints( default = ReflectionDefaults.unique_constraints for table_name, cols, con_name, comment, options in result: # ensure a list is created for each table. leave it empty if - # the table has no unique cosntraint + # the table has no unique constraint if con_name is None: uniques[(schema, table_name)] = default() continue @@ -4677,12 +5361,7 @@ def get_multi_unique_constraints( "comment": comment, } if options: - if options["nullsnotdistinct"]: - uc_dict["dialect_options"] = { - "postgresql_nulls_not_distinct": options[ - "nullsnotdistinct" - ] - } + uc_dict["dialect_options"] = options uniques[(schema, table_name)].append(uc_dict) return uniques.items() @@ -5026,6 +5705,36 @@ def _load_domains(self, connection, schema=None, **kw): return domains + @util.memoized_property + def _pg_am_query(self): + return sql.select(pg_catalog.pg_am.c.oid, pg_catalog.pg_am.c.amname) + + @reflection.cache + def _load_pg_am_dict(self, connection, **kw) -> dict[int, str]: + rows = connection.execute(self._pg_am_query) + return dict(rows.all()) + + def _load_pg_am_btree_oid(self, connection): + # this oid is assumed to be stable + if self._pg_am_btree_oid == -1: + self._pg_am_btree_oid = connection.scalar( + self._pg_am_query.where(pg_catalog.pg_am.c.amname == "btree") + ) + return self._pg_am_btree_oid + + @util.memoized_property + def _pg_opclass_notdefault_query(self): + return sql.select( + pg_catalog.pg_opclass.c.oid, pg_catalog.pg_opclass.c.opcname + ).where(~pg_catalog.pg_opclass.c.opcdefault) + + @reflection.cache + def _load_pg_opclass_notdefault_dict( + self, connection, **kw + ) -> dict[int, str]: + rows = connection.execute(self._pg_opclass_notdefault_query) + return dict(rows.all()) + def _set_backslash_escapes(self, connection): # this method is provided as an override hook for descendant # dialects (e.g. Redshift), so removing it may break them @@ -5033,3 +5742,48 @@ def _set_backslash_escapes(self, connection): "show standard_conforming_strings" ).scalar() self._backslash_escapes = std_string == "off" + + +class _NamedTypeLoader: + """Helper class used for deferred loading of named types (enums, domains) + only when needed. + """ + + def __init__( + self, dialect: PGDialect, connection, kw: Dict[str, Any] + ) -> None: + self.dialect = dialect + self.connection = connection + self.kw = kw + + @util.memoized_property + def enums(self) -> Dict[Tuple[str] | Tuple[str, str], ReflectedEnum]: + # dictionary with (name, ) if default search path or (schema, name) + # as keys + enums = dict( + ( + ((rec["name"],), rec) + if rec["visible"] + else ((rec["schema"], rec["name"]), rec) + ) + for rec in self.dialect._load_enums( + self.connection, + schema="*", + info_cache=self.kw.get("info_cache"), + ) + ) + return enums + + @util.memoized_property + def domains(self) -> Dict[Tuple[str] | Tuple[str, str], ReflectedDomain]: + # dictionary with (name, ) if default search path or (schema, name) + # as keys + domains = { + ((d["schema"], d["name"]) if not d["visible"] else (d["name"],)): d + for d in self.dialect._load_domains( + self.connection, + schema="*", + info_cache=self.kw.get("info_cache"), + ) + } + return domains diff --git a/lib/sqlalchemy/dialects/postgresql/bitstring.py b/lib/sqlalchemy/dialects/postgresql/bitstring.py new file mode 100644 index 00000000000..71e0dd9949a --- /dev/null +++ b/lib/sqlalchemy/dialects/postgresql/bitstring.py @@ -0,0 +1,327 @@ +# dialects/postgresql/bitstring.py +# Copyright (C) 2013-2026 the SQLAlchemy authors and contributors +# +# +# This module is part of SQLAlchemy and is released under +# the MIT License: https://www.opensource.org/licenses/mit-license.php +from __future__ import annotations + +import math +from typing import Any +from typing import cast +from typing import Literal +from typing import SupportsIndex + + +class BitString(str): + """Represent a PostgreSQL bit string in python. + + This object is used by the :class:`_postgresql.BIT` type when returning + values. :class:`_postgresql.BitString` values may also be constructed + directly and used with :class:`_postgresql.BIT` columns:: + + from sqlalchemy.dialects.postgresql import BitString + + with engine.connect() as conn: + conn.execute(table.insert(), {"data": BitString("011001101")}) + + .. versionadded:: 2.1 + + """ + + _DIGITS = frozenset("01") + + def __new__(cls, _value: str, _check: bool = True) -> BitString: + if isinstance(_value, BitString): + return _value + elif _check and cls._DIGITS.union(_value) > cls._DIGITS: + raise ValueError("BitString must only contain '0' and '1' chars") + else: + return super().__new__(cls, _value) + + @classmethod + def from_int(cls, value: int, length: int) -> BitString: + """Returns a BitString consisting of the bits in the integer ``value``. + A ``ValueError`` is raised if ``value`` is not a non-negative integer. + + If the provided ``value`` can not be represented in a bit string + of at most ``length``, a ``ValueError`` will be raised. The bitstring + will be padded on the left by ``'0'`` to bits to produce a + bitstring of the desired length. + """ + if value < 0: + raise ValueError("value must be non-negative") + if length < 0: + raise ValueError("length must be non-negative") + + template_str = f"{{0:0{length}b}}" if length > 0 else "" + r = template_str.format(value) + + if (length == 0 and value > 0) or len(r) > length: + raise ValueError( + f"Cannot encode {value} as a BitString of length {length}" + ) + + return cls(r) + + @classmethod + def from_bytes(cls, value: bytes, length: int = -1) -> BitString: + """Returns a ``BitString`` consisting of the bits in the given + ``value`` bytes. + + If ``length`` is provided, then the length of the provided string + will be exactly ``length``, with ``'0'`` bits inserted at the left of + the string in order to produce a value of the required length. + If the bits obtained by omitting the leading ``'0'`` bits of ``value`` + cannot be represented in a string of this length a ``ValueError`` + will be raised. + """ + str_v: str = "".join(f"{int(c):08b}" for c in value) + if length >= 0: + str_v = str_v.lstrip("0") + + if len(str_v) > length: + raise ValueError( + f"Cannot encode {value!r} as a BitString of " + f"length {length}" + ) + str_v = str_v.zfill(length) + + return cls(str_v) + + def get_bit(self, index: int) -> Literal["0", "1"]: + """Returns the value of the flag at the given + index:: + + BitString("0101").get_flag(4) == "1" + """ + return cast(Literal["0", "1"], super().__getitem__(index)) + + @property + def bit_length(self) -> int: + return len(self) + + @property + def octet_length(self) -> int: + return math.ceil(len(self) / 8) + + def has_bit(self, index: int) -> bool: + return self.get_bit(index) == "1" + + def set_bit( + self, index: int, value: bool | int | Literal["0", "1"] + ) -> BitString: + """Set the bit at index to the given value. + + If value is an int, then it is considered to be '1' iff nonzero. + """ + if index < 0 or index >= len(self): + raise IndexError("BitString index out of range") + + if isinstance(value, (bool, int)): + value = "1" if value else "0" + + if self.get_bit(index) == value: + return self + + return BitString( + "".join([self[:index], value, self[index + 1 :]]), False + ) + + def lstrip(self, char: str | None = None) -> BitString: + """Returns a copy of the BitString with leading characters removed. + + If omitted or None, 'chars' defaults '0':: + + BitString("00010101000").lstrip() == BitString("00010101") + BitString("11110101111").lstrip("1") == BitString("1111010") + """ + if char is None: + char = "0" + return BitString(super().lstrip(char), False) + + def rstrip(self, char: str | None = "0") -> BitString: + """Returns a copy of the BitString with trailing characters removed. + + If omitted or None, ``'char'`` defaults to "0":: + + BitString("00010101000").rstrip() == BitString("10101000") + BitString("11110101111").rstrip("1") == BitString("10101111") + """ + if char is None: + char = "0" + return BitString(super().rstrip(char), False) + + def strip(self, char: str | None = "0") -> BitString: + """Returns a copy of the BitString with both leading and trailing + characters removed. + If omitted or None, ``'char'`` defaults to ``"0"``:: + + BitString("00010101000").rstrip() == BitString("10101") + BitString("11110101111").rstrip("1") == BitString("1010") + """ + if char is None: + char = "0" + return BitString(super().strip(char)) + + def removeprefix(self, prefix: str, /) -> BitString: + return BitString(super().removeprefix(prefix), False) + + def removesuffix(self, suffix: str, /) -> BitString: + return BitString(super().removesuffix(suffix), False) + + def replace( + self, + old: str, + new: str, + count: SupportsIndex = -1, + ) -> BitString: + new = BitString(new) + return BitString(super().replace(old, new, count), False) + + def split( + self, + sep: str | None = None, + maxsplit: SupportsIndex = -1, + ) -> list[str]: + return [BitString(word) for word in super().split(sep, maxsplit)] + + def zfill(self, width: SupportsIndex) -> BitString: + return BitString(super().zfill(width), False) + + def __repr__(self) -> str: + return f'BitString("{self.__str__()}")' + + def __int__(self) -> int: + return int(self, 2) if self else 0 + + def to_bytes(self, length: int = -1) -> bytes: + return int(self).to_bytes( + length if length >= 0 else self.octet_length, byteorder="big" + ) + + def __bytes__(self) -> bytes: + return self.to_bytes() + + def __getitem__( + self, key: SupportsIndex | slice[Any, Any, Any] + ) -> BitString: + return BitString(super().__getitem__(key), False) + + def __add__(self, o: str) -> BitString: + """Return self + o""" + if not isinstance(o, str): + raise TypeError( + f"Can only concatenate str (not '{type(self)}') to BitString" + ) + return BitString("".join([self, o])) + + def __radd__(self, o: str) -> BitString: + if not isinstance(o, str): + raise TypeError( + f"Can only concatenate str (not '{type(self)}') to BitString" + ) + return BitString("".join([o, self])) + + def __lshift__(self, amount: int) -> BitString: + """Shifts each the bitstring to the left by the given amount. + String length is preserved:: + + BitString("000101") << 1 == BitString("001010") + """ + return BitString( + "".join([self, *("0" for _ in range(amount))])[-len(self) :], False + ) + + def __rshift__(self, amount: int) -> BitString: + """Shifts each bit in the bitstring to the right by the given amount. + String length is preserved:: + + BitString("101") >> 1 == BitString("010") + """ + return BitString(self[:-amount], False).zfill(width=len(self)) + + def __invert__(self) -> BitString: + """Inverts (~) each bit in the + bitstring:: + + ~BitString("01010") == BitString("10101") + """ + return BitString("".join("1" if x == "0" else "0" for x in self)) + + def __and__(self, o: str) -> BitString: + """Performs a bitwise and (``&``) with the given operand. + A ``ValueError`` is raised if the operand is not the same length. + + e.g.:: + + BitString("011") & BitString("011") == BitString("010") + """ + + if not isinstance(o, str): + return NotImplemented + o = BitString(o) + if len(self) != len(o): + raise ValueError("Operands must be the same length") + + return BitString( + "".join( + "1" if (x == "1" and y == "1") else "0" + for x, y in zip(self, o) + ), + False, + ) + + def __or__(self, o: str) -> BitString: + """Performs a bitwise or (``|``) with the given operand. + A ``ValueError`` is raised if the operand is not the same length. + + e.g.:: + + BitString("011") | BitString("010") == BitString("011") + """ + if not isinstance(o, str): + return NotImplemented + + if len(self) != len(o): + raise ValueError("Operands must be the same length") + + o = BitString(o) + return BitString( + "".join( + "1" if (x == "1" or y == "1") else "0" + for (x, y) in zip(self, o) + ), + False, + ) + + def __xor__(self, o: str) -> BitString: + """Performs a bitwise xor (``^``) with the given operand. + A ``ValueError`` is raised if the operand is not the same length. + + e.g.:: + + BitString("011") ^ BitString("010") == BitString("001") + """ + + if not isinstance(o, BitString): + return NotImplemented + + if len(self) != len(o): + raise ValueError("Operands must be the same length") + + return BitString( + "".join( + ( + "1" + if ((x == "1" and y == "0") or (x == "0" and y == "1")) + else "0" + ) + for (x, y) in zip(self, o) + ), + False, + ) + + __rand__ = __and__ + __ror__ = __or__ + __rxor__ = __xor__ diff --git a/lib/sqlalchemy/dialects/postgresql/dml.py b/lib/sqlalchemy/dialects/postgresql/dml.py index 69647546610..ee4037dc90e 100644 --- a/lib/sqlalchemy/dialects/postgresql/dml.py +++ b/lib/sqlalchemy/dialects/postgresql/dml.py @@ -1,5 +1,5 @@ # dialects/postgresql/dml.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/dialects/postgresql/ext.py b/lib/sqlalchemy/dialects/postgresql/ext.py index 94466ae0a13..ffe46c9dc7d 100644 --- a/lib/sqlalchemy/dialects/postgresql/ext.py +++ b/lib/sqlalchemy/dialects/postgresql/ext.py @@ -1,5 +1,5 @@ # dialects/postgresql/ext.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -8,28 +8,43 @@ from __future__ import annotations from typing import Any +from typing import Iterable +from typing import List +from typing import Optional +from typing import overload +from typing import Sequence +from typing import Tuple from typing import TYPE_CHECKING from typing import TypeVar from . import types from .array import ARRAY +from ... import exc from ...sql import coercions from ...sql import elements from ...sql import expression from ...sql import functions from ...sql import roles from ...sql import schema +from ...sql.base import SyntaxExtension from ...sql.schema import ColumnCollectionConstraint from ...sql.sqltypes import TEXT from ...sql.visitors import InternalTraversal -_T = TypeVar("_T", bound=Any) - if TYPE_CHECKING: + from ...sql._typing import _ColumnExpressionArgument + from ...sql._typing import _DDLColumnArgument + from ...sql.elements import ClauseElement + from ...sql.elements import ColumnElement + from ...sql.operators import OperatorType + from ...sql.selectable import FromClause + from ...sql.visitors import _CloneCallableType from ...sql.visitors import _TraverseInternalsType +_T = TypeVar("_T", bound=Any) -class aggregate_order_by(expression.ColumnElement): + +class aggregate_order_by(expression.ColumnElement[_T]): """Represent a PostgreSQL aggregate order by expression. E.g.:: @@ -45,23 +60,15 @@ class aggregate_order_by(expression.ColumnElement): SELECT array_agg(a ORDER BY b DESC) FROM table; - Similarly:: - - expr = func.string_agg( - table.c.a, aggregate_order_by(literal_column("','"), table.c.a) - ) - stmt = select(expr) - - Would represent: - - .. sourcecode:: sql - - SELECT string_agg(a, ',' ORDER BY a) FROM table; - - .. versionchanged:: 1.2.13 - the ORDER BY argument may be multiple terms + .. legacy:: An improved dialect-agnostic form of this function is now + available in Core by calling the + :meth:`_functions.Function.aggregate_order_by` method on any function + defined by the backend as an aggregate function. .. seealso:: + :func:`_sql.aggregate_order_by` - Core level function + :class:`_functions.array_agg` """ @@ -75,11 +82,32 @@ class aggregate_order_by(expression.ColumnElement): ("order_by", InternalTraversal.dp_clauseelement), ] - def __init__(self, target, *order_by): - self.target = coercions.expect(roles.ExpressionElementRole, target) + @overload + def __init__( + self, + target: ColumnElement[_T], + *order_by: _ColumnExpressionArgument[Any], + ): ... + + @overload + def __init__( + self, + target: _ColumnExpressionArgument[_T], + *order_by: _ColumnExpressionArgument[Any], + ): ... + + def __init__( + self, + target: _ColumnExpressionArgument[_T], + *order_by: _ColumnExpressionArgument[Any], + ): + self.target: ClauseElement = coercions.expect( + roles.ExpressionElementRole, target + ) self.type = self.target.type _lob = len(order_by) + self.order_by: ClauseElement if _lob == 0: raise TypeError("at least one ORDER BY element is required") elif _lob == 1: @@ -91,18 +119,22 @@ def __init__(self, target, *order_by): *order_by, _literal_as_text_role=roles.ExpressionElementRole ) - def self_group(self, against=None): + def self_group( + self, against: Optional[OperatorType] = None + ) -> ClauseElement: return self - def get_children(self, **kwargs): + def get_children(self, **kwargs: Any) -> Iterable[ClauseElement]: return self.target, self.order_by - def _copy_internals(self, clone=elements._clone, **kw): + def _copy_internals( + self, clone: _CloneCallableType = elements._clone, **kw: Any + ) -> None: self.target = clone(self.target, **kw) self.order_by = clone(self.order_by, **kw) @property - def _from_objects(self): + def _from_objects(self) -> List[FromClause]: return self.target._from_objects + self.order_by._from_objects @@ -128,7 +160,9 @@ class ExcludeConstraint(ColumnCollectionConstraint): ":class:`.ExcludeConstraint`", ":paramref:`.ExcludeConstraint.where`", ) - def __init__(self, *elements, **kw): + def __init__( + self, *elements: Tuple[_DDLColumnArgument, str], **kw: Any + ) -> None: r""" Create an :class:`.ExcludeConstraint` object. @@ -210,8 +244,6 @@ def __init__(self, *elements, **kw): :ref:`postgresql_ops ` parameter specified to the :class:`_schema.Index` construct. - .. versionadded:: 1.3.21 - .. seealso:: :ref:`postgresql_operator_classes` - general description of how @@ -499,3 +531,63 @@ def __init__(self, *args, **kwargs): for c in args ] super().__init__(*(initial_arg + addtl_args), **kwargs) + + +def distinct_on(*expr: _ColumnExpressionArgument[Any]) -> DistinctOnClause: + """apply a DISTINCT_ON to a SELECT statement + + e.g.:: + + stmt = select(tbl).ext(distinct_on(t.c.some_col)) + + this supersedes the previous approach of using + ``select(tbl).distinct(t.c.some_col))`` to apply a similar construct. + + .. versionadded:: 2.1 + + """ + return DistinctOnClause(expr) + + +class DistinctOnClause(SyntaxExtension, expression.ClauseElement): + stringify_dialect = "postgresql" + __visit_name__ = "postgresql_distinct_on" + + _traverse_internals: _TraverseInternalsType = [ + ("_distinct_on", InternalTraversal.dp_clauseelement_tuple), + ] + + def __init__(self, distinct_on: Sequence[_ColumnExpressionArgument[Any]]): + self._distinct_on = tuple( + coercions.expect(roles.ByOfRole, e, apply_propagate_attrs=self) + for e in distinct_on + ) + + def apply_to_select(self, select_stmt: expression.Select[Any]) -> None: + if select_stmt._distinct_on: + raise exc.InvalidRequestError( + "Cannot mix ``select.ext(distinct_on(...))`` and " + "``select.distinct(...)``" + ) + # mark this select as a distinct + select_stmt.distinct.non_generative(select_stmt) + + select_stmt.apply_syntax_extension_point( + self._merge_other_distinct, "pre_columns" + ) + + def _merge_other_distinct( + self, existing: Sequence[elements.ClauseElement] + ) -> Sequence[elements.ClauseElement]: + res = [] + to_merge = () + for e in existing: + if isinstance(e, DistinctOnClause): + to_merge += e._distinct_on + else: + res.append(e) + if to_merge: + res.append(DistinctOnClause(to_merge + self._distinct_on)) + else: + res.append(self) + return res diff --git a/lib/sqlalchemy/dialects/postgresql/hstore.py b/lib/sqlalchemy/dialects/postgresql/hstore.py index 0a915b17dff..91666e71cea 100644 --- a/lib/sqlalchemy/dialects/postgresql/hstore.py +++ b/lib/sqlalchemy/dialects/postgresql/hstore.py @@ -1,13 +1,15 @@ # dialects/postgresql/hstore.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors +from __future__ import annotations import re +from typing import Any +from typing import Optional from .array import ARRAY from .operators import CONTAINED_BY @@ -18,12 +20,19 @@ from .operators import HAS_KEY from ... import types as sqltypes from ...sql import functions as sqlfunc +from ...types import OperatorClass __all__ = ("HSTORE", "hstore") +_HSTORE_VAL = dict[str, str | None] -class HSTORE(sqltypes.Indexable, sqltypes.Concatenable, sqltypes.TypeEngine): + +class HSTORE( + sqltypes.Indexable, + sqltypes.Concatenable, + sqltypes.TypeEngine[_HSTORE_VAL], +): """Represent the PostgreSQL HSTORE type. The :class:`.HSTORE` type stores dictionaries containing strings, e.g.:: @@ -105,7 +114,14 @@ class MyClass(Base): hashable = False text_type = sqltypes.Text() - def __init__(self, text_type=None): + operator_classes = ( + OperatorClass.BASE + | OperatorClass.CONTAINS + | OperatorClass.INDEXABLE + | OperatorClass.CONCATENABLE + ) + + def __init__(self, text_type: Optional[Any] = None) -> None: """Construct a new :class:`.HSTORE`. :param text_type: the type that should be used for indexed values. @@ -116,25 +132,26 @@ def __init__(self, text_type=None): self.text_type = text_type class Comparator( - sqltypes.Indexable.Comparator, sqltypes.Concatenable.Comparator + sqltypes.Indexable.Comparator[_HSTORE_VAL], + sqltypes.Concatenable.Comparator[_HSTORE_VAL], ): """Define comparison operations for :class:`.HSTORE`.""" - def has_key(self, other): + def has_key(self, other: Any) -> Any: """Boolean expression. Test for presence of a key. Note that the key may be a SQLA expression. """ return self.operate(HAS_KEY, other, result_type=sqltypes.Boolean) - def has_all(self, other): + def has_all(self, other: Any) -> Any: """Boolean expression. Test for presence of all keys in jsonb""" return self.operate(HAS_ALL, other, result_type=sqltypes.Boolean) - def has_any(self, other): + def has_any(self, other: Any) -> Any: """Boolean expression. Test for presence of any key in jsonb""" return self.operate(HAS_ANY, other, result_type=sqltypes.Boolean) - def contains(self, other, **kwargs): + def contains(self, other: Any, **kwargs: Any) -> Any: """Boolean expression. Test if keys (or array) are a superset of/contained the keys of the argument jsonb expression. @@ -143,7 +160,7 @@ def contains(self, other, **kwargs): """ return self.operate(CONTAINS, other, result_type=sqltypes.Boolean) - def contained_by(self, other): + def contained_by(self, other: Any) -> Any: """Boolean expression. Test if keys are a proper subset of the keys of the argument jsonb expression. """ @@ -151,16 +168,16 @@ def contained_by(self, other): CONTAINED_BY, other, result_type=sqltypes.Boolean ) - def _setup_getitem(self, index): - return GETITEM, index, self.type.text_type + def _setup_getitem(self, index: Any) -> Any: + return GETITEM, index, self.type.text_type # type: ignore - def defined(self, key): + def defined(self, key: Any) -> Any: """Boolean expression. Test for presence of a non-NULL value for the key. Note that the key may be a SQLA expression. """ return _HStoreDefinedFunction(self.expr, key) - def delete(self, key): + def delete(self, key: Any) -> Any: """HStore expression. Returns the contents of this hstore with the given key deleted. Note that the key may be a SQLA expression. """ @@ -168,37 +185,37 @@ def delete(self, key): key = _serialize_hstore(key) return _HStoreDeleteFunction(self.expr, key) - def slice(self, array): + def slice(self, array: Any) -> Any: """HStore expression. Returns a subset of an hstore defined by array of keys. """ return _HStoreSliceFunction(self.expr, array) - def keys(self): + def keys(self) -> Any: """Text array expression. Returns array of keys.""" return _HStoreKeysFunction(self.expr) - def vals(self): + def vals(self) -> Any: """Text array expression. Returns array of values.""" return _HStoreValsFunction(self.expr) - def array(self): + def array(self) -> Any: """Text array expression. Returns array of alternating keys and values. """ return _HStoreArrayFunction(self.expr) - def matrix(self): + def matrix(self) -> Any: """Text array expression. Returns array of [key, value] pairs.""" return _HStoreMatrixFunction(self.expr) comparator_factory = Comparator - def bind_processor(self, dialect): + def bind_processor(self, dialect: Any) -> Any: # note that dialect-specific types like that of psycopg and # psycopg2 will override this method to allow driver-level conversion # instead, see _PsycopgHStore - def process(value): + def process(value: Any) -> Any: if isinstance(value, dict): return _serialize_hstore(value) else: @@ -206,11 +223,11 @@ def process(value): return process - def result_processor(self, dialect, coltype): + def result_processor(self, dialect: Any, coltype: Any) -> Any: # note that dialect-specific types like that of psycopg and # psycopg2 will override this method to allow driver-level conversion # instead, see _PsycopgHStore - def process(value): + def process(value: Any) -> Any: if value is not None: return _parse_hstore(value) else: @@ -219,7 +236,7 @@ def process(value): return process -class hstore(sqlfunc.GenericFunction): +class hstore(sqlfunc.GenericFunction[_HSTORE_VAL]): """Construct an hstore value within a SQL expression using the PostgreSQL ``hstore()`` function. @@ -245,48 +262,48 @@ class hstore(sqlfunc.GenericFunction): """ - type = HSTORE + type = HSTORE() name = "hstore" inherit_cache = True -class _HStoreDefinedFunction(sqlfunc.GenericFunction): - type = sqltypes.Boolean +class _HStoreDefinedFunction(sqlfunc.GenericFunction[bool]): + type = sqltypes.Boolean() name = "defined" inherit_cache = True -class _HStoreDeleteFunction(sqlfunc.GenericFunction): - type = HSTORE +class _HStoreDeleteFunction(sqlfunc.GenericFunction[_HSTORE_VAL]): + type = HSTORE() name = "delete" inherit_cache = True -class _HStoreSliceFunction(sqlfunc.GenericFunction): - type = HSTORE +class _HStoreSliceFunction(sqlfunc.GenericFunction[_HSTORE_VAL]): + type = HSTORE() name = "slice" inherit_cache = True -class _HStoreKeysFunction(sqlfunc.GenericFunction): +class _HStoreKeysFunction(sqlfunc.GenericFunction[Any]): type = ARRAY(sqltypes.Text) name = "akeys" inherit_cache = True -class _HStoreValsFunction(sqlfunc.GenericFunction): +class _HStoreValsFunction(sqlfunc.GenericFunction[Any]): type = ARRAY(sqltypes.Text) name = "avals" inherit_cache = True -class _HStoreArrayFunction(sqlfunc.GenericFunction): +class _HStoreArrayFunction(sqlfunc.GenericFunction[Any]): type = ARRAY(sqltypes.Text) name = "hstore_to_array" inherit_cache = True -class _HStoreMatrixFunction(sqlfunc.GenericFunction): +class _HStoreMatrixFunction(sqlfunc.GenericFunction[Any]): type = ARRAY(sqltypes.Text) name = "hstore_to_matrix" inherit_cache = True @@ -322,7 +339,7 @@ class _HStoreMatrixFunction(sqlfunc.GenericFunction): ) -def _parse_error(hstore_str, pos): +def _parse_error(hstore_str: str, pos: int) -> str: """format an unmarshalling error.""" ctx = 20 @@ -343,7 +360,7 @@ def _parse_error(hstore_str, pos): ) -def _parse_hstore(hstore_str): +def _parse_hstore(hstore_str: str) -> _HSTORE_VAL: """Parse an hstore from its literal string representation. Attempts to approximate PG's hstore input parsing rules as closely as @@ -355,7 +372,7 @@ def _parse_hstore(hstore_str): """ - result = {} + result: _HSTORE_VAL = {} pos = 0 pair_match = HSTORE_PAIR_RE.match(hstore_str) @@ -385,13 +402,13 @@ def _parse_hstore(hstore_str): return result -def _serialize_hstore(val): +def _serialize_hstore(val: _HSTORE_VAL) -> str: """Serialize a dictionary into an hstore literal. Keys and values must both be strings (except None for values). """ - def esc(s, position): + def esc(s: Optional[str], position: str) -> str: if position == "value" and s is None: return "NULL" elif isinstance(s, str): diff --git a/lib/sqlalchemy/dialects/postgresql/json.py b/lib/sqlalchemy/dialects/postgresql/json.py index 663be8b7a2b..54b33fc65a5 100644 --- a/lib/sqlalchemy/dialects/postgresql/json.py +++ b/lib/sqlalchemy/dialects/postgresql/json.py @@ -1,5 +1,5 @@ # dialects/postgresql/json.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -28,11 +28,14 @@ from .operators import PATH_MATCH from ... import types as sqltypes from ...sql import cast -from ...sql._typing import _T +from ...sql.operators import OperatorClass +from ...sql.sqltypes import _CT_JSON +from ...sql.sqltypes import _T_JSON if TYPE_CHECKING: from ...engine.interfaces import Dialect from ...sql.elements import ColumnElement + from ...sql.operators import OperatorType from ...sql.type_api import _BindProcessorType from ...sql.type_api import _LiteralProcessorType from ...sql.type_api import TypeEngine @@ -88,7 +91,7 @@ class JSONPATH(JSONPathType): __visit_name__ = "JSONPATH" -class JSON(sqltypes.JSON): +class JSON(sqltypes.JSON[_T_JSON]): """Represent the PostgreSQL JSON type. :class:`_postgresql.JSON` is used automatically whenever the base @@ -198,10 +201,10 @@ def __init__( if astext_type is not None: self.astext_type = astext_type - class Comparator(sqltypes.JSON.Comparator[_T]): + class Comparator(sqltypes.JSON.Comparator[_CT_JSON]): """Define comparison operations for :class:`_types.JSON`.""" - type: JSON + type: JSON[_CT_JSON] @property def astext(self) -> ColumnElement[str]: @@ -231,7 +234,7 @@ def astext(self) -> ColumnElement[str]: comparator_factory = Comparator -class JSONB(JSON): +class JSONB(JSON[_T_JSON]): """Represent the PostgreSQL JSONB type. The :class:`_postgresql.JSONB` type stores arbitrary JSONB format data, @@ -279,14 +282,52 @@ class JSONB(JSON): :class:`_types.JSON` + .. warning:: + + **For applications that have indexes against JSONB subscript + expressions** + + SQLAlchemy 2.0.42 made a change in how the subscript operation for + :class:`.JSONB` is rendered, from ``-> 'element'`` to ``['element']``, + for PostgreSQL versions greater than 14. This change caused an + unintended side effect for indexes that were created against + expressions that use subscript notation, e.g. + ``Index("ix_entity_json_ab_text", data["a"]["b"].astext)``. If these + indexes were generated with the older syntax e.g. ``((entity.data -> + 'a') ->> 'b')``, they will not be used by the PostgreSQL query planner + when a query is made using SQLAlchemy 2.0.42 or higher on PostgreSQL + versions 14 or higher. This occurs because the new text will resemble + ``(entity.data['a'] ->> 'b')`` which will fail to produce the exact + textual syntax match required by the PostgreSQL query planner. + Therefore, for users upgrading to SQLAlchemy 2.0.42 or higher, existing + indexes that were created against :class:`.JSONB` expressions that use + subscripting would need to be dropped and re-created in order for them + to work with the new query syntax, e.g. an expression like + ``((entity.data -> 'a') ->> 'b')`` would become ``(entity.data['a'] ->> + 'b')``. + + .. seealso:: + + :ticket:`12868` - discussion of this issue + """ __visit_name__ = "JSONB" - class Comparator(JSON.Comparator[_T]): + operator_classes = OperatorClass.JSON | OperatorClass.CONCATENABLE + + def coerce_compared_value( + self, op: Optional[OperatorType], value: Any + ) -> TypeEngine[Any]: + if op in (PATH_MATCH, PATH_EXISTS): + return JSON.JSONPathType() + else: + return super().coerce_compared_value(op, value) + + class Comparator(JSON.Comparator[_CT_JSON]): """Define comparison operations for :class:`_types.JSON`.""" - type: JSONB + type: JSONB[_CT_JSON] def has_key(self, other: Any) -> ColumnElement[bool]: """Boolean expression. Test for presence of a key (equivalent of @@ -327,7 +368,7 @@ def contained_by(self, other: Any) -> ColumnElement[bool]: def delete_path( self, array: Union[List[str], _pg_array[str]] - ) -> ColumnElement[JSONB]: + ) -> ColumnElement[_CT_JSON]: """JSONB expression. Deletes field or array element specified in the argument array (equivalent of the ``#-`` operator). @@ -337,7 +378,7 @@ def delete_path( .. versionadded:: 2.0 """ if not isinstance(array, _pg_array): - array = _pg_array(array) # type: ignore[no-untyped-call] + array = _pg_array(array) right_side = cast(array, ARRAY(sqltypes.TEXT)) return self.operate(DELETE_PATH, right_side, result_type=JSONB) diff --git a/lib/sqlalchemy/dialects/postgresql/named_types.py b/lib/sqlalchemy/dialects/postgresql/named_types.py index e1b8e84ce85..4b9c4b3d0ab 100644 --- a/lib/sqlalchemy/dialects/postgresql/named_types.py +++ b/lib/sqlalchemy/dialects/postgresql/named_types.py @@ -1,5 +1,5 @@ # dialects/postgresql/named_types.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -7,7 +7,9 @@ # mypy: ignore-errors from __future__ import annotations +from types import ModuleType from typing import Any +from typing import Dict from typing import Optional from typing import Type from typing import TYPE_CHECKING @@ -21,14 +23,16 @@ from ...sql import sqltypes from ...sql import type_api from ...sql.base import _NoArg +from ...sql.ddl import CheckFirst from ...sql.ddl import InvokeCreateDDLBase from ...sql.ddl import InvokeDropDDLBase if TYPE_CHECKING: + from ...sql._typing import _CreateDropBind from ...sql._typing import _TypeEngineArgument -class NamedType(sqltypes.TypeEngine): +class NamedType(schema.SchemaVisitable, sqltypes.TypeEngine): """Base for named types.""" __abstract__ = True @@ -36,7 +40,9 @@ class NamedType(sqltypes.TypeEngine): DDLDropper: Type[NamedTypeDropper] create_type: bool - def create(self, bind, checkfirst=True, **kw): + def create( + self, bind: _CreateDropBind, checkfirst: bool = True, **kw: Any + ) -> None: """Emit ``CREATE`` DDL for this type. :param bind: a connectable :class:`_engine.Engine`, @@ -50,7 +56,9 @@ def create(self, bind, checkfirst=True, **kw): """ bind._run_ddl_visitor(self.DDLGenerator, self, checkfirst=checkfirst) - def drop(self, bind, checkfirst=True, **kw): + def drop( + self, bind: _CreateDropBind, checkfirst: bool = True, **kw: Any + ) -> None: """Emit ``DROP`` DDL for this type. :param bind: a connectable :class:`_engine.Engine`, @@ -63,7 +71,9 @@ def drop(self, bind, checkfirst=True, **kw): """ bind._run_ddl_visitor(self.DDLDropper, self, checkfirst=checkfirst) - def _check_for_name_in_memos(self, checkfirst, kw): + def _check_for_name_in_memos( + self, checkfirst: CheckFirst, kw: Dict[str, Any] + ) -> bool: """Look in the 'ddl runner' for 'memos', then note our name in that collection. @@ -87,31 +97,48 @@ def _check_for_name_in_memos(self, checkfirst, kw): else: return False - def _on_table_create(self, target, bind, checkfirst=False, **kw): - if ( - checkfirst - or ( - not self.metadata - and not kw.get("_is_metadata_operation", False) - ) - ) and not self._check_for_name_in_memos(checkfirst, kw): - self.create(bind=bind, checkfirst=checkfirst) + def _on_table_create( + self, + target: schema.Table, + bind: _CreateDropBind, + checkfirst: Union[bool, CheckFirst] = CheckFirst.NONE, + **kw: Any, + ) -> None: + checkfirst = CheckFirst(checkfirst) & CheckFirst.TYPES + if not self._check_for_name_in_memos(checkfirst, kw): + self.create(bind=bind, checkfirst=bool(checkfirst)) - def _on_table_drop(self, target, bind, checkfirst=False, **kw): - if ( - not self.metadata - and not kw.get("_is_metadata_operation", False) - and not self._check_for_name_in_memos(checkfirst, kw) - ): - self.drop(bind=bind, checkfirst=checkfirst) + def _on_table_drop( + self, + target: Any, + bind: _CreateDropBind, + checkfirst: CheckFirst = CheckFirst.NONE, + **kw: Any, + ) -> None: + # do nothing since the enum is attached to a metadata + assert self.metadata is not None - def _on_metadata_create(self, target, bind, checkfirst=False, **kw): + def _on_metadata_create( + self, + target: schema.MetaData, + bind: _CreateDropBind, + checkfirst: Union[bool, CheckFirst] = CheckFirst.NONE, + **kw: Any, + ) -> None: + checkfirst = CheckFirst(checkfirst) & CheckFirst.TYPES if not self._check_for_name_in_memos(checkfirst, kw): - self.create(bind=bind, checkfirst=checkfirst) + self.create(bind=bind, checkfirst=bool(checkfirst)) - def _on_metadata_drop(self, target, bind, checkfirst=False, **kw): + def _on_metadata_drop( + self, + target: schema.MetaData, + bind: _CreateDropBind, + checkfirst: Union[bool, CheckFirst] = CheckFirst.NONE, + **kw: Any, + ) -> None: + checkfirst = CheckFirst(checkfirst) & CheckFirst.TYPES if not self._check_for_name_in_memos(checkfirst, kw): - self.drop(bind=bind, checkfirst=checkfirst) + self.drop(bind=bind, checkfirst=bool(checkfirst)) class NamedTypeGenerator(InvokeCreateDDLBase): @@ -174,16 +201,12 @@ class ENUM(NamedType, type_api.NativeForEmulated, sqltypes.Enum): type as the implementation, so the special create/drop rules will be used. - The create/drop behavior of ENUM is necessarily intricate, due to the - awkward relationship the ENUM type has in relationship to the - parent table, in that it may be "owned" by just a single table, or - may be shared among many tables. + The create/drop behavior of ENUM tries to follow the PostgreSQL behavior, + with an usability improvement indicated below. When using :class:`_types.Enum` or :class:`_postgresql.ENUM` - in an "inline" fashion, the ``CREATE TYPE`` and ``DROP TYPE`` is emitted - corresponding to when the :meth:`_schema.Table.create` and - :meth:`_schema.Table.drop` - methods are called:: + in an "inline" fashion, the ``CREATE TYPE`` is emitted + corresponding to when the :meth:`_schema.Table.create` method is called:: table = Table( "sometable", @@ -191,13 +214,19 @@ class ENUM(NamedType, type_api.NativeForEmulated, sqltypes.Enum): Column("some_enum", ENUM("a", "b", "c", name="myenum")), ) - table.create(engine) # will emit CREATE ENUM and CREATE TABLE - table.drop(engine) # will emit DROP TABLE and DROP ENUM + # will check if enum exists and emit CREATE ENUM then CREATE TABLE + table.create(engine) + table.drop(engine) # will *not* drop the enum. + + The enum will not be dropped when the table is dropped, since it's + associated with the metadata, not the table itself. Call drop on the + :class:`_postgresql.ENUM` directly to drop the type:: + + metadata.get_schema_object_by_name("enum", "myenum").drop(engine) To use a common enumerated type between multiple tables, the best practice is to declare the :class:`_types.Enum` or - :class:`_postgresql.ENUM` independently, and associate it with the - :class:`_schema.MetaData` object itself:: + :class:`_postgresql.ENUM` independently:: my_enum = ENUM("a", "b", "c", name="myenum", metadata=metadata) @@ -205,20 +234,13 @@ class ENUM(NamedType, type_api.NativeForEmulated, sqltypes.Enum): t2 = Table("sometable_two", metadata, Column("some_enum", myenum)) - When this pattern is used, care must still be taken at the level - of individual table creates. Emitting CREATE TABLE without also - specifying ``checkfirst=True`` will still cause issues:: + Like before, the type will be created if it does not exist:: - t1.create(engine) # will fail: no such type 'myenum' + # will check if enum exists and emit CREATE ENUM then CREATE TABLE + t1.create(engine) - If we specify ``checkfirst=True``, the individual table-level create - operation will check for the ``ENUM`` and create if not exists:: - - # will check if enum exists, and emit CREATE TYPE if not - t1.create(engine, checkfirst=True) - - When using a metadata-level ENUM type, the type will always be created - and dropped if either the metadata-wide create/drop is called:: + The type will always be created and dropped if either the metadata-wide + create/drop is called:: metadata.create_all(engine) # will emit CREATE TYPE metadata.drop_all(engine) # will emit DROP TYPE @@ -228,6 +250,16 @@ class ENUM(NamedType, type_api.NativeForEmulated, sqltypes.Enum): my_enum.create(engine) my_enum.drop(engine) + .. versionchanged:: 2.1 The behavior of :class:`_postgresql.ENUM` and + other named types has been changed to better reflect how PostgreSQL + handles CREATE TYPE and DROP TYPE operations. + Named types are still created when needed during table + creation if they do not already exist. However, they are no longer + dropped for an individual :meth:`.Table.drop` operation, since the type + may be referenced by other tables as well. Instead, + :meth:`.Enum.drop` may be used or :meth:`.MetaData.drop_all` will drop + all associated types. + """ native_enum = True @@ -251,9 +283,7 @@ def __init__( Indicates that ``CREATE TYPE`` should be emitted, after optionally checking for the presence of the type, when the parent - table is being created; and additionally - that ``DROP TYPE`` is called when the table - is dropped. When ``False``, no check + table is being created. When ``False``, no check will be performed and no ``CREATE TYPE`` or ``DROP TYPE`` is emitted, unless :meth:`~.postgresql.ENUM.create` @@ -275,9 +305,9 @@ def __init__( "always refers to ENUM. Use sqlalchemy.types.Enum for " "non-native enum." ) - self.create_type = create_type if name is not _NoArg.NO_ARG: kw["name"] = name + kw["create_type"] = create_type super().__init__(*enums, **kw) def coerce_compared_value(self, op, value): @@ -302,19 +332,17 @@ def adapt_emulated_to_native(cls, impl, **kw): """ kw.setdefault("validate_strings", impl.validate_strings) kw.setdefault("name", impl.name) + kw.setdefault("create_type", impl.create_type) kw.setdefault("schema", impl.schema) - kw.setdefault("inherit_schema", impl.inherit_schema) kw.setdefault("metadata", impl.metadata) kw.setdefault("_create_events", False) kw.setdefault("values_callable", impl.values_callable) kw.setdefault("omit_aliases", impl._omit_aliases) kw.setdefault("_adapted_from", impl) - if type_api._is_native_for_emulated(impl.__class__): - kw.setdefault("create_type", impl.create_type) return cls(**kw) - def create(self, bind=None, checkfirst=True): + def create(self, bind: _CreateDropBind, checkfirst: bool = True) -> None: """Emit ``CREATE TYPE`` for this :class:`_postgresql.ENUM`. @@ -335,7 +363,7 @@ def create(self, bind=None, checkfirst=True): super().create(bind, checkfirst=checkfirst) - def drop(self, bind=None, checkfirst=True): + def drop(self, bind: _CreateDropBind, checkfirst: bool = True) -> None: """Emit ``DROP TYPE`` for this :class:`_postgresql.ENUM`. @@ -355,7 +383,7 @@ def drop(self, bind=None, checkfirst=True): super().drop(bind, checkfirst=checkfirst) - def get_dbapi_type(self, dbapi): + def get_dbapi_type(self, dbapi: ModuleType) -> None: """dont return dbapi.STRING for ENUM in PostgreSQL, since that's a different type""" @@ -393,6 +421,9 @@ class DOMAIN(NamedType, sqltypes.SchemaType): check="VALUE ~ '^\d{5}$' OR VALUE ~ '^\d{5}-\d{4}$'", ) + :class:`_postgresql.DOMAIN` has the same create/drop behavior specified + in :class:`_postgresql.ENUM`. + See the `PostgreSQL documentation`__ for additional details __ https://www.postgresql.org/docs/current/sql-createdomain.html @@ -463,27 +494,12 @@ def __init__( if check is not None: check = coercions.expect(roles.DDLExpressionRole, check) self.check = check - self.create_type = create_type - super().__init__(name=name, **kw) + super().__init__(name=name, create_type=create_type, **kw) @classmethod def __test_init__(cls): return cls("name", sqltypes.Integer) - def adapt(self, impl, **kw): - if self.default: - kw["default"] = self.default - if self.constraint_name is not None: - kw["constraint_name"] = self.constraint_name - if self.not_null: - kw["not_null"] = self.not_null - if self.check is not None: - kw["check"] = str(self.check) - if self.create_type: - kw["create_type"] = self.create_type - - return super().adapt(impl, **kw) - class CreateEnumType(schema._CreateDropBase): __visit_name__ = "create_enum_type" diff --git a/lib/sqlalchemy/dialects/postgresql/operators.py b/lib/sqlalchemy/dialects/postgresql/operators.py index ebcafcba991..2eb4f43138e 100644 --- a/lib/sqlalchemy/dialects/postgresql/operators.py +++ b/lib/sqlalchemy/dialects/postgresql/operators.py @@ -1,5 +1,5 @@ # dialects/postgresql/operators.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -126,4 +126,5 @@ precedence=_getitem_precedence, natural_self_precedent=True, eager_grouping=True, + visit_name="hstore_getitem", ) diff --git a/lib/sqlalchemy/dialects/postgresql/pg8000.py b/lib/sqlalchemy/dialects/postgresql/pg8000.py index e36709433c7..93cd7e156c2 100644 --- a/lib/sqlalchemy/dialects/postgresql/pg8000.py +++ b/lib/sqlalchemy/dialects/postgresql/pg8000.py @@ -1,5 +1,5 @@ # dialects/postgresql/pg8000.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under @@ -170,16 +170,10 @@ def bind_processor(self, dialect): class _PGJSON(JSON): render_bind_cast = True - def result_processor(self, dialect, coltype): - return None - class _PGJSONB(JSONB): render_bind_cast = True - def result_processor(self, dialect, coltype): - return None - class _PGJSONIndexType(sqltypes.JSON.JSONIndexType): def get_dbapi_type(self, dbapi): @@ -421,6 +415,10 @@ class PGDialect_pg8000(PGDialect): preparer = PGIdentifierPreparer_pg8000 supports_server_side_cursors = True + supports_native_json_serialization = False + supports_native_json_deserialization = True + dialect_injects_custom_json_deserializer = True + render_bind_cast = True # reversed as of pg8000 1.16.6. 1.16.5 and lower @@ -543,6 +541,9 @@ def set_isolation_level(self, dbapi_connection, level): cursor.execute("COMMIT") cursor.close() + def detect_autocommit_setting(self, dbapi_conn) -> bool: + return bool(dbapi_conn.autocommit) + def set_readonly(self, connection, value): cursor = connection.cursor() try: diff --git a/lib/sqlalchemy/dialects/postgresql/pg_catalog.py b/lib/sqlalchemy/dialects/postgresql/pg_catalog.py index 78f390a2118..d8f9987ec26 100644 --- a/lib/sqlalchemy/dialects/postgresql/pg_catalog.py +++ b/lib/sqlalchemy/dialects/postgresql/pg_catalog.py @@ -1,10 +1,16 @@ # dialects/postgresql/pg_catalog.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors + +from __future__ import annotations + +from typing import Any +from typing import Optional +from typing import Sequence +from typing import TYPE_CHECKING from .array import ARRAY from .types import OID @@ -23,31 +29,37 @@ from ...types import Text from ...types import TypeDecorator +if TYPE_CHECKING: + from ...engine.interfaces import Dialect + from ...sql.type_api import _ResultProcessorType + # types -class NAME(TypeDecorator): +class NAME(TypeDecorator[str]): impl = String(64, collation="C") cache_ok = True -class PG_NODE_TREE(TypeDecorator): +class PG_NODE_TREE(TypeDecorator[str]): impl = Text(collation="C") cache_ok = True -class INT2VECTOR(TypeDecorator): +class INT2VECTOR(TypeDecorator[Sequence[int]]): impl = ARRAY(SmallInteger) cache_ok = True -class OIDVECTOR(TypeDecorator): +class OIDVECTOR(TypeDecorator[Sequence[int]]): impl = ARRAY(OID) cache_ok = True class _SpaceVector: - def result_processor(self, dialect, coltype): - def process(value): + def result_processor( + self, dialect: Dialect, coltype: object + ) -> _ResultProcessorType[list[int]]: + def process(value: Any) -> Optional[list[int]]: if value is None: return value return [int(p) for p in value.split(" ")] @@ -298,3 +310,35 @@ def process(value): Column("collicurules", Text, info={"server_version": (16,)}), Column("collversion", Text, info={"server_version": (10,)}), ) + +pg_opclass = Table( + "pg_opclass", + pg_catalog_meta, + Column("oid", OID, info={"server_version": (9, 3)}), + Column("opcmethod", NAME), + Column("opcname", NAME), + Column("opsnamespace", OID), + Column("opsowner", OID), + Column("opcfamily", OID), + Column("opcintype", OID), + Column("opcdefault", Boolean), + Column("opckeytype", OID), +) + +pg_inherits = Table( + "pg_inherits", + pg_catalog_meta, + Column("inhrelid", OID), + Column("inhparent", OID), + Column("inhseqno", Integer), + Column("inhdetachpending", Boolean, info={"server_version": (14,)}), +) + +pg_tablespace = Table( + "pg_tablespace", + pg_catalog_meta, + Column("oid", OID), + Column("spcname", NAME), + Column("spcowner", OID), + Column("spcoptions", ARRAY(Text)), +) diff --git a/lib/sqlalchemy/dialects/postgresql/provision.py b/lib/sqlalchemy/dialects/postgresql/provision.py index c76f5f51849..dfe67dce9ca 100644 --- a/lib/sqlalchemy/dialects/postgresql/provision.py +++ b/lib/sqlalchemy/dialects/postgresql/provision.py @@ -1,5 +1,5 @@ # dialects/postgresql/provision.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -95,9 +95,10 @@ def _postgresql_set_default_schema_on_connection( def drop_all_schema_objects_pre_tables(cfg, eng): with eng.connect().execution_options(isolation_level="AUTOCOMMIT") as conn: for xid in conn.exec_driver_sql( - "select gid from pg_prepared_xacts" + "SELECT gid FROM pg_prepared_xacts " + "WHERE database = current_database()" ).scalars(): - conn.exec_driver_sql("ROLLBACK PREPARED '%s'" % xid) + eng.dialect.do_rollback_twophase(conn, xid, recover=True) @drop_all_schema_objects_post_tables.for_db("postgresql") @@ -137,7 +138,13 @@ def prepare_for_drop_tables(config, connection): @upsert.for_db("postgresql") def _upsert( - cfg, table, returning, *, set_lambda=None, sort_by_parameter_order=False + cfg, + table, + returning, + *, + set_lambda=None, + sort_by_parameter_order=False, + index_elements=None, ): from sqlalchemy.dialects.postgresql import insert @@ -146,8 +153,10 @@ def _upsert( table_pk = inspect(table).selectable if set_lambda: + if index_elements is None: + index_elements = table_pk.primary_key stmt = stmt.on_conflict_do_update( - index_elements=table_pk.primary_key, set_=set_lambda(stmt.excluded) + index_elements=index_elements, set_=set_lambda(stmt.excluded) ) else: stmt = stmt.on_conflict_do_nothing() diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg.py b/lib/sqlalchemy/dialects/postgresql/psycopg.py index 4df6f8a4fa2..5422849e82d 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg.py @@ -1,5 +1,5 @@ # dialects/postgresql/psycopg.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -59,6 +59,82 @@ dialect shares most of its behavior with the ``psycopg2`` dialect. Further documentation is available there. +Using psycopg Connection Pooling +-------------------------------- + +The ``psycopg`` driver provides its own connection pool implementation that +may be used in place of SQLAlchemy's pooling functionality. +This pool implementation provides support for fixed and dynamic pool sizes +(including automatic downsizing for unused connections), connection health +pre-checks, and support for both synchronous and asynchronous code +environments. + +Here is an example that uses the sync version of the pool, using +``psycopg_pool >= 3.3`` that introduces support for ``close_returns=True``:: + + import psycopg_pool + from sqlalchemy import create_engine + from sqlalchemy.pool import NullPool + + # Create a psycopg_pool connection pool + my_pool = psycopg_pool.ConnectionPool( + conninfo="postgresql://scott:tiger@localhost/test", + close_returns=True, # Return "closed" active connections to the pool + # ... other pool parameters as desired ... + ) + + # Create an engine that uses the connection pool to get a connection + engine = create_engine( + url="postgresql+psycopg://", # Only need the dialect now + poolclass=NullPool, # Disable SQLAlchemy's default connection pool + creator=my_pool.getconn, # Use Psycopg 3 connection pool to obtain connections + ) + +Similarly an the async example:: + + import psycopg_pool + from sqlalchemy.ext.asyncio import create_async_engine + from sqlalchemy.pool import NullPool + + + async def define_engine(): + # Create a psycopg_pool connection pool + my_pool = psycopg_pool.AsyncConnectionPool( + conninfo="postgresql://scott:tiger@localhost/test", + open=False, # See comment below + close_returns=True, # Return "closed" active connections to the pool + # ... other pool parameters as desired ... + ) + + # Must explicitly open AsyncConnectionPool outside constructor + # https://www.psycopg.org/psycopg3/docs/api/pool.html#psycopg_pool.AsyncConnectionPool + await my_pool.open() + + # Create an engine that uses the connection pool to get a connection + engine = create_async_engine( + url="postgresql+psycopg://", # Only need the dialect now + poolclass=NullPool, # Disable SQLAlchemy's default connection pool + async_creator=my_pool.getconn, # Use Psycopg 3 connection pool to obtain connections + ) + + return engine, my_pool + +The resulting engine may then be used normally. Internally, Psycopg 3 handles +connection pooling:: + + with engine.connect() as conn: + print(conn.scalar(text("select 42"))) + +.. seealso:: + + `Connection pools `_ - + the Psycopg 3 documentation for ``psycopg_pool.ConnectionPool``. + + `Example for older version of psycopg_pool + `_ - + An example about using the ``psycopg_pool<3.3`` that did not have the + ``close_returns``` parameter. + Using a different Cursor class ------------------------------ @@ -97,6 +173,7 @@ import collections import logging import re +from types import NoneType from typing import cast from typing import TYPE_CHECKING @@ -114,6 +191,7 @@ from ... import util from ...connectors.asyncio import AsyncAdapt_dbapi_connection from ...connectors.asyncio import AsyncAdapt_dbapi_cursor +from ...connectors.asyncio import AsyncAdapt_dbapi_module from ...connectors.asyncio import AsyncAdapt_dbapi_ss_cursor from ...sql import sqltypes from ...util.concurrency import await_ @@ -136,19 +214,17 @@ class _PGREGCONFIG(REGCONFIG): class _PGJSON(JSON): def bind_processor(self, dialect): + """psycopg's bind processor is assembled on the type adapter, + but we still need to wrap the value in a psycopg.Json() object""" return self._make_bind_processor(None, dialect._psycopg_Json) - def result_processor(self, dialect, coltype): - return None - class _PGJSONB(JSONB): def bind_processor(self, dialect): + """psycopg's bind processor is assembled on the type adapter, + but we still need to wrap the value in a psycopg.Jsonb() object""" return self._make_bind_processor(None, dialect._psycopg_Jsonb) - def result_processor(self, dialect, coltype): - return None - class _PGJSONIntIndexType(sqltypes.JSON.JSONIntIndexType): __visit_name__ = "json_int_index" @@ -236,8 +312,6 @@ def bind_processor(self, dialect): PGDialect_psycopg, dialect )._psycopg_Multirange - NoneType = type(None) - def to_range(value): if isinstance(value, (str, NoneType, psycopg_Multirange)): return value @@ -298,6 +372,10 @@ class PGDialect_psycopg(_PGDialect_common_psycopg): default_paramstyle = "pyformat" supports_sane_multi_rowcount = True + supports_native_json_serialization = True + supports_native_json_deserialization = True + dialect_injects_custom_json_deserializer = True + execution_ctx_cls = PGExecutionContext_psycopg statement_compiler = PGCompiler_psycopg preparer = PGIdentifierPreparer_psycopg @@ -520,45 +598,12 @@ def is_disconnect(self, e, connection, cursor): return True return False - def _do_prepared_twophase(self, connection, command, recover=False): - dbapi_conn = connection.connection.dbapi_connection - if ( - recover - # don't rely on psycopg providing enum symbols, compare with - # eq/ne - or dbapi_conn.info.transaction_status - != self._psycopg_TransactionStatus.IDLE - ): - dbapi_conn.rollback() - before_autocommit = dbapi_conn.autocommit - try: - if not before_autocommit: - self._do_autocommit(dbapi_conn, True) - with dbapi_conn.cursor() as cursor: - cursor.execute(command) - finally: - if not before_autocommit: - self._do_autocommit(dbapi_conn, before_autocommit) - - def do_rollback_twophase( - self, connection, xid, is_prepared=True, recover=False - ): - if is_prepared: - self._do_prepared_twophase( - connection, f"ROLLBACK PREPARED '{xid}'", recover=recover - ) - else: - self.do_rollback(connection.connection) - - def do_commit_twophase( - self, connection, xid, is_prepared=True, recover=False - ): - if is_prepared: - self._do_prepared_twophase( - connection, f"COMMIT PREPARED '{xid}'", recover=recover - ) - else: - self.do_commit(connection.connection) + def _twophase_idle_check(self, dbapi_conn): + # don't rely on psycopg providing enum symbols, compare with eq/ne + return ( + dbapi_conn.info.transaction_status + == self._psycopg_TransactionStatus.IDLE + ) @util.memoized_property def _dialect_specific_select_one(self): @@ -568,6 +613,8 @@ def _dialect_specific_select_one(self): class AsyncAdapt_psycopg_cursor(AsyncAdapt_dbapi_cursor): __slots__ = () + _awaitable_cursor_close: bool = False + def close(self): self._rows.clear() # Normal cursor just call _close() in a non-sync way. @@ -679,9 +726,25 @@ def cursor(self, name=None, /): else: return AsyncAdapt_psycopg_cursor(self) + def tpc_begin(self, xid): + return await_(self._connection.tpc_begin(xid)) + + def tpc_prepare(self): + return await_(self._connection.tpc_prepare()) + + def tpc_commit(self, xid=None): + return await_(self._connection.tpc_commit(xid)) + + def tpc_rollback(self, xid=None): + return await_(self._connection.tpc_rollback(xid)) + + def tpc_recover(self): + return await_(self._connection.tpc_recover()) + -class PsycopgAdaptDBAPI: +class PsycopgAdaptDBAPI(AsyncAdapt_dbapi_module): def __init__(self, psycopg, ExecStatus) -> None: + super().__init__(psycopg) self.psycopg = psycopg self.ExecStatus = ExecStatus @@ -693,8 +756,8 @@ def connect(self, *arg, **kw): creator_fn = kw.pop( "async_creator_fn", self.psycopg.AsyncConnection.connect ) - return AsyncAdapt_psycopg_connection( - self, await_(creator_fn(*arg, **kw)) + return await_( + AsyncAdapt_psycopg_connection.create(self, creator_fn(*arg, **kw)) ) diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py index eeb7604f796..bde9e1c93e6 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py @@ -1,5 +1,5 @@ # dialects/postgresql/psycopg2.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -171,9 +171,6 @@ is repaired, previously ports were not correctly interpreted in this context. libpq comma-separated format is also now supported. -.. versionadded:: 1.3.20 Support for multiple hosts in PostgreSQL connection - string. - .. seealso:: `libpq connection strings `_ - please refer @@ -198,8 +195,6 @@ In the above form, a blank "dsn" string is passed to the ``psycopg2.connect()`` function which in turn represents an empty DSN passed to libpq. -.. versionadded:: 1.3.2 support for parameter-less connections with psycopg2. - .. seealso:: `Environment Variables\ @@ -799,35 +794,8 @@ def do_executemany(self, cursor, statement, parameters, context=None): else: cursor.executemany(statement, parameters) - def do_begin_twophase(self, connection, xid): - connection.connection.tpc_begin(xid) - - def do_prepare_twophase(self, connection, xid): - connection.connection.tpc_prepare() - - def _do_twophase(self, dbapi_conn, operation, xid, recover=False): - if recover: - if dbapi_conn.status != self._psycopg2_extensions.STATUS_READY: - dbapi_conn.rollback() - operation(xid) - else: - operation() - - def do_rollback_twophase( - self, connection, xid, is_prepared=True, recover=False - ): - dbapi_conn = connection.connection.dbapi_connection - self._do_twophase( - dbapi_conn, dbapi_conn.tpc_rollback, xid, recover=recover - ) - - def do_commit_twophase( - self, connection, xid, is_prepared=True, recover=False - ): - dbapi_conn = connection.connection.dbapi_connection - self._do_twophase( - dbapi_conn, dbapi_conn.tpc_commit, xid, recover=recover - ) + def _twophase_idle_check(self, dbapi_conn): + return dbapi_conn.status == self._psycopg2_extensions.STATUS_READY @util.memoized_instancemethod def _hstore_oids(self, dbapi_connection): diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py b/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py index 55e17607044..f4dcfc7014a 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2cffi.py @@ -1,5 +1,5 @@ # dialects/postgresql/psycopg2cffi.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/dialects/postgresql/ranges.py b/lib/sqlalchemy/dialects/postgresql/ranges.py index 93253570c1b..7d423ef0123 100644 --- a/lib/sqlalchemy/dialects/postgresql/ranges.py +++ b/lib/sqlalchemy/dialects/postgresql/ranges.py @@ -1,5 +1,5 @@ # dialects/postgresql/ranges.py -# Copyright (C) 2013-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2013-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -16,6 +16,7 @@ from typing import cast from typing import Generic from typing import List +from typing import Literal from typing import Optional from typing import overload from typing import Sequence @@ -35,9 +36,8 @@ from .operators import STRICTLY_RIGHT_OF from ... import types as sqltypes from ...sql import operators +from ...sql.operators import OperatorClass from ...sql.type_api import TypeEngine -from ...util import py310 -from ...util.typing import Literal if TYPE_CHECKING: from ...sql.elements import ColumnElement @@ -48,15 +48,8 @@ _BoundsType = Literal["()", "[)", "(]", "[]"] -if py310: - dc_slots = {"slots": True} - dc_kwonly = {"kw_only": True} -else: - dc_slots = {} - dc_kwonly = {} - -@dataclasses.dataclass(frozen=True, **dc_slots) +@dataclasses.dataclass(frozen=True, slots=True) class Range(Generic[_T]): """Represent a PostgreSQL range. @@ -85,32 +78,8 @@ class Range(Generic[_T]): upper: Optional[_T] = None """the upper bound""" - if TYPE_CHECKING: - bounds: _BoundsType = dataclasses.field(default="[)") - empty: bool = dataclasses.field(default=False) - else: - bounds: _BoundsType = dataclasses.field(default="[)", **dc_kwonly) - empty: bool = dataclasses.field(default=False, **dc_kwonly) - - if not py310: - - def __init__( - self, - lower: Optional[_T] = None, - upper: Optional[_T] = None, - *, - bounds: _BoundsType = "[)", - empty: bool = False, - ): - # no __slots__ either so we can update dict - self.__dict__.update( - { - "lower": lower, - "upper": upper, - "bounds": bounds, - "empty": empty, - } - ) + bounds: _BoundsType = dataclasses.field(default="[)", kw_only=True) + empty: bool = dataclasses.field(default=False, kw_only=True) def __bool__(self) -> bool: return not self.empty @@ -271,9 +240,9 @@ def _compare_edges( value2 += step value2_inc = False - if value1 < value2: # type: ignore + if value1 < value2: return -1 - elif value1 > value2: # type: ignore + elif value1 > value2: return 1 elif only_values: return 0 @@ -743,6 +712,8 @@ class AbstractRange(sqltypes.TypeEngine[_T]): render_bind_cast = True + operator_classes = OperatorClass.NUMERIC + __abstract__ = True @overload diff --git a/lib/sqlalchemy/dialects/postgresql/types.py b/lib/sqlalchemy/dialects/postgresql/types.py index 1aed2bf4724..a8d213cca39 100644 --- a/lib/sqlalchemy/dialects/postgresql/types.py +++ b/lib/sqlalchemy/dialects/postgresql/types.py @@ -1,5 +1,5 @@ # dialects/postgresql/types.py -# Copyright (C) 2013-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2013-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -8,21 +8,26 @@ import datetime as dt from typing import Any +from typing import Literal from typing import Optional from typing import overload from typing import Type from typing import TYPE_CHECKING from uuid import UUID as _python_UUID +from .bitstring import BitString from ...sql import sqltypes from ...sql import type_api -from ...util.typing import Literal +from ...sql.type_api import TypeEngine +from ...types import OperatorClass if TYPE_CHECKING: from ...engine.interfaces import Dialect + from ...sql.operators import ColumnOperators from ...sql.operators import OperatorType + from ...sql.type_api import _BindProcessorType from ...sql.type_api import _LiteralProcessorType - from ...sql.type_api import TypeEngine + from ...sql.type_api import _ResultProcessorType _DECIMAL_TYPES = (1231, 1700) _FLOAT_TYPES = (700, 701, 1021, 1022) @@ -53,6 +58,7 @@ class BYTEA(sqltypes.LargeBinary): class _NetworkAddressTypeMixin: + operator_classes = OperatorClass.BASE | OperatorClass.COMPARISON def coerce_compared_value( self, op: Optional[OperatorType], value: Any @@ -130,8 +136,6 @@ class NumericMoney(TypeDecorator): def column_expression(self, column: Any): return cast(column, Numeric()) - .. versionadded:: 1.2 - """ # noqa: E501 __visit_name__ = "MONEY" @@ -142,6 +146,8 @@ class OID(sqltypes.TypeEngine[int]): __visit_name__ = "OID" + operator_classes = OperatorClass.BASE | OperatorClass.COMPARISON + class REGCONFIG(sqltypes.TypeEngine[str]): """Provide the PostgreSQL REGCONFIG type. @@ -152,6 +158,8 @@ class REGCONFIG(sqltypes.TypeEngine[str]): __visit_name__ = "REGCONFIG" + operator_classes = OperatorClass.BASE | OperatorClass.COMPARISON + class TSQUERY(sqltypes.TypeEngine[str]): """Provide the PostgreSQL TSQUERY type. @@ -162,16 +170,16 @@ class TSQUERY(sqltypes.TypeEngine[str]): __visit_name__ = "TSQUERY" + operator_classes = OperatorClass.BASE | OperatorClass.COMPARISON -class REGCLASS(sqltypes.TypeEngine[str]): - """Provide the PostgreSQL REGCLASS type. - - .. versionadded:: 1.2.7 - """ +class REGCLASS(sqltypes.TypeEngine[str]): + """Provide the PostgreSQL REGCLASS type.""" __visit_name__ = "REGCLASS" + operator_classes = OperatorClass.BASE | OperatorClass.COMPARISON + class TIMESTAMP(sqltypes.TIMESTAMP): """Provide the PostgreSQL TIMESTAMP type.""" @@ -229,8 +237,6 @@ def __init__( to be limited, such as ``"YEAR"``, ``"MONTH"``, ``"DAY TO HOUR"``, etc. - .. versionadded:: 1.2 - """ self.precision = precision self.fields = fields @@ -264,9 +270,24 @@ def process(value: dt.timedelta) -> str: PGInterval = INTERVAL -class BIT(sqltypes.TypeEngine[int]): +class BIT(sqltypes.TypeEngine[BitString]): + """Represent the PostgreSQL BIT type. + + The :class:`_postgresql.BIT` type yields values in the form of the + :class:`_postgresql.BitString` Python value type. + + .. versionchanged:: 2.1 The :class:`_postgresql.BIT` type now works + with :class:`_postgresql.BitString` values rather than plain strings. + + """ + + render_bind_cast = True __visit_name__ = "BIT" + operator_classes = ( + OperatorClass.BASE | OperatorClass.COMPARISON | OperatorClass.BITWISE + ) + def __init__( self, length: Optional[int] = None, varying: bool = False ) -> None: @@ -278,6 +299,58 @@ def __init__( self.length = length or 1 self.varying = varying + def bind_processor( + self, dialect: Dialect + ) -> _BindProcessorType[BitString]: + def bound_value(value: Any) -> Any: + if isinstance(value, BitString): + return str(value) + return value + + return bound_value + + def result_processor( + self, dialect: Dialect, coltype: object + ) -> _ResultProcessorType[BitString]: + def from_result_value(value: Any) -> Any: + if value is not None: + value = BitString(value) + return value + + return from_result_value + + def coerce_compared_value( + self, op: OperatorType | None, value: Any + ) -> TypeEngine[Any]: + if isinstance(value, str): + return self + return super().coerce_compared_value(op, value) + + @property + def python_type(self) -> type[Any]: + return BitString + + class comparator_factory(TypeEngine.Comparator[BitString]): + def __lshift__(self, other: Any) -> ColumnOperators: + return self.bitwise_lshift(other) + + def __rshift__(self, other: Any) -> ColumnOperators: + return self.bitwise_rshift(other) + + def __and__(self, other: Any) -> ColumnOperators: + return self.bitwise_and(other) + + def __or__(self, other: Any) -> ColumnOperators: + return self.bitwise_or(other) + + # NOTE: __xor__ is not defined on sql.operators.ColumnOperators. + # Use `bitwise_xor` directly instead. + # def __xor__(self, other: Any) -> ColumnOperators: + # return self.bitwise_xor(other) + + def __invert__(self) -> ColumnOperators: + return self.bitwise_not() + PGBit = BIT @@ -297,6 +370,8 @@ class TSVECTOR(sqltypes.TypeEngine[str]): __visit_name__ = "TSVECTOR" + operator_classes = OperatorClass.STRING + class CITEXT(sqltypes.TEXT): """Provide the PostgreSQL CITEXT type. diff --git a/lib/sqlalchemy/dialects/sqlite/__init__.py b/lib/sqlalchemy/dialects/sqlite/__init__.py index 7b381fa6f52..8609082242e 100644 --- a/lib/sqlalchemy/dialects/sqlite/__init__.py +++ b/lib/sqlalchemy/dialects/sqlite/__init__.py @@ -1,5 +1,5 @@ # dialects/sqlite/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/dialects/sqlite/aiosqlite.py b/lib/sqlalchemy/dialects/sqlite/aiosqlite.py index ab27e834620..0353b39e828 100644 --- a/lib/sqlalchemy/dialects/sqlite/aiosqlite.py +++ b/lib/sqlalchemy/dialects/sqlite/aiosqlite.py @@ -1,10 +1,9 @@ # dialects/sqlite/aiosqlite.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors r""" @@ -50,33 +49,10 @@ Serializable isolation / Savepoints / Transactional DDL (asyncio version) ------------------------------------------------------------------------- -Similarly to pysqlite, aiosqlite does not support SAVEPOINT feature. +A newly revised version of this important section is now available +at the top level of the SQLAlchemy SQLite documentation, in the section +:ref:`sqlite_transactions`. -The solution is similar to :ref:`pysqlite_serializable`. This is achieved by the event listeners in async:: - - from sqlalchemy import create_engine, event - from sqlalchemy.ext.asyncio import create_async_engine - - engine = create_async_engine("sqlite+aiosqlite:///myfile.db") - - - @event.listens_for(engine.sync_engine, "connect") - def do_connect(dbapi_connection, connection_record): - # disable aiosqlite's emitting of the BEGIN statement entirely. - # also stops it from emitting COMMIT before any DDL. - dbapi_connection.isolation_level = None - - - @event.listens_for(engine.sync_engine, "begin") - def do_begin(conn): - # emit our own BEGIN - conn.exec_driver_sql("BEGIN") - -.. warning:: When using the above recipe, it is advised to not use the - :paramref:`.Connection.execution_options.isolation_level` setting on - :class:`_engine.Connection` and :func:`_sa.create_engine` - with the SQLite driver, - as this function necessarily will also alter the ".isolation_level" setting. .. _aiosqlite_pooling: @@ -101,18 +77,37 @@ def do_begin(conn): :paramref:`_sa.create_engine.poolclass` parameter. """ # noqa +from __future__ import annotations import asyncio from functools import partial +from threading import Thread +from types import ModuleType +from typing import Any +from typing import cast +from typing import NoReturn +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union from .base import SQLiteExecutionContext from .pysqlite import SQLiteDialect_pysqlite from ... import pool from ...connectors.asyncio import AsyncAdapt_dbapi_connection from ...connectors.asyncio import AsyncAdapt_dbapi_cursor +from ...connectors.asyncio import AsyncAdapt_dbapi_module from ...connectors.asyncio import AsyncAdapt_dbapi_ss_cursor +from ...connectors.asyncio import AsyncAdapt_terminate from ...util.concurrency import await_ +if TYPE_CHECKING: + from ...connectors.asyncio import AsyncIODBAPIConnection + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import DBAPICursor + from ...engine.interfaces import DBAPIModule + from ...engine.url import URL + from ...pool.base import PoolProxiedConnection + class AsyncAdapt_aiosqlite_cursor(AsyncAdapt_dbapi_cursor): __slots__ = () @@ -122,24 +117,28 @@ class AsyncAdapt_aiosqlite_ss_cursor(AsyncAdapt_dbapi_ss_cursor): __slots__ = () -class AsyncAdapt_aiosqlite_connection(AsyncAdapt_dbapi_connection): +class AsyncAdapt_aiosqlite_connection( + AsyncAdapt_terminate, AsyncAdapt_dbapi_connection +): __slots__ = () _cursor_cls = AsyncAdapt_aiosqlite_cursor _ss_cursor_cls = AsyncAdapt_aiosqlite_ss_cursor @property - def isolation_level(self): - return self._connection.isolation_level + def isolation_level(self) -> Optional[str]: + return cast(str, self._connection.isolation_level) @isolation_level.setter - def isolation_level(self, value): + def isolation_level(self, value: Optional[str]) -> None: # aiosqlite's isolation_level setter works outside the Thread # that it's supposed to, necessitating setting check_same_thread=False. # for improved stability, we instead invent our own awaitable version # using aiosqlite's async queue directly. - def set_iso(connection, value): + def set_iso( + connection: AsyncAdapt_aiosqlite_connection, value: Optional[str] + ) -> None: connection.isolation_level = value function = partial(set_iso, self._connection._conn, value) @@ -148,25 +147,25 @@ def set_iso(connection, value): self._connection._tx.put_nowait((future, function)) try: - return await_(future) + await_(future) except Exception as error: self._handle_exception(error) - def create_function(self, *args, **kw): + def create_function(self, *args: Any, **kw: Any) -> None: try: await_(self._connection.create_function(*args, **kw)) except Exception as error: self._handle_exception(error) - def rollback(self): + def rollback(self) -> None: if self._connection._connection: super().rollback() - def commit(self): + def commit(self) -> None: if self._connection._connection: super().commit() - def close(self): + def close(self) -> None: try: await_(self._connection.close()) except ValueError: @@ -182,24 +181,47 @@ def close(self): except Exception as error: self._handle_exception(error) - def _handle_exception(self, error): + @classmethod + def _handle_exception_no_connection( + cls, dbapi: Any, error: Exception + ) -> NoReturn: if isinstance(error, ValueError) and error.args[0].lower() in ( "no active connection", "connection closed", ): - raise self.dbapi.sqlite.OperationalError(error.args[0]) from error + raise dbapi.sqlite.OperationalError(error.args[0]) from error else: - super()._handle_exception(error) + super()._handle_exception_no_connection(dbapi, error) + async def _terminate_graceful_close(self) -> None: + """Try to close connection gracefully""" + await self._connection.close() -class AsyncAdapt_aiosqlite_dbapi: - def __init__(self, aiosqlite, sqlite): + def _terminate_force_close(self) -> None: + """Terminate the connection""" + + # this was added in aiosqlite 0.22.1. if stop() is not present, + # the dialect should indicate has_terminate=False + try: + meth = self._connection.stop + except AttributeError as ae: + raise NotImplementedError( + "terminate_force_close() not implemented by this DBAPI shim" + ) from ae + else: + meth() + + +class AsyncAdapt_aiosqlite_dbapi(AsyncAdapt_dbapi_module): + def __init__(self, aiosqlite: ModuleType, sqlite: ModuleType): + super().__init__(aiosqlite, dbapi_module=sqlite) self.aiosqlite = aiosqlite self.sqlite = sqlite self.paramstyle = "qmark" + self.has_stop = hasattr(aiosqlite.Connection, "stop") self._init_dbapi_attributes() - def _init_dbapi_attributes(self): + def _init_dbapi_attributes(self) -> None: for name in ( "DatabaseError", "Error", @@ -218,23 +240,26 @@ def _init_dbapi_attributes(self): for name in ("Binary",): setattr(self, name, getattr(self.sqlite, name)) - def connect(self, *arg, **kw): + def connect(self, *arg: Any, **kw: Any) -> AsyncAdapt_aiosqlite_connection: creator_fn = kw.pop("async_creator_fn", None) if creator_fn: connection = creator_fn(*arg, **kw) else: connection = self.aiosqlite.connect(*arg, **kw) - # it's a Thread. you'll thank us later - connection.daemon = True - return AsyncAdapt_aiosqlite_connection( - self, - await_(connection), - ) + # aiosqlite uses a Thread. you'll thank us later + if isinstance(connection, Thread): + # Connection itself was a thread in version prior to 0.22 + connection.daemon = True + else: + # in 0.22+ instead it contains a thread. + connection._thread.daemon = True + + return AsyncAdapt_aiosqlite_connection(self, await_(connection)) class SQLiteExecutionContext_aiosqlite(SQLiteExecutionContext): - def create_server_side_cursor(self): + def create_server_side_cursor(self) -> DBAPICursor: return self._dbapi_connection.cursor(server_side=True) @@ -243,25 +268,37 @@ class SQLiteDialect_aiosqlite(SQLiteDialect_pysqlite): supports_statement_cache = True is_async = True + has_terminate = True supports_server_side_cursors = True execution_ctx_cls = SQLiteExecutionContext_aiosqlite + def __init__(self, **kwargs: Any): + super().__init__(**kwargs) + if self.dbapi and not self.dbapi.has_stop: + self.has_terminate = False + @classmethod - def import_dbapi(cls): + def import_dbapi(cls) -> AsyncAdapt_aiosqlite_dbapi: return AsyncAdapt_aiosqlite_dbapi( __import__("aiosqlite"), __import__("sqlite3") ) @classmethod - def get_pool_class(cls, url): + def get_pool_class(cls, url: URL) -> type[pool.Pool]: if cls._is_url_file_db(url): return pool.AsyncAdaptedQueuePool else: return pool.StaticPool - def is_disconnect(self, e, connection, cursor): + def is_disconnect( + self, + e: DBAPIModule.Error, + connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]], + cursor: Optional[DBAPICursor], + ) -> bool: + self.dbapi = cast("DBAPIModule", self.dbapi) if isinstance(e, self.dbapi.OperationalError): err_lower = str(e).lower() if ( @@ -272,8 +309,13 @@ def is_disconnect(self, e, connection, cursor): return super().is_disconnect(e, connection, cursor) - def get_driver_connection(self, connection): - return connection._connection + def get_driver_connection( + self, connection: DBAPIConnection + ) -> AsyncIODBAPIConnection: + return connection._connection # type: ignore[no-any-return] + + def do_terminate(self, dbapi_connection: DBAPIConnection) -> None: + dbapi_connection.terminate() dialect = SQLiteDialect_aiosqlite diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index 7b8e42a2854..61d61d9a1e4 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -1,5 +1,5 @@ # dialects/sqlite/base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -136,99 +136,199 @@ def bi_c(element, compiler, **kw): `Datatypes In SQLite Version 3 `_ -.. _sqlite_concurrency: - -Database Locking Behavior / Concurrency ---------------------------------------- - -SQLite is not designed for a high level of write concurrency. The database -itself, being a file, is locked completely during write operations within -transactions, meaning exactly one "connection" (in reality a file handle) -has exclusive access to the database during this period - all other -"connections" will be blocked during this time. - -The Python DBAPI specification also calls for a connection model that is -always in a transaction; there is no ``connection.begin()`` method, -only ``connection.commit()`` and ``connection.rollback()``, upon which a -new transaction is to be begun immediately. This may seem to imply -that the SQLite driver would in theory allow only a single filehandle on a -particular database file at any time; however, there are several -factors both within SQLite itself as well as within the pysqlite driver -which loosen this restriction significantly. - -However, no matter what locking modes are used, SQLite will still always -lock the database file once a transaction is started and DML (e.g. INSERT, -UPDATE, DELETE) has at least been emitted, and this will block -other transactions at least at the point that they also attempt to emit DML. -By default, the length of time on this block is very short before it times out -with an error. - -This behavior becomes more critical when used in conjunction with the -SQLAlchemy ORM. SQLAlchemy's :class:`.Session` object by default runs -within a transaction, and with its autoflush model, may emit DML preceding -any SELECT statement. This may lead to a SQLite database that locks -more quickly than is expected. The locking mode of SQLite and the pysqlite -driver can be manipulated to some degree, however it should be noted that -achieving a high degree of write-concurrency with SQLite is a losing battle. - -For more information on SQLite's lack of write concurrency by design, please -see -`Situations Where Another RDBMS May Work Better - High Concurrency -`_ near the bottom of the page. - -The following subsections introduce areas that are impacted by SQLite's -file-based architecture and additionally will usually require workarounds to -work when using the pysqlite driver. +.. _sqlite_transactions: + +Transactions with SQLite and the sqlite3 driver +----------------------------------------------- + +As a file-based database, SQLite's approach to transactions differs from +traditional databases in many ways. Additionally, the ``sqlite3`` driver +standard with Python (as well as the async version ``aiosqlite`` which builds +on top of it) has several quirks, workarounds, and API features in the +area of transaction control, all of which generally need to be addressed when +constructing a SQLAlchemy application that uses SQLite. + +Legacy Transaction Mode with the sqlite3 driver +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The most important aspect of transaction handling with the sqlite3 driver is +that it defaults (which will continue through Python 3.15 before being +removed in Python 3.16) to legacy transactional behavior which does +not strictly follow :pep:`249`. The way in which the driver diverges from the +PEP is that it does not "begin" a transaction automatically as dictated by +:pep:`249` except in the case of DML statements, e.g. INSERT, UPDATE, and +DELETE. Normally, :pep:`249` dictates that a BEGIN must be emitted upon +the first SQL statement of any kind, so that all subsequent operations will +be established within a transaction until ``connection.commit()`` has been +called. The ``sqlite3`` driver, in an effort to be easier to use in +highly concurrent environments, skips this step for DQL (e.g. SELECT) statements, +and also skips it for DDL (e.g. CREATE TABLE etc.) statements for more legacy +reasons. Statements such as SAVEPOINT are also skipped. + +In modern versions of the ``sqlite3`` driver as of Python 3.12, this legacy +mode of operation is referred to as +`"legacy transaction control" `_, and is in +effect by default due to the ``Connection.autocommit`` parameter being set to +the constant ``sqlite3.LEGACY_TRANSACTION_CONTROL``. Prior to Python 3.12, +the ``Connection.autocommit`` attribute did not exist. + +The implications of legacy transaction mode include: + +* **Incorrect support for transactional DDL** - statements like CREATE TABLE, ALTER TABLE, + CREATE INDEX etc. will not automatically BEGIN a transaction if one were not + started already, leading to the changes by each statement being + "autocommitted" immediately unless BEGIN were otherwise emitted first. Very + old (pre Python 3.6) versions of SQLite would also force a COMMIT for these + operations even if a transaction were present, however this is no longer the + case. +* **SERIALIZABLE behavior not fully functional** - SQLite's transaction isolation + behavior is normally consistent with SERIALIZABLE isolation, as it is a file- + based system that locks the database file entirely for write operations, + preventing COMMIT until all reader transactions (and associated file locks) + have completed. However, sqlite3's legacy transaction mode fails to emit BEGIN for SELECT + statements, which causes these SELECT statements to no longer be "repeatable", + failing one of the consistency guarantees of SERIALIZABLE. +* **Incorrect behavior for SAVEPOINT** - as the SAVEPOINT statement does not + imply a BEGIN, a new SAVEPOINT emitted before a BEGIN will function on its + own but fails to participate in the enclosing transaction, meaning a ROLLBACK + of the transaction will not rollback elements that were part of a released + savepoint. + +Legacy transaction mode first existed in order to facilitate working around +SQLite's file locks. Because SQLite relies upon whole-file locks, it is easy to +get "database is locked" errors, particularly when newer features like "write +ahead logging" are disabled. This is a key reason why ``sqlite3``'s legacy +transaction mode is still the default mode of operation; disabling it will +produce behavior that is more susceptible to locked database errors. However +note that **legacy transaction mode will no longer be the default** in a future +Python version (3.16 as of this writing). + +.. _sqlite_enabling_transactions: + +Enabling Non-Legacy SQLite Transactional Modes with the sqlite3 or aiosqlite driver +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Current SQLAlchemy support allows either for setting the +``.Connection.autocommit`` attribute, most directly by using a +:func:`._sa.create_engine` parameter, or if on an older version of Python where +the attribute is not available, using event hooks to control the behavior of +BEGIN. + +* **Enabling modern sqlite3 transaction control via the autocommit connect parameter** (Python 3.12 and above) + + To use SQLite in the mode described at `Transaction control via the autocommit attribute `_, + the most straightforward approach is to set the attribute to its recommended value + of ``False`` at the connect level using :paramref:`_sa.create_engine.connect_args``:: + + from sqlalchemy import create_engine + + engine = create_engine( + "sqlite:///myfile.db", connect_args={"autocommit": False} + ) + + This parameter is also passed through when using the aiosqlite driver:: + + from sqlalchemy.ext.asyncio import create_async_engine + + engine = create_async_engine( + "sqlite+aiosqlite:///myfile.db", connect_args={"autocommit": False} + ) + + The parameter can also be set at the attribute level using the :meth:`.PoolEvents.connect` + event hook, however this will only work for sqlite3, as aiosqlite does not yet expose this + attribute on its ``Connection`` object:: + + from sqlalchemy import create_engine, event + + engine = create_engine("sqlite:///myfile.db") + + + @event.listens_for(engine, "connect") + def do_connect(dbapi_connection, connection_record): + # enable autocommit=False mode + dbapi_connection.autocommit = False + +* **Using SQLAlchemy to emit BEGIN in lieu of SQLite's transaction control** (all Python versions, sqlite3 and aiosqlite) + + For older versions of ``sqlite3`` or for cross-compatibility with older and + newer versions, SQLAlchemy can also take over the job of transaction control. + This is achieved by using the :meth:`.ConnectionEvents.begin` hook + to emit the "BEGIN" command directly, while also disabling SQLite's control + of this command using the :meth:`.PoolEvents.connect` event hook to set the + ``Connection.isolation_level`` attribute to ``None``:: + + + from sqlalchemy import create_engine, event + + engine = create_engine("sqlite:///myfile.db") + + + @event.listens_for(engine, "connect") + def do_connect(dbapi_connection, connection_record): + # disable sqlite3's emitting of the BEGIN statement entirely. + dbapi_connection.isolation_level = None + + + @event.listens_for(engine, "begin") + def do_begin(conn): + # emit our own BEGIN. sqlite3 still emits COMMIT/ROLLBACK correctly + conn.exec_driver_sql("BEGIN") + + When using the asyncio variant ``aiosqlite``, refer to ``engine.sync_engine`` + as in the example below:: + + from sqlalchemy import create_engine, event + from sqlalchemy.ext.asyncio import create_async_engine + + engine = create_async_engine("sqlite+aiosqlite:///myfile.db") + + + @event.listens_for(engine.sync_engine, "connect") + def do_connect(dbapi_connection, connection_record): + # disable aiosqlite's emitting of the BEGIN statement entirely. + dbapi_connection.isolation_level = None + + + @event.listens_for(engine.sync_engine, "begin") + def do_begin(conn): + # emit our own BEGIN. aiosqlite still emits COMMIT/ROLLBACK correctly + conn.exec_driver_sql("BEGIN") .. _sqlite_isolation_level: -Transaction Isolation Level / Autocommit ----------------------------------------- - -SQLite supports "transaction isolation" in a non-standard way, along two -axes. One is that of the -`PRAGMA read_uncommitted `_ -instruction. This setting can essentially switch SQLite between its -default mode of ``SERIALIZABLE`` isolation, and a "dirty read" isolation -mode normally referred to as ``READ UNCOMMITTED``. - -SQLAlchemy ties into this PRAGMA statement using the -:paramref:`_sa.create_engine.isolation_level` parameter of -:func:`_sa.create_engine`. -Valid values for this parameter when used with SQLite are ``"SERIALIZABLE"`` -and ``"READ UNCOMMITTED"`` corresponding to a value of 0 and 1, respectively. -SQLite defaults to ``SERIALIZABLE``, however its behavior is impacted by -the pysqlite driver's default behavior. - -When using the pysqlite driver, the ``"AUTOCOMMIT"`` isolation level is also -available, which will alter the pysqlite connection using the ``.isolation_level`` -attribute on the DBAPI connection and set it to None for the duration -of the setting. - -.. versionadded:: 1.3.16 added support for SQLite AUTOCOMMIT isolation level - when using the pysqlite / sqlite3 SQLite driver. - - -The other axis along which SQLite's transactional locking is impacted is -via the nature of the ``BEGIN`` statement used. The three varieties -are "deferred", "immediate", and "exclusive", as described at -`BEGIN TRANSACTION `_. A straight -``BEGIN`` statement uses the "deferred" mode, where the database file is -not locked until the first read or write operation, and read access remains -open to other transactions until the first write operation. But again, -it is critical to note that the pysqlite driver interferes with this behavior -by *not even emitting BEGIN* until the first write operation. +Using SQLAlchemy's Driver Level AUTOCOMMIT Feature with SQLite +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. warning:: +SQLAlchemy has a comprehensive database isolation feature with optional +autocommit support that is introduced in the section :ref:`dbapi_autocommit`. - SQLite's transactional scope is impacted by unresolved - issues in the pysqlite driver, which defers BEGIN statements to a greater - degree than is often feasible. See the section :ref:`pysqlite_serializable` - or :ref:`aiosqlite_serializable` for techniques to work around this behavior. +For the ``sqlite3`` and ``aiosqlite`` drivers, SQLAlchemy only includes +built-in support for "AUTOCOMMIT". Note that this mode is currently incompatible +with the non-legacy isolation mode hooks documented in the previous +section at :ref:`sqlite_enabling_transactions`. -.. seealso:: +To use the ``sqlite3`` driver with SQLAlchemy driver-level autocommit, +create an engine setting the :paramref:`_sa.create_engine.isolation_level` +parameter to "AUTOCOMMIT":: + + eng = create_engine("sqlite:///myfile.db", isolation_level="AUTOCOMMIT") + +When using the above mode, any event hooks that set the sqlite3 ``Connection.autocommit`` +parameter away from its default of ``sqlite3.LEGACY_TRANSACTION_CONTROL`` +as well as hooks that emit ``BEGIN`` should be disabled. + +Additional Reading for SQLite / sqlite3 transaction control +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Links with important information on SQLite, the sqlite3 driver, +as well as long historical conversations on how things got to their current state: + +* `Isolation in SQLite `_ - on the SQLite website +* `Transaction control `_ - describes the sqlite3 autocommit attribute as well + as the legacy isolation_level attribute. +* `sqlite3 SELECT does not BEGIN a transaction, but should according to spec `_ - imported Python standard library issue on github +* `sqlite3 module breaks transactions and potentially corrupts data `_ - imported Python standard library issue on github - :ref:`dbapi_autocommit` INSERT/UPDATE/DELETE...RETURNING --------------------------------- @@ -268,38 +368,6 @@ def bi_c(element, compiler, **kw): .. versionadded:: 2.0 Added support for SQLite RETURNING -SAVEPOINT Support ----------------------------- - -SQLite supports SAVEPOINTs, which only function once a transaction is -begun. SQLAlchemy's SAVEPOINT support is available using the -:meth:`_engine.Connection.begin_nested` method at the Core level, and -:meth:`.Session.begin_nested` at the ORM level. However, SAVEPOINTs -won't work at all with pysqlite unless workarounds are taken. - -.. warning:: - - SQLite's SAVEPOINT feature is impacted by unresolved - issues in the pysqlite and aiosqlite drivers, which defer BEGIN statements - to a greater degree than is often feasible. See the sections - :ref:`pysqlite_serializable` and :ref:`aiosqlite_serializable` - for techniques to work around this behavior. - -Transactional DDL ----------------------------- - -The SQLite database supports transactional :term:`DDL` as well. -In this case, the pysqlite driver is not only failing to start transactions, -it also is ending any existing transaction when DDL is detected, so again, -workarounds are required. - -.. warning:: - - SQLite's transactional DDL is impacted by unresolved issues - in the pysqlite driver, which fails to emit BEGIN and additionally - forces a COMMIT to cancel any transaction when DDL is encountered. - See the section :ref:`pysqlite_serializable` - for techniques to work around this behavior. .. _sqlite_foreign_keys: @@ -328,10 +396,18 @@ def bi_c(element, compiler, **kw): @event.listens_for(Engine, "connect") def set_sqlite_pragma(dbapi_connection, connection_record): + # the sqlite3 driver will not set PRAGMA foreign_keys + # if autocommit=False; set to True temporarily + ac = dbapi_connection.autocommit + dbapi_connection.autocommit = True + cursor = dbapi_connection.cursor() cursor.execute("PRAGMA foreign_keys=ON") cursor.close() + # restore previous autocommit setting + dbapi_connection.autocommit = ac + .. warning:: When SQLite foreign keys are enabled, it is **not possible** @@ -379,9 +455,6 @@ def set_sqlite_pragma(dbapi_connection, connection_record): `ON CONFLICT `_ - in the SQLite documentation -.. versionadded:: 1.3 - - The ``sqlite_on_conflict`` parameters accept a string argument which is just the resolution name to be chosen, which on SQLite can be one of ROLLBACK, ABORT, FAIL, IGNORE, and REPLACE. For example, to add a UNIQUE constraint @@ -916,7 +989,10 @@ def set_sqlite_pragma(dbapi_connection, connection_record): import datetime import numbers import re +from typing import Any +from typing import Callable from typing import Optional +from typing import TYPE_CHECKING from .json import JSON from .json import JSONIndexType @@ -932,8 +1008,8 @@ def set_sqlite_pragma(dbapi_connection, connection_record): from ...engine import reflection from ...engine.reflection import ReflectionDefaults from ...sql import coercions -from ...sql import ColumnElement from ...sql import compiler +from ...sql import ddl as sa_ddl from ...sql import elements from ...sql import roles from ...sql import schema @@ -950,6 +1026,14 @@ def set_sqlite_pragma(dbapi_connection, connection_record): from ...types import TIMESTAMP # noqa from ...types import VARCHAR # noqa +if TYPE_CHECKING: + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import Dialect + from ...engine.interfaces import IsolationLevel + from ...sql.sqltypes import _JSON_VALUE + from ...sql.type_api import _BindProcessorType + from ...sql.type_api import _ResultProcessorType + class _SQliteJson(JSON): def result_processor(self, dialect, coltype): @@ -1049,6 +1133,10 @@ class DATETIME(_DateTimeMixin, sqltypes.DateTime): regexp=r"(\d+)/(\d+)/(\d+) (\d+)-(\d+)-(\d+)", ) + :param truncate_microseconds: when ``True`` microseconds will be truncated + from the datetime. Can't be specified together with ``storage_format`` + or ``regexp``. + :param storage_format: format string which will be applied to the dict with keys year, month, day, hour, minute, second, and microsecond. @@ -1084,7 +1172,9 @@ def __init__(self, *args, **kwargs): "%(hour)02d:%(minute)02d:%(second)02d" ) - def bind_processor(self, dialect): + def bind_processor( + self, dialect: Dialect + ) -> Optional[_BindProcessorType[Any]]: datetime_datetime = datetime.datetime datetime_date = datetime.date format_ = self._storage_format @@ -1120,7 +1210,9 @@ def process(value): return process - def result_processor(self, dialect, coltype): + def result_processor( + self, dialect: Dialect, coltype: object + ) -> Optional[_ResultProcessorType[Any]]: if self._reg: return processors.str_to_datetime_processor_factory( self._reg, datetime.datetime @@ -1175,7 +1267,9 @@ class DATE(_DateTimeMixin, sqltypes.Date): _storage_format = "%(year)04d-%(month)02d-%(day)02d" - def bind_processor(self, dialect): + def bind_processor( + self, dialect: Dialect + ) -> Optional[_BindProcessorType[Any]]: datetime_date = datetime.date format_ = self._storage_format @@ -1196,7 +1290,9 @@ def process(value): return process - def result_processor(self, dialect, coltype): + def result_processor( + self, dialect: Dialect, coltype: object + ) -> Optional[_ResultProcessorType[Any]]: if self._reg: return processors.str_to_datetime_processor_factory( self._reg, datetime.date @@ -1235,6 +1331,10 @@ class TIME(_DateTimeMixin, sqltypes.Time): regexp=re.compile("(\d+)-(\d+)-(\d+)-(?:-(\d+))?"), ) + :param truncate_microseconds: when ``True`` microseconds will be truncated + from the time. Can't be specified together with ``storage_format`` + or ``regexp``. + :param storage_format: format string which will be applied to the dict with keys hour, minute, second, and microsecond. @@ -1360,7 +1460,7 @@ def visit_now_func(self, fn, **kw): return "CURRENT_TIMESTAMP" def visit_localtimestamp_func(self, func, **kw): - return 'DATETIME(CURRENT_TIMESTAMP, "localtime")' + return "DATETIME(CURRENT_TIMESTAMP, 'localtime')" def visit_true(self, expr, **kw): return "1" @@ -1372,7 +1472,9 @@ def visit_char_length_func(self, fn, **kw): return "length%s" % self.function_argspec(fn) def visit_aggregate_strings_func(self, fn, **kw): - return "group_concat%s" % self.function_argspec(fn) + return super().visit_aggregate_strings_func( + fn, use_function_name="group_concat", **kw + ) def visit_cast(self, cast, **kwargs): if self.dialect.supports_cast: @@ -1441,7 +1543,16 @@ def visit_is_not_distinct_from_binary(self, binary, operator, **kw): self.process(binary.right), ) - def visit_json_getitem_op_binary(self, binary, operator, **kw): + def visit_json_getitem_op_binary( + self, binary, operator, _cast_applied=False, **kw + ): + if ( + not _cast_applied + and binary.type._type_affinity is not sqltypes.JSON + ): + kw["_cast_applied"] = True + return self.process(sql.cast(binary, binary.type), **kw) + if binary.type._type_affinity is sqltypes.JSON: expr = "JSON_QUOTE(JSON_EXTRACT(%s, %s))" else: @@ -1452,7 +1563,16 @@ def visit_json_getitem_op_binary(self, binary, operator, **kw): self.process(binary.right, **kw), ) - def visit_json_path_getitem_op_binary(self, binary, operator, **kw): + def visit_json_path_getitem_op_binary( + self, binary, operator, _cast_applied=False, **kw + ): + if ( + not _cast_applied + and binary.type._type_affinity is not sqltypes.JSON + ): + kw["_cast_applied"] = True + return self.process(sql.cast(binary, binary.type), **kw) + if binary.type._type_affinity is sqltypes.JSON: expr = "JSON_QUOTE(JSON_EXTRACT(%s, %s))" else: @@ -1491,12 +1611,16 @@ def _on_conflict_target(self, clause, **kw): for c in clause.inferred_target_elements ) if clause.inferred_target_whereclause is not None: - target_text += " WHERE %s" % self.process( - clause.inferred_target_whereclause, + whereclause_kw = dict(kw) + whereclause_kw.update( include_table=False, use_schema=False, literal_execute=True, ) + target_text += " WHERE %s" % self.process( + clause.inferred_target_whereclause, + **whereclause_kw, + ) else: target_text = "" @@ -1523,6 +1647,8 @@ def visit_on_conflict_do_update(self, on_conflict, **kw): insert_statement = self.stack[-1]["selectable"] cols = insert_statement.table.c + set_kw = dict(kw) + set_kw.update(use_schema=False) for c in cols: col_key = c.key @@ -1538,7 +1664,10 @@ def visit_on_conflict_do_update(self, on_conflict, **kw): and value.type._isnull ): value = value._with_binary_element_type(c.type) - value_text = self.process(value.self_group(), use_schema=False) + + value_text = self.process( + value.self_group(), is_upsert_set=True, **set_kw + ) key_text = self.preparer.quote(c.name) action_set_ops.append("%s = %s" % (key_text, value_text)) @@ -1557,18 +1686,21 @@ def visit_on_conflict_do_update(self, on_conflict, **kw): key_text = ( self.preparer.quote(k) if isinstance(k, str) - else self.process(k, use_schema=False) + else self.process(k, **set_kw) ) value_text = self.process( coercions.expect(roles.ExpressionElementRole, v), - use_schema=False, + is_upsert_set=True, + **set_kw, ) action_set_ops.append("%s = %s" % (key_text, value_text)) action_text = ", ".join(action_set_ops) if clause.update_whereclause is not None: + where_kw = dict(kw) + where_kw.update(include_table=True, use_schema=False) action_text += " WHERE %s" % self.process( - clause.update_whereclause, include_table=True, use_schema=False + clause.update_whereclause, **where_kw ) return "ON CONFLICT %s DO UPDATE SET %s" % (target_text, action_text) @@ -1589,9 +1721,13 @@ def get_column_specification(self, column, **kwargs): colspec = self.preparer.format_column(column) + " " + coltype default = self.get_column_default_string(column) if default is not None: - if isinstance(column.server_default.arg, ColumnElement): - default = "(" + default + ")" - colspec += " DEFAULT " + default + + if not re.match(r"""^\s*[\'\"\(]""", default) and re.match( + r".*\W.*", default + ): + colspec += f" DEFAULT ({default})" + else: + colspec += f" DEFAULT {default}" if not column.nullable: colspec += " NOT NULL" @@ -1766,6 +1902,17 @@ def post_create_table(self, table): else: return "" + def visit_create_view(self, create, **kw): + """Handle SQLite if_not_exists dialect option for CREATE VIEW.""" + # Get the if_not_exists dialect option from the CreateView object + if_not_exists = create.dialect_options["sqlite"].get( + "if_not_exists", False + ) + + # Pass if_not_exists through kw to the parent's _generate_table_select + kw["if_not_exists"] = if_not_exists + return super().visit_create_view(create, **kw) + class SQLiteTypeCompiler(compiler.GenericTypeCompiler): def visit_large_binary(self, type_, **kw): @@ -2012,40 +2159,21 @@ class SQLiteDialect(default.DefaultDialect): }, ), (sa_schema.Constraint, {"on_conflict": None}), + (sa_ddl.CreateView, {"if_not_exists": False}), ] _broken_fk_pragma_quotes = False _broken_dotted_colnames = False - @util.deprecated_params( - _json_serializer=( - "1.3.7", - "The _json_serializer argument to the SQLite dialect has " - "been renamed to the correct name of json_serializer. The old " - "argument name will be removed in a future release.", - ), - _json_deserializer=( - "1.3.7", - "The _json_deserializer argument to the SQLite dialect has " - "been renamed to the correct name of json_deserializer. The old " - "argument name will be removed in a future release.", - ), - ) def __init__( self, - native_datetime=False, - json_serializer=None, - json_deserializer=None, - _json_serializer=None, - _json_deserializer=None, - **kwargs, - ): + native_datetime: bool = False, + json_serializer: Callable[[_JSON_VALUE], str] | None = None, + json_deserializer: Callable[[str], _JSON_VALUE] | None = None, + **kwargs: Any, + ) -> None: default.DefaultDialect.__init__(self, **kwargs) - if _json_serializer: - json_serializer = _json_serializer - if _json_deserializer: - json_deserializer = _json_deserializer self._json_serializer = json_serializer self._json_deserializer = json_deserializer @@ -2107,7 +2235,9 @@ def __init__( def get_isolation_level_values(self, dbapi_connection): return list(self._isolation_lookup) - def set_isolation_level(self, dbapi_connection, level): + def set_isolation_level( + self, dbapi_connection: DBAPIConnection, level: IsolationLevel + ) -> None: isolation_level = self._isolation_lookup[level] cursor = dbapi_connection.cursor() @@ -2295,7 +2425,10 @@ def get_columns(self, connection, table_name, schema=None, **kw): ) # remove create table match = re.match( - r"create table .*?\((.*)\)$", + ( + r"create table .*?\((.*)\)" + r"(?:\s*,?\s*(?:WITHOUT\s+ROWID|STRICT))*$" + ), tablesql.strip(), re.DOTALL | re.IGNORECASE, ) @@ -2426,9 +2559,12 @@ def get_pk_constraint(self, connection, table_name, schema=None, **kw): constraint_name = None table_data = self._get_table_sql(connection, table_name, schema=schema) if table_data: - PK_PATTERN = r"CONSTRAINT (\w+) PRIMARY KEY" + PK_PATTERN = r'CONSTRAINT +(?:"(.+?)"|(\w+)) +PRIMARY KEY' result = re.search(PK_PATTERN, table_data, re.I) - constraint_name = result.group(1) if result else None + if result: + constraint_name = result.group(1) or result.group(2) + else: + constraint_name = None cols = self.get_columns(connection, table_name, schema, **kw) # consider only pk columns. This also avoids sorting the cached @@ -2528,7 +2664,7 @@ def parse_fks(): # so parsing the columns is really about matching it up to what # we already have. FK_PATTERN = ( - r"(?:CONSTRAINT (\w+) +)?" + r'(?:CONSTRAINT +(?:"(.+?)"|(\w+)) +)?' r"FOREIGN KEY *\( *(.+?) *\) +" r'REFERENCES +(?:(?:"(.+?)")|([a-z0-9_]+)) *\( *((?:(?:"[^"]+"|[a-z0-9_]+) *(?:, *)?)+)\) *' # noqa: E501 r"((?:ON (?:DELETE|UPDATE) " @@ -2538,6 +2674,7 @@ def parse_fks(): ) for match in re.finditer(FK_PATTERN, table_data, re.I): ( + constraint_quoted_name, constraint_name, constrained_columns, referred_quoted_name, @@ -2546,7 +2683,8 @@ def parse_fks(): onupdatedelete, deferrable, initially, - ) = match.group(1, 2, 3, 4, 5, 6, 7, 8) + ) = match.group(1, 2, 3, 4, 5, 6, 7, 8, 9) + constraint_name = constraint_quoted_name or constraint_name constrained_columns = list( self._find_cols_in_sig(constrained_columns) ) @@ -2641,14 +2779,17 @@ def get_unique_constraints( def parse_uqs(): if table_data is None: return - UNIQUE_PATTERN = r'(?:CONSTRAINT "?(.+?)"? +)?UNIQUE *\((.+?)\)' + UNIQUE_PATTERN = ( + r'(?:CONSTRAINT +(?:"(.+?)"|(\w+)) +)?UNIQUE *\((.+?)\)' + ) INLINE_UNIQUE_PATTERN = ( r'(?:(".+?")|(?:[\[`])?([a-z0-9_]+)(?:[\]`])?)[\t ]' r"+[a-z0-9_ ]+?[\t ]+UNIQUE" ) for match in re.finditer(UNIQUE_PATTERN, table_data, re.I): - name, cols = match.group(1, 2) + quoted_name, unquoted_name, cols = match.group(1, 2, 3) + name = quoted_name or unquoted_name yield name, list(self._find_cols_in_sig(cols)) # we need to match inlines as well, as we seek to differentiate @@ -2679,27 +2820,88 @@ def get_check_constraints(self, connection, table_name, schema=None, **kw): connection, table_name, schema=schema, **kw ) - # NOTE NOTE NOTE - # DO NOT CHANGE THIS REGULAR EXPRESSION. There is no known way - # to parse CHECK constraints that contain newlines themselves using - # regular expressions, and the approach here relies upon each - # individual - # CHECK constraint being on a single line by itself. This - # necessarily makes assumptions as to how the CREATE TABLE - # was emitted. A more comprehensive DDL parsing solution would be - # needed to improve upon the current situation. See #11840 for - # background - CHECK_PATTERN = r"(?:CONSTRAINT (.+) +)?CHECK *\( *(.+) *\),? *" - cks = [] + # Extract CHECK constraints by properly handling balanced parentheses + # and avoiding false matches when CHECK/CONSTRAINT appear in table + # names. See #12924 for context. + # + # SQLite supports 4 identifier quote styles (see + # sqlite.org/lang_keywords.html): + # - Double quotes "..." (standard SQL) + # - Brackets [...] (MS Access/SQL Server compatibility) + # - Backticks `...` (MySQL compatibility) + # - Single quotes '...' (SQLite extension) + # + # NOTE: there is not currently a way to parse CHECK constraints that + # contain newlines as the approach here relies upon each individual + # CHECK constraint being on a single line by itself. This necessarily + # makes assumptions as to how the CREATE TABLE was emitted. + CHECK_PATTERN = re.compile( + r""" + (? name + # Single quotes: 'name' -> name + # Brackets: [name] -> name + # Backticks: `name` -> name + constraint_name = re.sub( + r'^(["\'`])(.+)\1$|^\[(.+)\]$', + lambda m: m.group(2) or m.group(3), + constraint_name, + flags=re.DOTALL, + ) - if name: - name = re.sub(r'^"|"$', "", name) + # Find the matching closing parenthesis by counting balanced parens + # Must track string context to ignore parens inside string literals + start = match.end() # Position after 'CHECK (' + paren_count = 1 + in_single_quote = False + in_double_quote = False + + for pos, char in enumerate(table_data[start:], start): + # Track string literal context + if char == "'" and not in_double_quote: + in_single_quote = not in_single_quote + elif char == '"' and not in_single_quote: + in_double_quote = not in_double_quote + # Only count parens when not inside a string literal + elif not in_single_quote and not in_double_quote: + if char == "(": + paren_count += 1 + elif char == ")": + paren_count -= 1 + if paren_count == 0: + # Successfully found matching closing parenthesis + sqltext = table_data[start:pos].strip() + cks.append( + {"sqltext": sqltext, "name": constraint_name} + ) + break - cks.append({"sqltext": match.group(2), "name": name}) cks.sort(key=lambda d: d["name"] or "~") # sort None as last if cks: return cks diff --git a/lib/sqlalchemy/dialects/sqlite/dml.py b/lib/sqlalchemy/dialects/sqlite/dml.py index fc16f1eaa43..f877f0a12a5 100644 --- a/lib/sqlalchemy/dialects/sqlite/dml.py +++ b/lib/sqlalchemy/dialects/sqlite/dml.py @@ -1,5 +1,5 @@ # dialects/sqlite/dml.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/dialects/sqlite/json.py b/lib/sqlalchemy/dialects/sqlite/json.py index 02f4ea4c90f..ac705d661d5 100644 --- a/lib/sqlalchemy/dialects/sqlite/json.py +++ b/lib/sqlalchemy/dialects/sqlite/json.py @@ -1,15 +1,24 @@ # dialects/sqlite/json.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors +from __future__ import annotations + +from typing import Any +from typing import TYPE_CHECKING from ... import types as sqltypes +from ...sql.sqltypes import _T_JSON + +if TYPE_CHECKING: + from ...engine.interfaces import Dialect + from ...sql.type_api import _BindProcessorType + from ...sql.type_api import _LiteralProcessorType -class JSON(sqltypes.JSON): +class JSON(sqltypes.JSON[_T_JSON]): """SQLite JSON type. SQLite supports JSON as of version 3.9 through its JSON1_ extension. Note @@ -33,9 +42,6 @@ class JSON(sqltypes.JSON): always JSON string values. - .. versionadded:: 1.3 - - .. _JSON1: https://www.sqlite.org/json1.html """ @@ -45,13 +51,13 @@ class JSON(sqltypes.JSON): # these are not generalizable to all JSON implementations, remain separately # implemented for each dialect. class _FormatTypeMixin: - def _format_value(self, value): + def _format_value(self, value: Any) -> str: raise NotImplementedError() - def bind_processor(self, dialect): - super_proc = self.string_bind_processor(dialect) + def bind_processor(self, dialect: Dialect) -> _BindProcessorType[Any]: + super_proc = self.string_bind_processor(dialect) # type: ignore[attr-defined] # noqa: E501 - def process(value): + def process(value: Any) -> Any: value = self._format_value(value) if super_proc: value = super_proc(value) @@ -59,29 +65,31 @@ def process(value): return process - def literal_processor(self, dialect): - super_proc = self.string_literal_processor(dialect) + def literal_processor( + self, dialect: Dialect + ) -> _LiteralProcessorType[Any]: + super_proc = self.string_literal_processor(dialect) # type: ignore[attr-defined] # noqa: E501 - def process(value): + def process(value: Any) -> str: value = self._format_value(value) if super_proc: value = super_proc(value) - return value + return value # type: ignore[no-any-return] return process class JSONIndexType(_FormatTypeMixin, sqltypes.JSON.JSONIndexType): - def _format_value(self, value): + def _format_value(self, value: Any) -> str: if isinstance(value, int): - value = "$[%s]" % value + formatted_value = "$[%s]" % value else: - value = '$."%s"' % value - return value + formatted_value = '$."%s"' % value + return formatted_value class JSONPathType(_FormatTypeMixin, sqltypes.JSON.JSONPathType): - def _format_value(self, value): + def _format_value(self, value: Any) -> str: return "$%s" % ( "".join( [ diff --git a/lib/sqlalchemy/dialects/sqlite/provision.py b/lib/sqlalchemy/dialects/sqlite/provision.py index 97f882e7f28..fa63ab5ed97 100644 --- a/lib/sqlalchemy/dialects/sqlite/provision.py +++ b/lib/sqlalchemy/dialects/sqlite/provision.py @@ -1,5 +1,5 @@ # dialects/sqlite/provision.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -9,20 +9,22 @@ import os import re +from ... import event from ... import exc from ...engine import url as sa_url +from ...testing import config from ...testing.provision import create_db from ...testing.provision import drop_db from ...testing.provision import follower_url_from_main from ...testing.provision import generate_driver_url from ...testing.provision import log from ...testing.provision import post_configure_engine +from ...testing.provision import post_configure_testing_engine from ...testing.provision import run_reap_dbs from ...testing.provision import stop_test_class_outside_fixtures from ...testing.provision import temp_table_keyword_args from ...testing.provision import upsert - # TODO: I can't get this to build dynamically with pytest-xdist procs _drivernames = { "pysqlite", @@ -52,8 +54,6 @@ def _format_url(url, driver, ident): assert "test_schema" not in filename tokens = re.split(r"[_\.]", filename) - new_filename = f"{driver}" - for token in tokens: if token in _drivernames: if driver is None: @@ -141,6 +141,31 @@ def dispose(engine): os.remove(filename) +@post_configure_testing_engine.for_db("sqlite") +def _sqlite_post_configure_testing_engine(url, engine, options, scope): + + sqlite_savepoint = options.get("sqlite_savepoint", False) + sqlite_share_pool = options.get("sqlite_share_pool", False) + + if sqlite_savepoint and engine.name == "sqlite": + # apply SQLite savepoint workaround + @event.listens_for(engine, "connect") + def do_connect(dbapi_connection, connection_record): + dbapi_connection.isolation_level = None + + @event.listens_for(engine, "begin") + def do_begin(conn): + conn.exec_driver_sql("BEGIN") + + if sqlite_share_pool: + # SingletonThreadPool, StaticPool both support "transfer" + # so a new pool can share the same SQLite connection + # (single thread only) + if hasattr(engine.pool, "_transfer_from"): + options["use_reaper"] = False + engine.pool._transfer_from(config.db.pool) + + @create_db.for_db("sqlite") def _sqlite_create_db(cfg, eng, ident): pass @@ -181,7 +206,13 @@ def _reap_sqlite_dbs(url, idents): @upsert.for_db("sqlite") def _upsert( - cfg, table, returning, *, set_lambda=None, sort_by_parameter_order=False + cfg, + table, + returning, + *, + set_lambda=None, + sort_by_parameter_order=False, + index_elements=None, ): from sqlalchemy.dialects.sqlite import insert diff --git a/lib/sqlalchemy/dialects/sqlite/pysqlcipher.py b/lib/sqlalchemy/dialects/sqlite/pysqlcipher.py index 7a3dc1bae13..7f2c3d4b796 100644 --- a/lib/sqlalchemy/dialects/sqlite/pysqlcipher.py +++ b/lib/sqlalchemy/dialects/sqlite/pysqlcipher.py @@ -1,5 +1,5 @@ # dialects/sqlite/pysqlcipher.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -131,16 +131,20 @@ def on_connect_url(self, url): # pull the info we need from the URL early. Even though URL # is immutable, we don't want any in-place changes to the URL # to affect things - passphrase = url.password or "" - url_query = dict(url.query) + ip = self.identifier_preparer + passphrase = ip.quote_identifier(url.password or "") + query_pragmas = { + prag: ip.quote_identifier(url.query[prag]) + for prag in self.pragmas + if url.query.get(prag) is not None + } def on_connect(conn): cursor = conn.cursor() - cursor.execute('pragma key="%s"' % passphrase) - for prag in self.pragmas: - value = url_query.get(prag, None) - if value is not None: - cursor.execute('pragma %s="%s"' % (prag, value)) + cursor.execute(f"pragma key={passphrase}") + for prag, value in query_pragmas.items(): + cursor.execute(f"pragma {prag}={value}") + print(query_pragmas) cursor.close() if super_on_connect: diff --git a/lib/sqlalchemy/dialects/sqlite/pysqlite.py b/lib/sqlalchemy/dialects/sqlite/pysqlite.py index 73a74eb7108..366ca2e4600 100644 --- a/lib/sqlalchemy/dialects/sqlite/pysqlite.py +++ b/lib/sqlalchemy/dialects/sqlite/pysqlite.py @@ -1,10 +1,9 @@ # dialects/sqlite/pysqlite.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors r""" @@ -122,8 +121,6 @@ parameter which allows for a custom callable that creates a Python sqlite3 driver level connection directly. -.. versionadded:: 1.3.9 - .. seealso:: `Uniform Resource Identifiers `_ - in @@ -258,7 +255,7 @@ def regexp(a, b): It's been observed that the :class:`.NullPool` implementation incurs an extremely small performance overhead for repeated checkouts due to the lack of -connection re-use implemented by :class:`.QueuePool`. However, it still +connection reuse implemented by :class:`.QueuePool`. However, it still may be beneficial to use this class if the application is experiencing issues with files being locked. @@ -354,76 +351,10 @@ def process_result_value(self, value, dialect): Serializable isolation / Savepoints / Transactional DDL ------------------------------------------------------- -In the section :ref:`sqlite_concurrency`, we refer to the pysqlite -driver's assortment of issues that prevent several features of SQLite -from working correctly. The pysqlite DBAPI driver has several -long-standing bugs which impact the correctness of its transactional -behavior. In its default mode of operation, SQLite features such as -SERIALIZABLE isolation, transactional DDL, and SAVEPOINT support are -non-functional, and in order to use these features, workarounds must -be taken. - -The issue is essentially that the driver attempts to second-guess the user's -intent, failing to start transactions and sometimes ending them prematurely, in -an effort to minimize the SQLite databases's file locking behavior, even -though SQLite itself uses "shared" locks for read-only activities. - -SQLAlchemy chooses to not alter this behavior by default, as it is the -long-expected behavior of the pysqlite driver; if and when the pysqlite -driver attempts to repair these issues, that will be more of a driver towards -defaults for SQLAlchemy. - -The good news is that with a few events, we can implement transactional -support fully, by disabling pysqlite's feature entirely and emitting BEGIN -ourselves. This is achieved using two event listeners:: - - from sqlalchemy import create_engine, event - - engine = create_engine("sqlite:///myfile.db") - - - @event.listens_for(engine, "connect") - def do_connect(dbapi_connection, connection_record): - # disable pysqlite's emitting of the BEGIN statement entirely. - # also stops it from emitting COMMIT before any DDL. - dbapi_connection.isolation_level = None - - - @event.listens_for(engine, "begin") - def do_begin(conn): - # emit our own BEGIN - conn.exec_driver_sql("BEGIN") +A newly revised version of this important section is now available +at the top level of the SQLAlchemy SQLite documentation, in the section +:ref:`sqlite_transactions`. -.. warning:: When using the above recipe, it is advised to not use the - :paramref:`.Connection.execution_options.isolation_level` setting on - :class:`_engine.Connection` and :func:`_sa.create_engine` - with the SQLite driver, - as this function necessarily will also alter the ".isolation_level" setting. - - -Above, we intercept a new pysqlite connection and disable any transactional -integration. Then, at the point at which SQLAlchemy knows that transaction -scope is to begin, we emit ``"BEGIN"`` ourselves. - -When we take control of ``"BEGIN"``, we can also control directly SQLite's -locking modes, introduced at -`BEGIN TRANSACTION `_, -by adding the desired locking mode to our ``"BEGIN"``:: - - @event.listens_for(engine, "begin") - def do_begin(conn): - conn.exec_driver_sql("BEGIN EXCLUSIVE") - -.. seealso:: - - `BEGIN TRANSACTION `_ - - on the SQLite site - - `sqlite3 SELECT does not BEGIN a transaction `_ - - on the Python bug tracker - - `sqlite3 module breaks transactions and potentially corrupts data `_ - - on the Python bug tracker .. _pysqlite_udfs: @@ -459,10 +390,19 @@ def connect(conn, rec): print(conn.scalar(text("SELECT UDF()"))) """ # noqa +from __future__ import annotations import math import os import re +from typing import Any +from typing import Callable +from typing import cast +from typing import Optional +from typing import Pattern +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union from .base import DATE from .base import DATETIME @@ -471,16 +411,33 @@ def connect(conn, rec): from ... import pool from ... import types as sqltypes from ... import util +from ...util.typing import Self + +if TYPE_CHECKING: + from ...engine.interfaces import ConnectArgsType + from ...engine.interfaces import DBAPIConnection + from ...engine.interfaces import DBAPICursor + from ...engine.interfaces import DBAPIModule + from ...engine.interfaces import IsolationLevel + from ...engine.interfaces import VersionInfoType + from ...engine.url import URL + from ...pool.base import PoolProxiedConnection + from ...sql.type_api import _BindProcessorType + from ...sql.type_api import _ResultProcessorType class _SQLite_pysqliteTimeStamp(DATETIME): - def bind_processor(self, dialect): + def bind_processor( # type: ignore[override] + self, dialect: SQLiteDialect + ) -> Optional[_BindProcessorType[Any]]: if dialect.native_datetime: return None else: return DATETIME.bind_processor(self, dialect) - def result_processor(self, dialect, coltype): + def result_processor( # type: ignore[override] + self, dialect: SQLiteDialect, coltype: object + ) -> Optional[_ResultProcessorType[Any]]: if dialect.native_datetime: return None else: @@ -488,13 +445,17 @@ def result_processor(self, dialect, coltype): class _SQLite_pysqliteDate(DATE): - def bind_processor(self, dialect): + def bind_processor( # type: ignore[override] + self, dialect: SQLiteDialect + ) -> Optional[_BindProcessorType[Any]]: if dialect.native_datetime: return None else: return DATE.bind_processor(self, dialect) - def result_processor(self, dialect, coltype): + def result_processor( # type: ignore[override] + self, dialect: SQLiteDialect, coltype: object + ) -> Optional[_ResultProcessorType[Any]]: if dialect.native_datetime: return None else: @@ -519,13 +480,13 @@ class SQLiteDialect_pysqlite(SQLiteDialect): driver = "pysqlite" @classmethod - def import_dbapi(cls): + def import_dbapi(cls) -> DBAPIModule: from sqlite3 import dbapi2 as sqlite - return sqlite + return cast("DBAPIModule", sqlite) @classmethod - def _is_url_file_db(cls, url): + def _is_url_file_db(cls, url: URL) -> bool: if (url.database and url.database != ":memory:") and ( url.query.get("mode", None) != "memory" ): @@ -534,14 +495,14 @@ def _is_url_file_db(cls, url): return False @classmethod - def get_pool_class(cls, url): + def get_pool_class(cls, url: URL) -> type[pool.Pool]: if cls._is_url_file_db(url): return pool.QueuePool else: return pool.SingletonThreadPool - def _get_server_version_info(self, connection): - return self.dbapi.sqlite_version_info + def _get_server_version_info(self, connection: Any) -> VersionInfoType: + return self.dbapi.sqlite_version_info # type: ignore _isolation_lookup = SQLiteDialect._isolation_lookup.union( { @@ -549,15 +510,20 @@ def _get_server_version_info(self, connection): } ) - def set_isolation_level(self, dbapi_connection, level): + def set_isolation_level( + self, dbapi_connection: DBAPIConnection, level: IsolationLevel + ) -> None: if level == "AUTOCOMMIT": dbapi_connection.isolation_level = None else: dbapi_connection.isolation_level = "" return super().set_isolation_level(dbapi_connection, level) - def on_connect(self): - def regexp(a, b): + def detect_autocommit_setting(self, dbapi_conn: DBAPIConnection) -> bool: + return dbapi_conn.isolation_level is None + + def on_connect(self) -> Callable[[DBAPIConnection], None]: + def regexp(a: str, b: Optional[str]) -> Optional[bool]: if b is None: return None return re.search(a, b) is not None @@ -571,12 +537,12 @@ def regexp(a, b): else: create_func_kw = {} - def set_regexp(dbapi_connection): + def set_regexp(dbapi_connection: DBAPIConnection) -> None: dbapi_connection.create_function( "regexp", 2, regexp, **create_func_kw ) - def floor_func(dbapi_connection): + def floor_func(dbapi_connection: DBAPIConnection) -> None: # NOTE: floor is optionally present in sqlite 3.35+ , however # as it is normally non-present we deliver floor() unconditionally # for now. @@ -587,13 +553,13 @@ def floor_func(dbapi_connection): fns = [set_regexp, floor_func] - def connect(conn): + def connect(conn: DBAPIConnection) -> None: for fn in fns: fn(conn) return connect - def create_connect_args(self, url): + def create_connect_args(self, url: URL) -> ConnectArgsType: if url.username or url.password or url.host or url.port: raise exc.ArgumentError( "Invalid SQLite URL: %s\n" @@ -618,7 +584,7 @@ def create_connect_args(self, url): ("cached_statements", int), ] opts = url.query - pysqlite_opts = {} + pysqlite_opts: dict[str, Any] = {} for key, type_ in pysqlite_args: util.coerce_kw_type(opts, key, type_, dest=pysqlite_opts) @@ -635,7 +601,7 @@ def create_connect_args(self, url): # to adjust for that here. for key, type_ in pysqlite_args: uri_opts.pop(key, None) - filename = url.database + filename: str = url.database # type: ignore[assignment] if uri_opts: # sorting of keys is for unit test support filename += "?" + ( @@ -655,7 +621,13 @@ def create_connect_args(self, url): return ([filename], pysqlite_opts) - def is_disconnect(self, e, connection, cursor): + def is_disconnect( + self, + e: DBAPIModule.Error, + connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]], + cursor: Optional[DBAPICursor], + ) -> bool: + self.dbapi = cast("DBAPIModule", self.dbapi) return isinstance( e, self.dbapi.ProgrammingError ) and "Cannot operate on a closed database." in str(e) @@ -677,34 +649,36 @@ class _SQLiteDialect_pysqlite_numeric(SQLiteDialect_pysqlite): driver = "pysqlite_numeric" _first_bind = ":1" - _not_in_statement_regexp = None + _not_in_statement_regexp: Optional[Pattern[str]] = None - def __init__(self, *arg, **kw): + def __init__(self, *arg: Any, **kw: Any) -> None: kw.setdefault("paramstyle", "numeric") super().__init__(*arg, **kw) - def create_connect_args(self, url): + def create_connect_args(self, url: URL) -> ConnectArgsType: arg, opts = super().create_connect_args(url) opts["factory"] = self._fix_sqlite_issue_99953() return arg, opts - def _fix_sqlite_issue_99953(self): + def _fix_sqlite_issue_99953(self) -> Any: import sqlite3 first_bind = self._first_bind if self._not_in_statement_regexp: nis = self._not_in_statement_regexp - def _test_sql(sql): + def _test_sql(sql: str) -> None: m = nis.search(sql) assert not m, f"Found {nis.pattern!r} in {sql!r}" else: - def _test_sql(sql): + def _test_sql(sql: str) -> None: pass - def _numeric_param_as_dict(parameters): + def _numeric_param_as_dict( + parameters: Any, + ) -> Union[dict[str, Any], tuple[Any, ...]]: if parameters: assert isinstance(parameters, tuple) return { @@ -714,13 +688,13 @@ def _numeric_param_as_dict(parameters): return () class SQLiteFix99953Cursor(sqlite3.Cursor): - def execute(self, sql, parameters=()): + def execute(self, sql: str, parameters: Any = ()) -> Self: _test_sql(sql) if first_bind in sql: parameters = _numeric_param_as_dict(parameters) return super().execute(sql, parameters) - def executemany(self, sql, parameters): + def executemany(self, sql: str, parameters: Any) -> Self: _test_sql(sql) if first_bind in sql: parameters = [ @@ -729,18 +703,27 @@ def executemany(self, sql, parameters): return super().executemany(sql, parameters) class SQLiteFix99953Connection(sqlite3.Connection): - def cursor(self, factory=None): + _CursorT = TypeVar("_CursorT", bound=sqlite3.Cursor) + + def cursor( + self, + factory: Optional[ + Callable[[sqlite3.Connection], _CursorT] + ] = None, + ) -> _CursorT: if factory is None: - factory = SQLiteFix99953Cursor - return super().cursor(factory=factory) + factory = SQLiteFix99953Cursor # type: ignore[assignment] + return super().cursor(factory=factory) # type: ignore[return-value] # noqa[E501] - def execute(self, sql, parameters=()): + def execute( + self, sql: str, parameters: Any = () + ) -> sqlite3.Cursor: _test_sql(sql) if first_bind in sql: parameters = _numeric_param_as_dict(parameters) return super().execute(sql, parameters) - def executemany(self, sql, parameters): + def executemany(self, sql: str, parameters: Any) -> sqlite3.Cursor: _test_sql(sql) if first_bind in sql: parameters = [ @@ -766,6 +749,6 @@ class _SQLiteDialect_pysqlite_dollar(_SQLiteDialect_pysqlite_numeric): _first_bind = "$1" _not_in_statement_regexp = re.compile(r"[^\d]:\d+") - def __init__(self, *arg, **kw): + def __init__(self, *arg: Any, **kw: Any) -> None: kw.setdefault("paramstyle", "numeric_dollar") super().__init__(*arg, **kw) diff --git a/lib/sqlalchemy/engine/__init__.py b/lib/sqlalchemy/engine/__init__.py index f4205d89260..20a09189d64 100644 --- a/lib/sqlalchemy/engine/__init__.py +++ b/lib/sqlalchemy/engine/__init__.py @@ -1,5 +1,5 @@ # engine/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/engine/_processors_cy.py b/lib/sqlalchemy/engine/_processors_cy.py index 16a44841acc..62afcead1fc 100644 --- a/lib/sqlalchemy/engine/_processors_cy.py +++ b/lib/sqlalchemy/engine/_processors_cy.py @@ -1,10 +1,10 @@ # engine/_processors_cy.py -# Copyright (C) 2010-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2010-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: disable-error-code="misc" +# mypy: disable-error-code="misc, untyped-decorator" from __future__ import annotations from datetime import date as date_cls @@ -26,7 +26,7 @@ def _is_compiled() -> bool: """Utility function to indicate if this module is compiled or not.""" - return cython.compiled # type: ignore[no-any-return] + return cython.compiled # type: ignore[no-any-return,unused-ignore] # END GENERATED CYTHON IMPORT diff --git a/lib/sqlalchemy/engine/_result_cy.py b/lib/sqlalchemy/engine/_result_cy.py new file mode 100644 index 00000000000..f99e351074e --- /dev/null +++ b/lib/sqlalchemy/engine/_result_cy.py @@ -0,0 +1,633 @@ +# engine/_result_cy.py +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors +# +# +# This module is part of SQLAlchemy and is released under +# the MIT License: https://www.opensource.org/licenses/mit-license.php +# mypy: disable-error-code="misc,no-redef,type-arg,untyped-decorator" +from __future__ import annotations + +from collections.abc import Callable +from collections.abc import Iterator +from collections.abc import Sequence +from enum import Enum +import operator +from typing import Any +from typing import Generic +from typing import Literal +from typing import overload +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union + +from .row import Row +from .row import RowMapping +from .. import exc +from ..util import HasMemoized_ro_memoized_attribute +from ..util.typing import Self +from ..util.typing import TupleAny +from ..util.typing import Unpack + +if TYPE_CHECKING: + from .result import _ProcessorsType + from .result import Result + from .result import ResultMetaData + +# START GENERATED CYTHON IMPORT +# This section is automatically generated by the script tools/cython_imports.py +try: + # NOTE: the cython compiler needs this "import cython" in the file, it + # can't be only "from sqlalchemy.util import cython" with the fallback + # in that module + import cython +except ModuleNotFoundError: + from sqlalchemy.util import cython + + +def _is_compiled() -> bool: + """Utility function to indicate if this module is compiled or not.""" + return cython.compiled # type: ignore[no-any-return,unused-ignore] + + +# END GENERATED CYTHON IMPORT + +if cython.compiled: + from cython.cimports.cpython import Py_INCREF + from cython.cimports.cpython import PyList_New + from cython.cimports.cpython import PyList_SET_ITEM + from cython.cimports.cpython import PyTuple_New + from cython.cimports.cpython import PyTuple_SET_ITEM + + +_RowData = Row[Unpack[TupleAny]] | RowMapping | Any +"""A generic form of "row" that accommodates for the different kinds of +"rows" that different result objects return, including row, row mapping, and +scalar values""" +_R = TypeVar("_R", bound=_RowData) +_T = TypeVar("_T", bound=Any) + +_InterimRowType = Union[_R, TupleAny] +"""a catchall "anything" kind of return type that can be applied +across all the result types + +""" + +_UniqueFilterType = Callable[[Any], Any] +_UniqueFilterStateType = tuple[set[Any], _UniqueFilterType | None] + +_FLAG_SIMPLE = cython.declare(cython.char, 0) +_FLAG_SCALAR_TO_TUPLE = cython.declare(cython.char, 1) +_FLAG_TUPLE_FILTER = cython.declare(cython.char, 2) + + +# a symbol that indicates to internal Result methods that +# "no row is returned". We can't use None for those cases where a scalar +# filter is applied to rows. +class _NoRow(Enum): + _NO_ROW = 0 + + +_NO_ROW = _NoRow._NO_ROW + + +class BaseResultInternal(Generic[_R]): + __slots__ = () + + _real_result: Result[Unpack[TupleAny]] | None = None + _generate_rows: bool = True + _row_logging_fn: Callable[[Any], Any] | None + + _unique_filter_state: _UniqueFilterStateType | None = None + _post_creational_filter: Callable[[Any], Any] | None = None + + _metadata: ResultMetaData + + _source_supports_scalars: bool + _yield_per: int | None + + def _fetchiter_impl( + self, + ) -> Iterator[_InterimRowType[Row[Unpack[TupleAny]]]]: + raise NotImplementedError() + + def _fetchone_impl( + self, hard_close: bool = False + ) -> _InterimRowType[Row[Unpack[TupleAny]]] | None: + raise NotImplementedError() + + def _fetchmany_impl( + self, size: int | None = None + ) -> list[_InterimRowType[Row[Unpack[TupleAny]]]]: + raise NotImplementedError() + + def _fetchall_impl( + self, + ) -> list[_InterimRowType[Row[Unpack[TupleAny]]]]: + raise NotImplementedError() + + def _soft_close(self, hard: bool = False) -> None: + raise NotImplementedError() + + @HasMemoized_ro_memoized_attribute + def _row_getter( + self, + ) -> tuple[Callable[..., _R] | None, Callable[..., Sequence[_R]] | None]: + real_result = self if self._real_result is None else self._real_result + + metadata = self._metadata + tuple_filters = metadata._tuplefilter + flag: cython.char = _FLAG_SIMPLE + + if real_result._source_supports_scalars: + if not self._generate_rows: + return None, None + else: + flag = _FLAG_SCALAR_TO_TUPLE + elif tuple_filters is not None: + flag = _FLAG_TUPLE_FILTER + + processors: tuple + proc_valid: tuple + + if metadata._effective_processors is not None: + ep = metadata._effective_processors + if flag == _FLAG_TUPLE_FILTER: + ep = tuple_filters(ep) + + processors = tuple(ep) + proc_valid = tuple( + [i for i, p in enumerate(processors) if p is not None] + ) + else: + processors = () + proc_valid = () + + proc_size: cython.Py_ssize_t = len(processors) + log_row = real_result._row_logging_fn + has_log_row: cython.bint = log_row is not None + + key_to_index = metadata._key_to_index + _Row = Row + + if flag == _FLAG_SIMPLE and proc_size == 0 and not has_log_row: + # just build the rows + + def single_row_simple(input_row: Sequence[Any], /) -> Row: + return _Row(metadata, None, key_to_index, input_row) + + if cython.compiled: + + def many_rows_simple(rows: Sequence[Any], /) -> list[Any]: + size: cython.Py_hash_t = len(rows) + i: cython.Py_ssize_t + result: list = PyList_New(size) + for i in range(size): + row: object = _Row( + metadata, None, key_to_index, rows[i] + ) + Py_INCREF(row) + PyList_SET_ITEM(result, i, row) + return result + + else: + + def many_rows_simple(rows: Sequence[Any], /) -> list[Any]: + return [ + _Row(metadata, None, key_to_index, row) for row in rows + ] + + return single_row_simple, many_rows_simple # type: ignore[return-value] # noqa: E501 + + first_row: cython.bint = True + + def single_row(input_row: Sequence[Any], /) -> Row: + nonlocal first_row + + if flag == _FLAG_SCALAR_TO_TUPLE: + input_row = (input_row,) + elif flag == _FLAG_TUPLE_FILTER: + input_row = tuple_filters(input_row) + + if proc_size != 0: + if first_row: + first_row = False + assert len(input_row) == proc_size + input_row = _apply_processors( + processors, proc_size, proc_valid, input_row + ) + + row: Row = _Row(metadata, None, key_to_index, input_row) + if has_log_row: + row = log_row(row) + return row + + if cython.compiled: + + def many_rows(rows: Sequence[Any], /) -> list[Any]: + size: cython.Py_hash_t = len(rows) + i: cython.Py_ssize_t + result: list = PyList_New(size) + for i in range(size): + row: object = single_row(rows[i]) + Py_INCREF(row) + PyList_SET_ITEM(result, i, row) + return result + + else: + + def many_rows(rows: Sequence[Any], /) -> list[Any]: + return [single_row(row) for row in rows] + + return single_row, many_rows # type: ignore[return-value] + + @HasMemoized_ro_memoized_attribute + def _iterator_getter(self) -> Callable[[], Iterator[_R]]: + make_row = self._row_getter[0] + + post_creational_filter = self._post_creational_filter + + if self._unique_filter_state is not None: + uniques: set + uniques, strategy = self._unique_strategy + + def iterrows() -> Iterator[_R]: + for raw_row in self._fetchiter_impl(): + row = ( + make_row(raw_row) if make_row is not None else raw_row + ) + hashed = strategy(row) if strategy is not None else row + if hashed in uniques: + continue + uniques.add(hashed) + if post_creational_filter is not None: + row = post_creational_filter(row) + yield row + + else: + + def iterrows() -> Iterator[_R]: + for raw_row in self._fetchiter_impl(): + row = ( + make_row(raw_row) if make_row is not None else raw_row + ) + if post_creational_filter is not None: + row = post_creational_filter(row) + yield row + + return iterrows + + def _raw_all_rows(self) -> Sequence[_R]: + make_rows = self._row_getter[1] + assert make_rows is not None + return make_rows(self._fetchall_impl()) + + def _allrows(self) -> Sequence[_R]: + post_creational_filter = self._post_creational_filter + + make_rows = self._row_getter[1] + + rows = self._fetchall_impl() + made_rows: Sequence[_InterimRowType[_R]] + if make_rows is not None: + made_rows = make_rows(rows) + else: + made_rows = rows + + interim_rows: Sequence[_R] + + if self._unique_filter_state is not None: + uniques: set + uniques, strategy = self._unique_strategy + interim_rows = _apply_unique_strategy( + made_rows, [], uniques, strategy + ) + else: + interim_rows = made_rows # type: ignore + + if post_creational_filter is not None: + interim_rows = [ + post_creational_filter(row) for row in interim_rows + ] + return interim_rows + + @HasMemoized_ro_memoized_attribute + def _onerow_getter( + self, + ) -> Callable[[Self], Literal[_NoRow._NO_ROW] | _R]: + make_row = self._row_getter[0] + + post_creational_filter = self._post_creational_filter + + if self._unique_filter_state is not None: + uniques: set + uniques, strategy = self._unique_strategy + + def onerow(self: Self) -> Literal[_NoRow._NO_ROW] | _R: + while True: + row = self._fetchone_impl() + if row is None: + return _NO_ROW + else: + obj: _InterimRowType[Any] = ( + make_row(row) if make_row is not None else row + ) + hashed = strategy(obj) if strategy is not None else obj + if hashed in uniques: + continue + uniques.add(hashed) + if post_creational_filter is not None: + obj = post_creational_filter(obj) + return obj # type: ignore + + else: + + def onerow(self: Self) -> Literal[_NoRow._NO_ROW] | _R: + row = self._fetchone_impl() + if row is None: + return _NO_ROW + else: + interim_row: _InterimRowType[Any] = ( + make_row(row) if make_row is not None else row + ) + if post_creational_filter is not None: + interim_row = post_creational_filter(interim_row) + return interim_row # type: ignore + + return onerow + + @HasMemoized_ro_memoized_attribute + def _manyrow_getter(self) -> Callable[[Self, int | None], Sequence[_R]]: + make_rows = self._row_getter[1] + real_result = self if self._real_result is None else self._real_result + yield_per = real_result._yield_per + + post_creational_filter = self._post_creational_filter + + if self._unique_filter_state: + uniques: set + uniques, strategy = self._unique_strategy + + def manyrows(self: Self, num: int | None, /) -> Sequence[_R]: + made_rows: Sequence[Any] + collect: list[_R] = [] + + _manyrows = self._fetchmany_impl + + if num is None: + # if None is passed, we don't know the default + # manyrows number, DBAPI has this as cursor.arraysize + # different DBAPIs / fetch strategies may be different. + # do a fetch to find what the number is. if there are + # only fewer rows left, then it doesn't matter. + if yield_per: + num_required = num = yield_per + else: + rows = _manyrows() + num = len(rows) + made_rows = ( + rows if make_rows is None else make_rows(rows) + ) + _apply_unique_strategy( + made_rows, collect, uniques, strategy + ) + num_required = num - len(collect) + else: + num_required = num + + assert num is not None + + while num_required: + rows = _manyrows(num_required) + if not rows: + break + + made_rows = rows if make_rows is None else make_rows(rows) + _apply_unique_strategy( + made_rows, collect, uniques, strategy + ) + num_required = num - len(collect) + + if post_creational_filter is not None: + collect = [post_creational_filter(row) for row in collect] + return collect + + else: + + def manyrows(self: Self, num: int | None, /) -> Sequence[_R]: + if num is None: + num = yield_per + + rows: Sequence = self._fetchmany_impl(num) + if make_rows is not None: + rows = make_rows(rows) + if post_creational_filter is not None: + rows = [post_creational_filter(row) for row in rows] + return rows + + return manyrows + + @overload + def _only_one_row( + self: BaseResultInternal[Row[_T, Unpack[TupleAny]]], + raise_for_second_row: bool, + raise_for_none: bool, + scalar: Literal[True], + ) -> _T: ... + + @overload + def _only_one_row( + self, + raise_for_second_row: bool, + raise_for_none: Literal[True], + scalar: bool, + ) -> _R: ... + + @overload + def _only_one_row( + self, + raise_for_second_row: bool, + raise_for_none: bool, + scalar: bool, + ) -> _R | None: ... + + def _only_one_row( + self, + raise_for_second_row: bool, + raise_for_none: bool, + scalar: bool, + ) -> _R | None: + onerow = self._fetchone_impl + + row = onerow(hard_close=True) + if row is None: + if raise_for_none: + raise exc.NoResultFound( + "No row was found when one was required" + ) + else: + return None + + if scalar and self._source_supports_scalars: + self._generate_rows = False + make_row = None + else: + make_row = self._row_getter[0] + + try: + row = make_row(row) if make_row is not None else row # type: ignore[assignment] # noqa: E501 + except: + self._soft_close(hard=True) + raise + + if raise_for_second_row: + if self._unique_filter_state: + # for no second row but uniqueness, need to essentially + # consume the entire result :( + strategy = self._unique_strategy[1] + + existing_row_hash = ( + strategy(row) if strategy is not None else row + ) + + while True: + next_row: Any = onerow(hard_close=True) + if next_row is None: + next_row = _NO_ROW + break + + try: + next_row = ( + make_row(next_row) + if make_row is not None + else next_row + ) + + if strategy is not None: + # assert next_row is not _NO_ROW + if existing_row_hash == strategy(next_row): + continue + elif row == next_row: + continue + # here, we have a row and it's different + break + except: + self._soft_close(hard=True) + raise + else: + next_row = onerow(hard_close=True) + if next_row is None: + next_row = _NO_ROW + + if next_row is not _NO_ROW: + self._soft_close(hard=True) + raise exc.MultipleResultsFound( + "Multiple rows were found when exactly one was required" + if raise_for_none + else "Multiple rows were found when one or none " + "was required" + ) + else: + next_row = _NO_ROW + # if we checked for second row then that would have + # closed us :) + self._soft_close(hard=True) + + if not scalar: + post_creational_filter = self._post_creational_filter + if post_creational_filter is not None: + row = post_creational_filter(row) + + if scalar and make_row is not None: + return row[0] # type: ignore + else: + return row # type: ignore + + def _iter_impl(self) -> Iterator[_R]: + return self._iterator_getter() + + def _next_impl(self) -> _R: + row = self._onerow_getter(self) + if row is _NO_ROW: + raise StopIteration() + else: + return row + + @HasMemoized_ro_memoized_attribute + def _unique_strategy(self) -> _UniqueFilterStateType: + assert self._unique_filter_state is not None + uniques, strategy = self._unique_filter_state + + if strategy is None and self._metadata._unique_filters is not None: + real_result = ( + self if self._real_result is None else self._real_result + ) + if ( + real_result._source_supports_scalars + and not self._generate_rows + ): + strategy = self._metadata._unique_filters[0] + else: + filters = self._metadata._unique_filters + if self._metadata._tuplefilter is not None: + filters = self._metadata._tuplefilter(filters) + + strategy = operator.methodcaller("_filter_on_values", filters) + return uniques, strategy + + +if cython.compiled: + + @cython.inline + @cython.cfunc + @cython.wraparound(False) + @cython.boundscheck(False) + def _apply_processors( + proc: tuple, + proc_size: cython.Py_ssize_t, + proc_valid: object, # used only by python impl + data: Sequence, + ) -> tuple[Any, ...]: + res: tuple = PyTuple_New(proc_size) + i: cython.Py_ssize_t + for i in range(proc_size): + p = proc[i] + if p is not None: + value = p(data[i]) + else: + value = data[i] + Py_INCREF(value) + PyTuple_SET_ITEM(res, i, value) + return res + +else: + + def _apply_processors( + proc: _ProcessorsType, + proc_size: int, # used only by cython impl + proc_valid: tuple[int, ...], + data: Sequence[Any], + ) -> tuple[Any, ...]: + res = list(data) + for i in proc_valid: + res[i] = proc[i](res[i]) + return tuple(res) + + +@cython.inline +@cython.cfunc +def _apply_unique_strategy( + rows: Sequence[Any], + destination: list[Any], + uniques: set[Any], + strategy: Callable[[Any], Any] | None, +) -> list[Any]: + i: cython.Py_ssize_t + has_strategy: cython.bint = strategy is not None + for i in range(len(rows)): + row = rows[i] + hashed = strategy(row) if has_strategy else row + if hashed in uniques: + continue + uniques.add(hashed) + destination.append(row) + return destination diff --git a/lib/sqlalchemy/engine/_row_cy.py b/lib/sqlalchemy/engine/_row_cy.py index 4319e05f0bb..60b40500258 100644 --- a/lib/sqlalchemy/engine/_row_cy.py +++ b/lib/sqlalchemy/engine/_row_cy.py @@ -1,16 +1,19 @@ # engine/_row_cy.py -# Copyright (C) 2010-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2010-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: disable-error-code="misc" +# mypy: disable-error-code="misc,no-redef,valid-type,no-untyped-call" +# mypy: disable-error-code="index,no-any-return,arg-type,assignment" +# mypy: disable-error-code="untyped-decorator" from __future__ import annotations from typing import Any from typing import Dict from typing import Iterator from typing import List +from typing import NoReturn from typing import Optional from typing import Sequence from typing import Tuple @@ -35,7 +38,7 @@ def _is_compiled() -> bool: """Utility function to indicate if this module is compiled or not.""" - return cython.compiled # type: ignore[no-any-return] + return cython.compiled # type: ignore[no-any-return,unused-ignore] # END GENERATED CYTHON IMPORT @@ -60,13 +63,15 @@ def __init__( data: Sequence[Any], ) -> None: """Row objects are constructed by CursorResult objects.""" - - data_tuple: Tuple[Any, ...] = ( - _apply_processors(processors, data) - if processors is not None - else tuple(data) + self._set_attrs( + parent, + key_to_index, + ( + _apply_processors(processors, data) + if processors is not None + else data if isinstance(data, tuple) else tuple(data) + ), ) - self._set_attrs(parent, key_to_index, data_tuple) @cython.cfunc @cython.inline @@ -112,41 +117,106 @@ def __len__(self) -> int: def __hash__(self) -> int: return hash(self._data) - def __getitem__(self, key: Any) -> Any: - return self._data[key] + if not TYPE_CHECKING: + + def __getitem__(self, key: Any) -> Any: + return self._data[key] def _get_by_key_impl_mapping(self, key: _KeyType) -> Any: return self._get_by_key_impl(key, False) @cython.cfunc + @cython.inline def _get_by_key_impl(self, key: _KeyType, attr_err: cython.bint) -> object: - index: Optional[int] = self._key_to_index.get(key) + # NOTE: don't type index since there is no advantage in making cython + # do a type check + index = self._key_to_index.get(key) if index is not None: return self._data[index] self._parent._key_not_found(key, attr_err) + if cython.compiled: + + @cython.annotation_typing(False) + def __getattribute__(self, name: str) -> Any: + # this optimizes getattr access on cython, that's otherwise + # quite slow compared with python. The assumption is that + # most columns will not start with _. If they do they will + # fallback on __getattr__ in any case. + if name != "" and name[0] != "_": + # inline of _get_by_key_impl. Attribute on the class + # take precedence over column names. + index = self._key_to_index.get(name) + if index is not None and not hasattr(type(self), name): + return self._data[index] + + return object.__getattribute__(self, name) + @cython.annotation_typing(False) def __getattr__(self, name: str) -> Any: return self._get_by_key_impl(name, True) + def __setattr__(self, name: str, value: Any) -> NoReturn: + raise AttributeError("can't set attribute") + + def __delattr__(self, name: str) -> NoReturn: + raise AttributeError("can't delete attribute") + def _to_tuple_instance(self) -> Tuple[Any, ...]: return self._data + def __contains__(self, key: Any) -> cython.bint: + return key in self._data + + +if cython.compiled: -@cython.inline -@cython.cfunc -def _apply_processors( - proc: _ProcessorsType, data: Sequence[Any] -) -> Tuple[Any, ...]: - res: List[Any] = list(data) - proc_size: cython.Py_ssize_t = len(proc) - # TODO: would be nice to do this only on the fist row - assert len(res) == proc_size - for i in range(proc_size): - p = proc[i] - if p is not None: - res[i] = p(res[i]) - return tuple(res) + from cython.cimports.cpython import PyTuple_New + from cython.cimports.cpython import Py_INCREF + from cython.cimports.cpython import PyTuple_SET_ITEM + + @cython.inline + @cython.cfunc + @cython.wraparound(False) + @cython.boundscheck(False) + @cython.locals( + res=tuple, + proc_size=cython.Py_ssize_t, + i=cython.Py_ssize_t, + p=object, + value=object, + ) + def _apply_processors( + proc: Sequence[Any], data: Sequence[Any] + ) -> Tuple[Any, ...]: + proc_size = len(proc) + # TODO: would be nice to do this only on the fist row + assert len(data) == proc_size + res = PyTuple_New(proc_size) + for i in range(proc_size): + p = proc[i] + if p is not None: + value = p(data[i]) + else: + value = data[i] + Py_INCREF(value) + PyTuple_SET_ITEM(res, i, value) + return res + +else: + + def _apply_processors( + proc: _ProcessorsType, data: Sequence[Any] + ) -> Tuple[Any, ...]: + res: List[Any] = list(data) + proc_size = len(proc) + # TODO: would be nice to do this only on the fist row + assert len(res) == proc_size + for i in range(proc_size): + p = proc[i] + if p is not None: + res[i] = p(res[i]) + return tuple(res) # This reconstructor is necessary so that pickles with the Cy extension or diff --git a/lib/sqlalchemy/engine/_util_cy.py b/lib/sqlalchemy/engine/_util_cy.py index 218fcd2b7b8..118b4d7865c 100644 --- a/lib/sqlalchemy/engine/_util_cy.py +++ b/lib/sqlalchemy/engine/_util_cy.py @@ -1,10 +1,10 @@ # engine/_util_cy.py -# Copyright (C) 2010-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2010-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: disable-error-code="misc, type-arg" +# mypy: disable-error-code="misc, type-arg, untyped-decorator" from __future__ import annotations from collections.abc import Mapping @@ -37,7 +37,7 @@ def _is_compiled() -> bool: """Utility function to indicate if this module is compiled or not.""" - return cython.compiled # type: ignore[no-any-return] + return cython.compiled # type: ignore[no-any-return,unused-ignore] # END GENERATED CYTHON IMPORT @@ -57,7 +57,17 @@ def _is_mapping_or_tuple(value: object, /) -> cython.bint: ) -# _is_mapping_or_tuple could be inlined if pure python perf is a problem +@cython.inline +@cython.cfunc +def _is_mapping(value: object, /) -> cython.bint: + return ( + isinstance(value, dict) + or isinstance(value, Mapping) + # only do immutabledict or abc.__instancecheck__ for Mapping after + # we've checked for plain dictionaries and would otherwise raise + ) + + def _distill_params_20( params: Optional[_CoreAnyExecuteParams], ) -> _CoreMultiExecuteParams: @@ -73,19 +83,18 @@ def _distill_params_20( "future SQLAlchemy release", "2.1", ) - elif not _is_mapping_or_tuple(params[0]): + elif not _is_mapping(params[0]): raise exc.ArgumentError( - "List argument must consist only of tuples or dictionaries" + "List argument must consist only of dictionaries" ) return params - elif isinstance(params, dict) or isinstance(params, Mapping): - # only do immutabledict or abc.__instancecheck__ for Mapping after - # we've checked for plain dictionaries and would otherwise raise - return [params] + elif _is_mapping(params): + return [params] # type: ignore[list-item] else: raise exc.ArgumentError("mapping or list expected for parameters") +# _is_mapping_or_tuple could be inlined if pure python perf is a problem def _distill_raw_params( params: Optional[_DBAPIAnyExecuteParams], ) -> _DBAPIMultiExecuteParams: diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index fbbbb2cff01..7f8af56a8c4 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -1,12 +1,10 @@ # engine/base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -"""Defines :class:`_engine.Connection` and :class:`_engine.Engine`. - -""" +"""Defines :class:`_engine.Connection` and :class:`_engine.Engine`.""" from __future__ import annotations import contextlib @@ -43,6 +41,7 @@ from .. import util from ..sql import compiler from ..sql import util as sql_util +from ..util.typing import Never from ..util.typing import TupleAny from ..util.typing import TypeVarTuple from ..util.typing import Unpack @@ -73,12 +72,11 @@ from ..sql._typing import _InfoType from ..sql.compiler import Compiled from ..sql.ddl import ExecutableDDLElement - from ..sql.ddl import SchemaDropper - from ..sql.ddl import SchemaGenerator + from ..sql.ddl import InvokeDDLBase from ..sql.functions import FunctionElement from ..sql.schema import DefaultGenerator from ..sql.schema import HasSchemaAttr - from ..sql.schema import SchemaItem + from ..sql.schema import SchemaVisitable from ..sql.selectable import TypedReturnsRows @@ -537,8 +535,6 @@ def execution_options(self, **opt: Any) -> Connection: def get_execution_options(self) -> _ExecuteOptions: """Get the non-SQL options which will take effect during execution. - .. versionadded:: 1.3 - .. seealso:: :meth:`_engine.Connection.execution_options` @@ -971,7 +967,8 @@ def begin_twophase(self, xid: Optional[Any] = None) -> TwoPhaseTransaction: :meth:`~.TwoPhaseTransaction.prepare` method. :param xid: the two phase transaction id. If not supplied, a - random id will be generated. + random id will be generated. The accepted type and value depends on + the driver in use. .. seealso:: @@ -1132,10 +1129,16 @@ def _rollback_impl(self) -> None: if self._still_open_and_dbapi_connection_is_valid: if self._echo: if self._is_autocommit_isolation(): - self._log_info( - "ROLLBACK using DBAPI connection.rollback(), " - "DBAPI should ignore due to autocommit mode" - ) + if self.dialect.skip_autocommit_rollback: + self._log_info( + "ROLLBACK will be skipped by " + "skip_autocommit_rollback" + ) + else: + self._log_info( + "ROLLBACK using DBAPI connection.rollback(); " + "set skip_autocommit_rollback to prevent fully" + ) else: self._log_info("ROLLBACK") try: @@ -1151,7 +1154,7 @@ def _commit_impl(self) -> None: if self._is_autocommit_isolation(): self._log_info( "COMMIT using DBAPI connection.commit(), " - "DBAPI should ignore due to autocommit mode" + "has no effect due to autocommit mode" ) else: self._log_info("COMMIT") @@ -1278,6 +1281,17 @@ def close(self) -> None: self._dbapi_connection = None self.__can_reconnect = False + # special case to handle mypy issue: + # https://github.com/python/mypy/issues/20651 + @overload + def scalar( + self, + statement: TypedReturnsRows[Never], + parameters: Optional[_CoreSingleExecuteParams] = None, + *, + execution_options: Optional[CoreExecuteOptionsParameter] = None, + ) -> Optional[Any]: ... + @overload def scalar( self, @@ -1640,13 +1654,15 @@ def _execute_clauseelement( "compiled_cache", self.engine._compiled_cache ) - compiled_sql, extracted_params, cache_hit = elem._compile_w_cache( - dialect=dialect, - compiled_cache=compiled_cache, - column_keys=keys, - for_executemany=for_executemany, - schema_translate_map=schema_translate_map, - linting=self.dialect.compiler_linting | compiler.WARN_LINTING, + compiled_sql, extracted_params, param_dict, cache_hit = ( + elem._compile_w_cache( + dialect=dialect, + compiled_cache=compiled_cache, + column_keys=keys, + for_executemany=for_executemany, + schema_translate_map=schema_translate_map, + linting=self.dialect.compiler_linting | compiler.WARN_LINTING, + ) ) ret = self._execute_context( dialect, @@ -1659,6 +1675,7 @@ def _execute_clauseelement( elem, extracted_params, cache_hit=cache_hit, + param_dict=param_dict, ) if has_events: self.dispatch.after_execute( @@ -1671,56 +1688,6 @@ def _execute_clauseelement( ) return ret - def _execute_compiled( - self, - compiled: Compiled, - distilled_parameters: _CoreMultiExecuteParams, - execution_options: CoreExecuteOptionsParameter = _EMPTY_EXECUTION_OPTS, - ) -> CursorResult[Unpack[TupleAny]]: - """Execute a sql.Compiled object. - - TODO: why do we have this? likely deprecate or remove - - """ - - exec_opts = compiled.execution_options.merge_with( - self._execution_options, execution_options - ) - - if self._has_events or self.engine._has_events: - ( - compiled, - distilled_parameters, - event_multiparams, - event_params, - ) = self._invoke_before_exec_event( - compiled, distilled_parameters, exec_opts - ) - - dialect = self.dialect - - ret = self._execute_context( - dialect, - dialect.execution_ctx_cls._init_compiled, - compiled, - distilled_parameters, - exec_opts, - compiled, - distilled_parameters, - None, - None, - ) - if self._has_events or self.engine._has_events: - self.dispatch.after_execute( - self, - compiled, - event_multiparams, - event_params, - exec_opts, - ret, - ) - return ret - def exec_driver_sql( self, statement: str, @@ -2129,7 +2096,6 @@ def _exec_insertmany_context( sub_params, context, ) - except BaseException as e: self._handle_dbapi_exception( e, @@ -2144,8 +2110,8 @@ def _exec_insertmany_context( self.dispatch.after_cursor_execute( self, cursor, - str_statement, - effective_parameters, + sub_stmt, + sub_params, context, context.executemany, ) @@ -2439,9 +2405,7 @@ def _handle_dbapi_exception_noconnection( break if sqlalchemy_exception and is_disconnect != ctx.is_disconnect: - sqlalchemy_exception.connection_invalidated = is_disconnect = ( - ctx.is_disconnect - ) + sqlalchemy_exception.connection_invalidated = ctx.is_disconnect if newraise: raise newraise.with_traceback(exc_info[2]) from e @@ -2454,8 +2418,8 @@ def _handle_dbapi_exception_noconnection( def _run_ddl_visitor( self, - visitorcallable: Type[Union[SchemaGenerator, SchemaDropper]], - element: SchemaItem, + visitorcallable: Type[InvokeDDLBase], + element: SchemaVisitable, **kwargs: Any, ) -> None: """run a DDL visitor. @@ -2464,7 +2428,9 @@ def _run_ddl_visitor( options given to the visitor so that "checkfirst" is skipped. """ - visitorcallable(self.dialect, self, **kwargs).traverse_single(element) + visitorcallable( + dialect=self.dialect, connection=self, **kwargs + ).traverse_single(element) class ExceptionContextImpl(ExceptionContext): @@ -3138,8 +3104,6 @@ def _switch_shard(conn, cursor, stmt, params, context, executemany): def get_execution_options(self) -> _ExecuteOptions: """Get the non-SQL options which will take effect during execution. - .. versionadded: 1.3 - .. seealso:: :meth:`_engine.Engine.execution_options` @@ -3180,6 +3144,12 @@ def dispose(self, close: bool = True) -> None: connections. The latter strategy is more appropriate for an initializer in a forked Python process. + Event listeners associated with the old pool via :class:`.PoolEvents` + are **transferred to the new pool**; this is to support the pattern + by which :class:`.PoolEvents` are set up in terms of the owning + :class:`.Engine` without the need to refer to the :class:`.Pool` + directly. + :param close: if left at its default of ``True``, has the effect of fully closing all **currently checked in** database connections. Connections that are still checked out @@ -3205,6 +3175,8 @@ def dispose(self, close: bool = True) -> None: :ref:`pooling_multiprocessing` + :meth:`.ConnectionEvents.engine_disposed` + """ if close: self.pool.dispose() @@ -3252,8 +3224,8 @@ def begin(self) -> Iterator[Connection]: def _run_ddl_visitor( self, - visitorcallable: Type[Union[SchemaGenerator, SchemaDropper]], - element: SchemaItem, + visitorcallable: Type[InvokeDDLBase], + element: SchemaVisitable, **kwargs: Any, ) -> None: with self.begin() as conn: diff --git a/lib/sqlalchemy/engine/characteristics.py b/lib/sqlalchemy/engine/characteristics.py index 322c28b5aa7..ef7d65fec83 100644 --- a/lib/sqlalchemy/engine/characteristics.py +++ b/lib/sqlalchemy/engine/characteristics.py @@ -1,5 +1,5 @@ # engine/characteristics.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/engine/create.py b/lib/sqlalchemy/engine/create.py index 88690785d7b..47a7a510d7e 100644 --- a/lib/sqlalchemy/engine/create.py +++ b/lib/sqlalchemy/engine/create.py @@ -1,5 +1,5 @@ # engine/create.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -32,6 +32,8 @@ from ..util import immutabledict if typing.TYPE_CHECKING: + from typing import Literal + from .base import Engine from .interfaces import _ExecuteOptions from .interfaces import _ParamStyle @@ -42,7 +44,6 @@ from ..pool import _CreatorWRecFnType from ..pool import _ResetStyleArgType from ..pool import Pool - from ..util.typing import Literal @overload @@ -262,8 +263,6 @@ def create_engine(url: Union[str, _url.URL], **kwargs: Any) -> Engine: will not be displayed in INFO logging nor will they be formatted into the string representation of :class:`.StatementError` objects. - .. versionadded:: 1.3.8 - .. seealso:: :ref:`dbengine_logging` - further detail on how to configure @@ -323,19 +322,13 @@ def create_engine(url: Union[str, _url.URL], **kwargs: Any) -> Engine: :param json_deserializer: for dialects that support the :class:`_types.JSON` datatype, this is a Python callable that will convert a JSON string - to a Python object. By default, the Python ``json.loads`` function is - used. - - .. versionchanged:: 1.3.7 The SQLite dialect renamed this from - ``_json_deserializer``. + to a Python object. By default, either the driver's built-in + capabilities are used, or if none are available, the Python + ``json.loads`` function is used. :param json_serializer: for dialects that support the :class:`_types.JSON` - datatype, this is a Python callable that will render a given object - as JSON. By default, the Python ``json.dumps`` function is used. - - .. versionchanged:: 1.3.7 The SQLite dialect renamed this from - ``_json_serializer``. - + datatype, this is a Python callable that will render a given object as + JSON. By default, the Python ``json.dumps`` function is used. :param label_length=None: optional integer value which limits the size of dynamically generated column labels to that many @@ -373,8 +366,6 @@ def create_engine(url: Union[str, _url.URL], **kwargs: Any) -> Engine: SQLAlchemy's dialect has not been adjusted, the value may be passed here. - .. versionadded:: 1.3.9 - .. seealso:: :paramref:`_sa.create_engine.label_length` @@ -432,8 +423,6 @@ def create_engine(url: Union[str, _url.URL], **kwargs: Any) -> Engine: "pre-ping" feature that tests connections for liveness upon each checkout. - .. versionadded:: 1.2 - .. seealso:: :ref:`pool_disconnects_pessimistic` @@ -468,6 +457,9 @@ def create_engine(url: Union[str, _url.URL], **kwargs: Any) -> Engine: :ref:`pool_reset_on_return` + :ref:`dbapi_autocommit_skip_rollback` - a more modern approach + to using connections with no transactional instructions + :param pool_timeout=30: number of seconds to wait before giving up on getting a connection from the pool. This is only used with :class:`~sqlalchemy.pool.QueuePool`. This can be a float but is @@ -483,8 +475,6 @@ def create_engine(url: Union[str, _url.URL], **kwargs: Any) -> Engine: use. When planning for server-side timeouts, ensure that a recycle or pre-ping strategy is in use to gracefully handle stale connections. - .. versionadded:: 1.3 - .. seealso:: :ref:`pool_use_lifo` @@ -494,8 +484,6 @@ def create_engine(url: Union[str, _url.URL], **kwargs: Any) -> Engine: :param plugins: string list of plugin names to load. See :class:`.CreateEnginePlugin` for background. - .. versionadded:: 1.2.3 - :param query_cache_size: size of the cache used to cache the SQL string form of queries. Set to zero to disable caching. @@ -524,6 +512,18 @@ def create_engine(url: Union[str, _url.URL], **kwargs: Any) -> Engine: .. versionadded:: 1.4 + :param skip_autocommit_rollback: When True, the dialect will + unconditionally skip all calls to the DBAPI ``connection.rollback()`` + method if the DBAPI connection is confirmed to be in "autocommit" mode. + The availability of this feature is dialect specific; if not available, + a ``NotImplementedError`` is raised by the dialect when rollback occurs. + + .. seealso:: + + :ref:`dbapi_autocommit_skip_rollback` + + .. versionadded:: 2.0.43 + :param use_insertmanyvalues: True by default, use the "insertmanyvalues" execution style for INSERT..RETURNING statements by default. @@ -615,9 +615,8 @@ def pop_kwarg(key: str, default: Optional[Any] = None) -> Any: dialect = dialect_cls(**dialect_args) # assemble connection arguments - (cargs_tup, cparams) = dialect.create_connect_args(u) - cparams.update(pop_kwarg("connect_args", {})) - cargs = list(cargs_tup) # allow mutability + (cargs_tup, _cparams) = dialect.create_connect_args(u) + cparams = util.immutabledict(_cparams).union(pop_kwarg("connect_args", {})) # look for existing pool or create pool = pop_kwarg("pool", None) @@ -627,15 +626,23 @@ def connect( connection_record: Optional[ConnectionPoolEntry] = None, ) -> DBAPIConnection: if dialect._has_events: + mutable_cargs = list(cargs_tup) + mutable_cparams = dict(cparams) for fn in dialect.dispatch.do_connect: connection = cast( DBAPIConnection, - fn(dialect, connection_record, cargs, cparams), + fn( + dialect, + connection_record, + mutable_cargs, + mutable_cparams, + ), ) if connection is not None: return connection - - return dialect.connect(*cargs, **cparams) + return dialect.connect(*mutable_cargs, **mutable_cparams) + else: + return dialect.connect(*cargs_tup, **cparams) creator = pop_kwarg("creator", connect) diff --git a/lib/sqlalchemy/engine/cursor.py b/lib/sqlalchemy/engine/cursor.py index 56d7ee75885..ef264ec559a 100644 --- a/lib/sqlalchemy/engine/cursor.py +++ b/lib/sqlalchemy/engine/cursor.py @@ -1,10 +1,9 @@ # engine/cursor.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: allow-untyped-defs, allow-untyped-calls """Define cursor-specific result set constructs including :class:`.CursorResult`.""" @@ -13,15 +12,18 @@ from __future__ import annotations import collections -import functools import operator import typing from typing import Any from typing import cast from typing import ClassVar +from typing import Deque from typing import Dict +from typing import Final +from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import Mapping from typing import NoReturn from typing import Optional @@ -49,7 +51,6 @@ from ..sql.compiler import RM_RENDERED_NAME from ..sql.compiler import RM_TYPE from ..sql.type_api import TypeEngine -from ..util.typing import Literal from ..util.typing import Self from ..util.typing import TupleAny from ..util.typing import TypeVarTuple @@ -60,7 +61,10 @@ from .base import Connection from .default import DefaultExecutionContext from .interfaces import _DBAPICursorDescription + from .interfaces import _MutableCoreSingleExecuteParams + from .interfaces import CoreExecuteOptionsParameter from .interfaces import DBAPICursor + from .interfaces import DBAPIType from .interfaces import Dialect from .interfaces import ExecutionContext from .result import _KeyIndexType @@ -69,6 +73,7 @@ from .result import _KeyType from .result import _ProcessorsType from .result import _TupleGetterType + from ..sql.schema import Column from ..sql.type_api import _ResultProcessorType @@ -79,46 +84,46 @@ # using raw tuple is faster than namedtuple. # these match up to the positions in # _CursorKeyMapRecType -MD_INDEX: Literal[0] = 0 +MD_INDEX: Final[Literal[0]] = 0 """integer index in cursor.description """ -MD_RESULT_MAP_INDEX: Literal[1] = 1 +MD_RESULT_MAP_INDEX: Final[Literal[1]] = 1 """integer index in compiled._result_columns""" -MD_OBJECTS: Literal[2] = 2 +MD_OBJECTS: Final[Literal[2]] = 2 """other string keys and ColumnElement obj that can match. This comes from compiler.RM_OBJECTS / compiler.ResultColumnsEntry.objects """ -MD_LOOKUP_KEY: Literal[3] = 3 +MD_LOOKUP_KEY: Final[Literal[3]] = 3 """string key we usually expect for key-based lookup this comes from compiler.RM_NAME / compiler.ResultColumnsEntry.name """ -MD_RENDERED_NAME: Literal[4] = 4 +MD_RENDERED_NAME: Final[Literal[4]] = 4 """name that is usually in cursor.description this comes from compiler.RENDERED_NAME / compiler.ResultColumnsEntry.keyname """ -MD_PROCESSOR: Literal[5] = 5 +MD_PROCESSOR: Final[Literal[5]] = 5 """callable to process a result value into a row""" -MD_UNTRANSLATED: Literal[6] = 6 +MD_UNTRANSLATED: Final[Literal[6]] = 6 """raw name from cursor.description""" _CursorKeyMapRecType = Tuple[ Optional[int], # MD_INDEX, None means the record is ambiguously named - int, # MD_RESULT_MAP_INDEX - List[Any], # MD_OBJECTS + int, # MD_RESULT_MAP_INDEX, -1 if MD_INDEX is None + TupleAny, # MD_OBJECTS str, # MD_LOOKUP_KEY str, # MD_RENDERED_NAME Optional["_ResultProcessorType[Any]"], # MD_PROCESSOR @@ -139,6 +144,16 @@ str, ] +_MergeColTuple = Tuple[ + int, + Optional[int], + str, + TypeEngine[Any], + "DBAPIType", + Optional[TupleAny], + Optional[str], +] + class CursorResultMetaData(ResultMetaData): """Result metadata for DBAPI cursors.""" @@ -199,11 +214,14 @@ def _make_new_metadata( new_obj._key_to_index = self._make_key_to_index(keymap, MD_INDEX) return new_obj - def _remove_processors(self) -> Self: - assert not self._tuplefilter + def _remove_processors_and_tuple_filter(self) -> Self: + if self._tuplefilter: + proc = self._tuplefilter(self._processors) + else: + proc = self._processors return self._make_new_metadata( unpickled=self._unpickled, - processors=[None] * len(self._processors), + processors=[None] * len(proc), tuplefilter=None, translated_indexes=None, keymap={ @@ -216,31 +234,38 @@ def _remove_processors(self) -> Self: ) def _splice_horizontally(self, other: CursorResultMetaData) -> Self: - assert not self._tuplefilter - keymap = dict(self._keymap) offset = len(self._keys) - keymap.update( - { - key: ( - # int index should be None for ambiguous key - ( - value[0] + offset - if value[0] is not None and key not in keymap - else None - ), - value[1] + offset, - *value[2:], - ) - for key, value in other._keymap.items() - } - ) + + for key, value in other._keymap.items(): + # int index should be None for ambiguous key + if value[MD_INDEX] is not None and key not in keymap: + md_index = value[MD_INDEX] + offset + md_object = value[MD_RESULT_MAP_INDEX] + offset + else: + md_index = None + md_object = -1 + keymap[key] = (md_index, md_object, *value[2:]) + + self_tf = self._tuplefilter + other_tf = other._tuplefilter + + proc: List[Any] = [] + for pp, tf in [ + (self._processors, self_tf), + (other._processors, other_tf), + ]: + proc.extend(pp if tf is None else tf(pp)) + + new_keys = [*self._keys, *other._keys] + assert len(proc) == len(new_keys) + return self._make_new_metadata( unpickled=self._unpickled, - processors=self._processors + other._processors, # type: ignore + processors=proc, tuplefilter=None, translated_indexes=None, - keys=self._keys + other._keys, # type: ignore + keys=new_keys, keymap=keymap, safe_for_cache=self._safe_for_cache, keymap_by_result_column_idx={ @@ -315,14 +340,13 @@ def _adapt_to_context(self, context: ExecutionContext) -> Self: keymap_by_position = self._keymap_by_result_column_idx if keymap_by_position is None: - # first retrival from cache, this map will not be set up yet, + # first retrieval from cache, this map will not be set up yet, # initialize lazily keymap_by_position = self._keymap_by_result_column_idx = { metadata_entry[MD_RESULT_MAP_INDEX]: metadata_entry for metadata_entry in self._keymap.values() } - assert not self._tuplefilter return self._make_new_metadata( keymap=self._keymap | { @@ -334,7 +358,7 @@ def _adapt_to_context(self, context: ExecutionContext) -> Self: }, unpickled=self._unpickled, processors=self._processors, - tuplefilter=None, + tuplefilter=self._tuplefilter, translated_indexes=None, keys=self._keys, safe_for_cache=self._safe_for_cache, @@ -347,9 +371,17 @@ def __init__( cursor_description: _DBAPICursorDescription, *, driver_column_names: bool = False, + num_sentinel_cols: int = 0, ): context = parent.context - self._tuplefilter = None + if num_sentinel_cols > 0: + # this is slightly faster than letting tuplegetter use the indexes + self._tuplefilter = tuplefilter = operator.itemgetter( + slice(-num_sentinel_cols) + ) + cursor_description = tuplefilter(cursor_description) + else: + self._tuplefilter = tuplefilter = None self._translated_indexes = None self._safe_for_cache = self._unpickled = False @@ -361,6 +393,8 @@ def __init__( ad_hoc_textual, loose_column_name_matching, ) = context.result_column_struct + if tuplefilter is not None: + result_columns = tuplefilter(result_columns) num_ctx_cols = len(result_columns) else: result_columns = cols_are_ordered = ( # type: ignore @@ -388,6 +422,10 @@ def __init__( self._processors = [ metadata_entry[MD_PROCESSOR] for metadata_entry in raw ] + if num_sentinel_cols > 0: + # add the number of sentinel columns since these are passed + # to the tuplefilters before being used + self._processors.extend([None] * num_sentinel_cols) # this is used when using this ResultMetaData in a Core-only cache # retrieval context. it's initialized on first cache retrieval @@ -400,7 +438,7 @@ def __init__( # column keys and other names if num_ctx_cols: # keymap by primary string... - by_key = { + by_key: Dict[_KeyType, _CursorKeyMapRecType] = { metadata_entry[MD_LOOKUP_KEY]: metadata_entry for metadata_entry in raw } @@ -446,7 +484,7 @@ def __init__( # record into by_key. by_key.update( { - key: (None, None, [], key, key, None, None) + key: (None, -1, (), key, key, None, None) for key in dupes } ) @@ -500,16 +538,16 @@ def __init__( def _merge_cursor_description( self, - context, - cursor_description, - result_columns, - num_ctx_cols, - cols_are_ordered, - textual_ordered, - ad_hoc_textual, - loose_column_name_matching, - driver_column_names, - ): + context: DefaultExecutionContext, + cursor_description: _DBAPICursorDescription, + result_columns: Sequence[ResultColumnsEntry], + num_ctx_cols: int, + cols_are_ordered: bool, + textual_ordered: bool, + ad_hoc_textual: bool, + loose_column_name_matching: bool, + driver_column_names: bool, + ) -> List[_CursorKeyMapRecType]: """Merge a cursor.description with compiled result column information. There are at least four separate strategies used here, selected @@ -646,7 +684,7 @@ def _merge_cursor_description( mapped_type, cursor_colname, coltype ), untranslated, - ) + ) # type: ignore[misc] for ( idx, ridx, @@ -659,8 +697,11 @@ def _merge_cursor_description( ] def _colnames_from_description( - self, context, cursor_description, driver_column_names - ): + self, + context: DefaultExecutionContext, + cursor_description: _DBAPICursorDescription, + driver_column_names: bool, + ) -> Iterator[Tuple[int, str, str, Optional[str], DBAPIType]]: """Extract column names and data types from a cursor.description. Applies unicode decoding, column translation, "normalization", @@ -698,8 +739,12 @@ def _colnames_from_description( yield idx, colname, unnormalized, untranslated, coltype def _merge_textual_cols_by_position( - self, context, cursor_description, result_columns, driver_column_names - ): + self, + context: DefaultExecutionContext, + cursor_description: _DBAPICursorDescription, + result_columns: Sequence[ResultColumnsEntry], + driver_column_names: bool, + ) -> Iterator[_MergeColTuple]: num_ctx_cols = len(result_columns) if num_ctx_cols > len(cursor_description): @@ -773,12 +818,12 @@ def _merge_textual_cols_by_position( def _merge_cols_by_name( self, - context, - cursor_description, - result_columns, - loose_column_name_matching, - driver_column_names, - ): + context: DefaultExecutionContext, + cursor_description: _DBAPICursorDescription, + result_columns: Sequence[ResultColumnsEntry], + loose_column_name_matching: bool, + driver_column_names: bool, + ) -> Iterator[_MergeColTuple]: match_map = self._create_description_match_map( result_columns, loose_column_name_matching ) @@ -824,11 +869,9 @@ def _merge_cols_by_name( @classmethod def _create_description_match_map( cls, - result_columns: List[ResultColumnsEntry], + result_columns: Sequence[ResultColumnsEntry], loose_column_name_matching: bool = False, - ) -> Dict[ - Union[str, object], Tuple[str, Tuple[Any, ...], TypeEngine[Any], int] - ]: + ) -> Dict[Union[str, object], Tuple[str, TupleAny, TypeEngine[Any], int]]: """when matching cursor.description to a set of names that are present in a Compiled object, as is the case with TextualSelect, get all the names we expect might match those in cursor.description. @@ -836,7 +879,7 @@ def _create_description_match_map( d: Dict[ Union[str, object], - Tuple[str, Tuple[Any, ...], TypeEngine[Any], int], + Tuple[str, TupleAny, TypeEngine[Any], int], ] = {} for ridx, elem in enumerate(result_columns): key = elem[RM_RENDERED_NAME] @@ -865,8 +908,11 @@ def _create_description_match_map( return d def _merge_cols_by_none( - self, context, cursor_description, driver_column_names - ): + self, + context: DefaultExecutionContext, + cursor_description: _DBAPICursorDescription, + driver_column_names: bool, + ) -> Iterator[_MergeColTuple]: self._keys = [] for ( @@ -914,13 +960,17 @@ def _key_fallback( else: return None - def _raise_for_ambiguous_column_name(self, rec): + def _raise_for_ambiguous_column_name( + self, rec: _KeyMapRecType + ) -> NoReturn: raise exc.InvalidRequestError( "Ambiguous column name '%s' in " "result set column descriptions" % rec[MD_LOOKUP_KEY] ) - def _index_for_key(self, key: Any, raiseerr: bool = True) -> Optional[int]: + def _index_for_key( + self, key: _KeyIndexType, raiseerr: bool = True + ) -> Optional[int]: # TODO: can consider pre-loading ints and negative ints # into _keymap - also no coverage here if isinstance(key, int): @@ -939,18 +989,20 @@ def _index_for_key(self, key: Any, raiseerr: bool = True) -> Optional[int]: self._raise_for_ambiguous_column_name(rec) return index - def _indexes_for_keys(self, keys): + def _indexes_for_keys( + self, keys: Sequence[_KeyIndexType] + ) -> Sequence[int]: try: - return [self._keymap[key][0] for key in keys] + return [self._keymap[key][0] for key in keys] # type: ignore[index,misc] # noqa: E501 except KeyError as ke: # ensure it raises CursorResultMetaData._key_fallback(self, ke.args[0], ke) def _metadata_for_keys( - self, keys: Sequence[Any] + self, keys: Sequence[_KeyIndexType] ) -> Iterator[_NonAmbigCursorKeyMapRecType]: for key in keys: - if int in key.__class__.__mro__: + if isinstance(key, int): key = self._keys[key] try: @@ -966,7 +1018,7 @@ def _metadata_for_keys( yield cast(_NonAmbigCursorKeyMapRecType, rec) - def __getstate__(self): + def __getstate__(self) -> Dict[str, Any]: # TODO: consider serializing this as SimpleResultMetaData return { "_keymap": { @@ -986,7 +1038,7 @@ def __getstate__(self): "_translated_indexes": self._translated_indexes, } - def __setstate__(self, state): + def __setstate__(self, state: Dict[str, Any]) -> None: self._processors = [None for _ in range(len(state["_keys"]))] self._keymap = state["_keymap"] self._keymap_by_result_column_idx = None @@ -994,10 +1046,11 @@ def __setstate__(self, state): self._keys = state["_keys"] self._unpickled = True if state["_translated_indexes"]: - self._translated_indexes = cast( - "List[int]", state["_translated_indexes"] - ) - self._tuplefilter = tuplegetter(*self._translated_indexes) + translated_indexes: List[Any] + self._translated_indexes = translated_indexes = state[ + "_translated_indexes" + ] + self._tuplefilter = tuplegetter(*translated_indexes) else: self._translated_indexes = self._tuplefilter = None @@ -1031,7 +1084,7 @@ def hard_close( def yield_per( self, result: CursorResult[Unpack[TupleAny]], - dbapi_cursor: Optional[DBAPICursor], + dbapi_cursor: DBAPICursor, num: int, ) -> None: return @@ -1079,22 +1132,47 @@ class NoCursorFetchStrategy(ResultFetchStrategy): __slots__ = () - def soft_close(self, result, dbapi_cursor): + def soft_close( + self, + result: CursorResult[Unpack[TupleAny]], + dbapi_cursor: Optional[DBAPICursor], + ) -> None: pass - def hard_close(self, result, dbapi_cursor): + def hard_close( + self, + result: CursorResult[Unpack[TupleAny]], + dbapi_cursor: Optional[DBAPICursor], + ) -> None: pass - def fetchone(self, result, dbapi_cursor, hard_close=False): + def fetchone( + self, + result: CursorResult[Unpack[TupleAny]], + dbapi_cursor: DBAPICursor, + hard_close: bool = False, + ) -> Any: return self._non_result(result, None) - def fetchmany(self, result, dbapi_cursor, size=None): + def fetchmany( + self, + result: CursorResult[Unpack[TupleAny]], + dbapi_cursor: DBAPICursor, + size: Optional[int] = None, + ) -> Any: return self._non_result(result, []) - def fetchall(self, result, dbapi_cursor): + def fetchall( + self, result: CursorResult[Unpack[TupleAny]], dbapi_cursor: DBAPICursor + ) -> Any: return self._non_result(result, []) - def _non_result(self, result, default, err=None): + def _non_result( + self, + result: CursorResult[Unpack[TupleAny]], + default: Any, + err: Optional[BaseException] = None, + ) -> Any: raise NotImplementedError() @@ -1111,7 +1189,12 @@ class NoCursorDQLFetchStrategy(NoCursorFetchStrategy): __slots__ = () - def _non_result(self, result, default, err=None): + def _non_result( + self, + result: CursorResult[Unpack[TupleAny]], + default: Any, + err: Optional[BaseException] = None, + ) -> Any: if result.closed: raise exc.ResourceClosedError( "This result object is closed." @@ -1133,10 +1216,15 @@ class NoCursorDMLFetchStrategy(NoCursorFetchStrategy): __slots__ = () - def _non_result(self, result, default, err=None): + def _non_result( + self, + result: CursorResult[Unpack[TupleAny]], + default: Any, + err: Optional[BaseException] = None, + ) -> Any: # we only expect to have a _NoResultMetaData() here right now. assert not result._metadata.returns_rows - result._metadata._we_dont_return_rows(err) + result._metadata._we_dont_return_rows(err) # type: ignore[union-attr] _NO_CURSOR_DML = NoCursorDMLFetchStrategy() @@ -1173,10 +1261,7 @@ def handle_exception( ) def yield_per( - self, - result: CursorResult[Any], - dbapi_cursor: Optional[DBAPICursor], - num: int, + self, result: CursorResult[Any], dbapi_cursor: DBAPICursor, num: int ) -> None: result.cursor_strategy = BufferedRowCursorFetchStrategy( dbapi_cursor, @@ -1265,11 +1350,11 @@ class BufferedRowCursorFetchStrategy(CursorFetchStrategy): def __init__( self, - dbapi_cursor, - execution_options, - growth_factor=5, - initial_buffer=None, - ): + dbapi_cursor: DBAPICursor, + execution_options: CoreExecuteOptionsParameter, + growth_factor: int = 5, + initial_buffer: Optional[Deque[Any]] = None, + ) -> None: self._max_row_buffer = execution_options.get("max_row_buffer", 1000) if initial_buffer is not None: @@ -1284,13 +1369,17 @@ def __init__( self._bufsize = self._max_row_buffer @classmethod - def create(cls, result): + def create( + cls, result: CursorResult[Any] + ) -> BufferedRowCursorFetchStrategy: return BufferedRowCursorFetchStrategy( result.cursor, result.context.execution_options, ) - def _buffer_rows(self, result, dbapi_cursor): + def _buffer_rows( + self, result: CursorResult[Any], dbapi_cursor: DBAPICursor + ) -> None: """this is currently used only by fetchone().""" size = self._bufsize @@ -1310,19 +1399,30 @@ def _buffer_rows(self, result, dbapi_cursor): self._max_row_buffer, size * self._growth_factor ) - def yield_per(self, result, dbapi_cursor, num): + def yield_per( + self, result: CursorResult[Any], dbapi_cursor: DBAPICursor, num: int + ) -> None: self._growth_factor = 0 self._max_row_buffer = self._bufsize = num - def soft_close(self, result, dbapi_cursor): + def soft_close( + self, result: CursorResult[Any], dbapi_cursor: Optional[DBAPICursor] + ) -> None: self._rowbuffer.clear() super().soft_close(result, dbapi_cursor) - def hard_close(self, result, dbapi_cursor): + def hard_close( + self, result: CursorResult[Any], dbapi_cursor: Optional[DBAPICursor] + ) -> None: self._rowbuffer.clear() super().hard_close(result, dbapi_cursor) - def fetchone(self, result, dbapi_cursor, hard_close=False): + def fetchone( + self, + result: CursorResult[Any], + dbapi_cursor: DBAPICursor, + hard_close: bool = False, + ) -> Any: if not self._rowbuffer: self._buffer_rows(result, dbapi_cursor) if not self._rowbuffer: @@ -1333,7 +1433,12 @@ def fetchone(self, result, dbapi_cursor, hard_close=False): return None return self._rowbuffer.popleft() - def fetchmany(self, result, dbapi_cursor, size=None): + def fetchmany( + self, + result: CursorResult[Any], + dbapi_cursor: DBAPICursor, + size: Optional[int] = None, + ) -> Any: if size is None: return self.fetchall(result, dbapi_cursor) @@ -1357,7 +1462,9 @@ def fetchmany(self, result, dbapi_cursor, size=None): result._soft_close() return res - def fetchall(self, result, dbapi_cursor): + def fetchall( + self, result: CursorResult[Any], dbapi_cursor: DBAPICursor + ) -> Any: try: ret = list(self._rowbuffer) + list(dbapi_cursor.fetchall()) self._rowbuffer.clear() @@ -1379,33 +1486,53 @@ class FullyBufferedCursorFetchStrategy(CursorFetchStrategy): __slots__ = ("_rowbuffer", "alternate_cursor_description") def __init__( - self, dbapi_cursor, alternate_description=None, initial_buffer=None + self, + dbapi_cursor: Optional[DBAPICursor], + alternate_description: Optional[_DBAPICursorDescription] = None, + initial_buffer: Optional[Iterable[Any]] = None, ): self.alternate_cursor_description = alternate_description if initial_buffer is not None: self._rowbuffer = collections.deque(initial_buffer) else: + assert dbapi_cursor is not None self._rowbuffer = collections.deque(dbapi_cursor.fetchall()) - def yield_per(self, result, dbapi_cursor, num): + def yield_per( + self, result: CursorResult[Any], dbapi_cursor: DBAPICursor, num: int + ) -> Any: pass - def soft_close(self, result, dbapi_cursor): + def soft_close( + self, result: CursorResult[Any], dbapi_cursor: Optional[DBAPICursor] + ) -> None: self._rowbuffer.clear() super().soft_close(result, dbapi_cursor) - def hard_close(self, result, dbapi_cursor): + def hard_close( + self, result: CursorResult[Any], dbapi_cursor: Optional[DBAPICursor] + ) -> None: self._rowbuffer.clear() super().hard_close(result, dbapi_cursor) - def fetchone(self, result, dbapi_cursor, hard_close=False): + def fetchone( + self, + result: CursorResult[Any], + dbapi_cursor: DBAPICursor, + hard_close: bool = False, + ) -> Any: if self._rowbuffer: return self._rowbuffer.popleft() else: result._soft_close(hard=hard_close) return None - def fetchmany(self, result, dbapi_cursor, size=None): + def fetchmany( + self, + result: CursorResult[Any], + dbapi_cursor: DBAPICursor, + size: Optional[int] = None, + ) -> Any: if size is None: return self.fetchall(result, dbapi_cursor) @@ -1415,7 +1542,9 @@ def fetchmany(self, result, dbapi_cursor, size=None): result._soft_close() return rows - def fetchall(self, result, dbapi_cursor): + def fetchall( + self, result: CursorResult[Any], dbapi_cursor: DBAPICursor + ) -> Any: ret = self._rowbuffer self._rowbuffer = collections.deque() result._soft_close() @@ -1427,35 +1556,37 @@ class _NoResultMetaData(ResultMetaData): returns_rows = False - def _we_dont_return_rows(self, err=None): + def _we_dont_return_rows( + self, err: Optional[BaseException] = None + ) -> NoReturn: raise exc.ResourceClosedError( "This result object does not return rows. " "It has been closed automatically." ) from err - def _index_for_key(self, keys, raiseerr): + def _index_for_key(self, keys: _KeyIndexType, raiseerr: bool) -> NoReturn: self._we_dont_return_rows() - def _metadata_for_keys(self, key): + def _metadata_for_keys(self, keys: Sequence[_KeyIndexType]) -> NoReturn: self._we_dont_return_rows() - def _reduce(self, keys): + def _reduce(self, keys: Sequence[_KeyIndexType]) -> NoReturn: self._we_dont_return_rows() @property - def _keymap(self): + def _keymap(self) -> NoReturn: # type: ignore[override] self._we_dont_return_rows() @property - def _key_to_index(self): + def _key_to_index(self) -> NoReturn: # type: ignore[override] self._we_dont_return_rows() @property - def _processors(self): + def _processors(self) -> NoReturn: # type: ignore[override] self._we_dont_return_rows() @property - def keys(self): + def keys(self) -> NoReturn: self._we_dont_return_rows() @@ -1525,54 +1656,29 @@ def __init__( ) if cursor_description is not None: - # inline of Result._row_getter(), set up an initial row - # getter assuming no transformations will be called as this - # is the most common case - - metadata = self._init_metadata(context, cursor_description) - - _make_row: Any - _make_row = functools.partial( - Row, - metadata, - metadata._effective_processors, - metadata._key_to_index, - ) - - if context._num_sentinel_cols: - sentinel_filter = operator.itemgetter( - slice(-context._num_sentinel_cols) - ) - - def _sliced_row(raw_data): - return _make_row(sentinel_filter(raw_data)) - - sliced_row = _sliced_row - else: - sliced_row = _make_row + self._init_metadata(context, cursor_description) if echo: log = self.context.connection._log_debug - def _log_row(row): + def _log_row(row: Any) -> Any: log("Row %r", sql_util._repr_row(row)) return row self._row_logging_fn = _log_row - def _make_row_2(row): - return _log_row(sliced_row(row)) - - make_row = _make_row_2 - else: - make_row = sliced_row - self._set_memoized_attribute("_row_getter", make_row) + # call Result._row_getter to set up the row factory + self._row_getter else: assert context._num_sentinel_cols == 0 self._metadata = self._no_result_metadata - def _init_metadata(self, context, cursor_description): + def _init_metadata( + self, + context: DefaultExecutionContext, + cursor_description: _DBAPICursorDescription, + ) -> CursorResultMetaData: driver_column_names = context.execution_options.get( "driver_column_names", False ) @@ -1582,14 +1688,25 @@ def _init_metadata(self, context, cursor_description): metadata: CursorResultMetaData if driver_column_names: + # TODO: test this case metadata = CursorResultMetaData( - self, cursor_description, driver_column_names=True + self, + cursor_description, + driver_column_names=True, + num_sentinel_cols=context._num_sentinel_cols, ) assert not metadata._safe_for_cache elif compiled._cached_metadata: metadata = compiled._cached_metadata else: - metadata = CursorResultMetaData(self, cursor_description) + metadata = CursorResultMetaData( + self, + cursor_description, + # the number of sentinel columns is stored on the context + # but it's a characteristic of the compiled object + # so it's ok to apply it to a cacheable metadata. + num_sentinel_cols=context._num_sentinel_cols, + ) if metadata._safe_for_cache: compiled._cached_metadata = metadata @@ -1613,7 +1730,7 @@ def _init_metadata(self, context, cursor_description): ) and compiled._result_columns and context.cache_hit is context.dialect.CACHE_HIT - and compiled.statement is not context.invoked_statement + and compiled.statement is not context.invoked_statement # type: ignore[comparison-overlap] # noqa: E501 ): metadata = metadata._adapt_to_context(context) @@ -1631,7 +1748,7 @@ def _init_metadata(self, context, cursor_description): ) return metadata - def _soft_close(self, hard=False): + def _soft_close(self, hard: bool = False) -> None: """Soft close this :class:`_engine.CursorResult`. This releases all DBAPI cursor resources, but leaves the @@ -1669,7 +1786,7 @@ def _soft_close(self, hard=False): self._soft_closed = True @property - def inserted_primary_key_rows(self): + def inserted_primary_key_rows(self) -> List[Optional[Any]]: """Return the value of :attr:`_engine.CursorResult.inserted_primary_key` as a row contained within a list; some dialects may support a @@ -1728,10 +1845,10 @@ def inserted_primary_key_rows(self): "when returning() " "is used." ) - return self.context.inserted_primary_key_rows + return self.context.inserted_primary_key_rows # type: ignore[no-any-return] # noqa: E501 @property - def inserted_primary_key(self): + def inserted_primary_key(self) -> Optional[Any]: """Return the primary key for the row just inserted. The return value is a :class:`_result.Row` object representing @@ -1776,7 +1893,11 @@ def inserted_primary_key(self): else: return None - def last_updated_params(self): + def last_updated_params( + self, + ) -> Union[ + List[_MutableCoreSingleExecuteParams], _MutableCoreSingleExecuteParams + ]: """Return the collection of updated parameters from this execution. @@ -1798,7 +1919,11 @@ def last_updated_params(self): else: return self.context.compiled_parameters[0] - def last_inserted_params(self): + def last_inserted_params( + self, + ) -> Union[ + List[_MutableCoreSingleExecuteParams], _MutableCoreSingleExecuteParams + ]: """Return the collection of inserted parameters from this execution. @@ -1821,7 +1946,9 @@ def last_inserted_params(self): return self.context.compiled_parameters[0] @property - def returned_defaults_rows(self): + def returned_defaults_rows( + self, + ) -> Optional[Sequence[Row[Unpack[TupleAny]]]]: """Return a list of rows each containing the values of default columns that were fetched using the :meth:`.ValuesBase.return_defaults` feature. @@ -1833,7 +1960,7 @@ def returned_defaults_rows(self): """ return self.context.returned_default_rows - def splice_horizontally(self, other): + def splice_horizontally(self, other: CursorResult[Any]) -> Self: """Return a new :class:`.CursorResult` that "horizontally splices" together the rows of this :class:`.CursorResult` with that of another :class:`.CursorResult`. @@ -1889,16 +2016,22 @@ def splice_horizontally(self, other): """ # noqa: E501 clone = self._generate() + assert clone is self # just to note + assert isinstance(other._metadata, CursorResultMetaData) + assert isinstance(self._metadata, CursorResultMetaData) + self_tf = self._metadata._tuplefilter + other_tf = other._metadata._tuplefilter + clone._metadata = self._metadata._splice_horizontally(other._metadata) + total_rows = [ - tuple(r1) + tuple(r2) + tuple(r1 if self_tf is None else self_tf(r1)) + + tuple(r2 if other_tf is None else other_tf(r2)) for r1, r2 in zip( list(self._raw_row_iterator()), list(other._raw_row_iterator()), ) ] - clone._metadata = clone._metadata._splice_horizontally(other._metadata) - clone.cursor_strategy = FullyBufferedCursorFetchStrategy( None, initial_buffer=total_rows, @@ -1906,7 +2039,7 @@ def splice_horizontally(self, other): clone._reset_memoizations() return clone - def splice_vertically(self, other): + def splice_vertically(self, other: CursorResult[Any]) -> Self: """Return a new :class:`.CursorResult` that "vertically splices", i.e. "extends", the rows of this :class:`.CursorResult` with that of another :class:`.CursorResult`. @@ -1938,7 +2071,7 @@ def splice_vertically(self, other): clone._reset_memoizations() return clone - def _rewind(self, rows): + def _rewind(self, rows: Any) -> Self: """rewind this result back to the given rowset. this is used internally for the case where an :class:`.Insert` @@ -1946,6 +2079,9 @@ def _rewind(self, rows): :meth:`.Insert.return_defaults` along with the "supplemental columns" feature. + NOTE: this method has not effect then an unique filter is applied + to the result, meaning that no row will be returned. + """ if self._echo: @@ -1958,7 +2094,7 @@ def _rewind(self, rows): # rows self._metadata = cast( CursorResultMetaData, self._metadata - )._remove_processors() + )._remove_processors_and_tuple_filter() self.cursor_strategy = FullyBufferedCursorFetchStrategy( None, @@ -1971,7 +2107,7 @@ def _rewind(self, rows): return self @property - def returned_defaults(self): + def returned_defaults(self) -> Optional[Row[Unpack[TupleAny]]]: """Return the values of default columns that were fetched using the :meth:`.ValuesBase.return_defaults` feature. @@ -1997,7 +2133,7 @@ def returned_defaults(self): else: return None - def lastrow_has_defaults(self): + def lastrow_has_defaults(self) -> bool: """Return ``lastrow_has_defaults()`` from the underlying :class:`.ExecutionContext`. @@ -2007,7 +2143,7 @@ def lastrow_has_defaults(self): return self.context.lastrow_has_defaults() - def postfetch_cols(self): + def postfetch_cols(self) -> Optional[Sequence[Column[Any]]]: """Return ``postfetch_cols()`` from the underlying :class:`.ExecutionContext`. @@ -2030,7 +2166,7 @@ def postfetch_cols(self): ) return self.context.postfetch_cols - def prefetch_cols(self): + def prefetch_cols(self) -> Optional[Sequence[Column[Any]]]: """Return ``prefetch_cols()`` from the underlying :class:`.ExecutionContext`. @@ -2053,7 +2189,7 @@ def prefetch_cols(self): ) return self.context.prefetch_cols - def supports_sane_rowcount(self): + def supports_sane_rowcount(self) -> bool: """Return ``supports_sane_rowcount`` from the dialect. See :attr:`_engine.CursorResult.rowcount` for background. @@ -2062,7 +2198,7 @@ def supports_sane_rowcount(self): return self.dialect.supports_sane_rowcount - def supports_sane_multi_rowcount(self): + def supports_sane_multi_rowcount(self) -> bool: """Return ``supports_sane_multi_rowcount`` from the dialect. See :attr:`_engine.CursorResult.rowcount` for background. @@ -2154,7 +2290,7 @@ def rowcount(self) -> int: raise # not called @property - def lastrowid(self): + def lastrowid(self) -> int: """Return the 'lastrowid' accessor on the DBAPI cursor. This is a DBAPI specific method and is only functional @@ -2175,7 +2311,7 @@ def lastrowid(self): self.cursor_strategy.handle_exception(self, self.cursor, e) @property - def returns_rows(self): + def returns_rows(self) -> bool: """True if this :class:`_engine.CursorResult` returns zero or more rows. @@ -2198,12 +2334,17 @@ def returns_rows(self): using the MSSQL / pyodbc dialect a SELECT is emitted inline in order to retrieve an inserted primary key value. + .. seealso:: + + :meth:`.Result.close` + + :attr:`.Result.closed` """ return self._metadata.returns_rows @property - def is_insert(self): + def is_insert(self) -> bool: """True if this :class:`_engine.CursorResult` is the result of a executing an expression language compiled :func:`_expression.insert` construct. @@ -2216,7 +2357,7 @@ def is_insert(self): """ return self.context.isinsert - def _fetchiter_impl(self): + def _fetchiter_impl(self) -> Iterator[Any]: fetchone = self.cursor_strategy.fetchone while True: @@ -2225,16 +2366,16 @@ def _fetchiter_impl(self): break yield row - def _fetchone_impl(self, hard_close=False): + def _fetchone_impl(self, hard_close: bool = False) -> Any: return self.cursor_strategy.fetchone(self, self.cursor, hard_close) - def _fetchall_impl(self): + def _fetchall_impl(self) -> Any: return self.cursor_strategy.fetchall(self, self.cursor) - def _fetchmany_impl(self, size=None): + def _fetchmany_impl(self, size: Optional[int] = None) -> Any: return self.cursor_strategy.fetchmany(self, self.cursor, size) - def _raw_row_iterator(self): + def _raw_row_iterator(self) -> Any: return self._fetchiter_impl() def merge( @@ -2248,7 +2389,7 @@ def merge( ) return merged_result - def close(self) -> Any: + def close(self) -> None: """Close this :class:`_engine.CursorResult`. This closes out the underlying DBAPI cursor corresponding to the diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index ba59ac297bc..833ce049642 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -1,5 +1,5 @@ # engine/default.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -28,6 +28,7 @@ from typing import Dict from typing import Final from typing import List +from typing import Literal from typing import Mapping from typing import MutableMapping from typing import MutableSequence @@ -62,28 +63,29 @@ from ..sql import util as sql_util from ..sql._typing import is_tuple_type from ..sql.base import _NoArg +from ..sql.compiler import AggregateOrderByStyle from ..sql.compiler import DDLCompiler from ..sql.compiler import InsertmanyvaluesSentinelOpts from ..sql.compiler import SQLCompiler from ..sql.elements import quoted_name -from ..util.typing import Literal from ..util.typing import TupleAny from ..util.typing import Unpack - if typing.TYPE_CHECKING: - from types import ModuleType - from .base import Engine from .cursor import ResultFetchStrategy from .interfaces import _CoreMultiExecuteParams from .interfaces import _CoreSingleExecuteParams from .interfaces import _DBAPICursorDescription from .interfaces import _DBAPIMultiExecuteParams + from .interfaces import _DBAPISingleExecuteParams from .interfaces import _ExecuteOptions from .interfaces import _MutableCoreSingleExecuteParams from .interfaces import _ParamStyle + from .interfaces import ConnectArgsType from .interfaces import DBAPIConnection + from .interfaces import DBAPIModule + from .interfaces import DBAPIType from .interfaces import IsolationLevel from .row import Row from .url import URL @@ -98,10 +100,12 @@ from ..sql.dml import UpdateBase from ..sql.elements import BindParameter from ..sql.schema import Column + from ..sql.sqltypes import _JSON_VALUE from ..sql.type_api import _BindProcessorType from ..sql.type_api import _ResultProcessorType from ..sql.type_api import TypeEngine + # When we're handed literal SQL, ensure it's a SELECT query SERVER_SIDE_CURSOR_RE = re.compile(r"\s*SELECT", re.I | re.UNICODE) @@ -158,6 +162,8 @@ class DefaultDialect(Dialect): delete_returning_multifrom = False insert_returning = False + aggregate_order_by_style = AggregateOrderByStyle.INLINE + cte_follows_insert = False supports_native_enum = False @@ -165,6 +171,13 @@ class DefaultDialect(Dialect): supports_native_uuid = False returns_native_bytes = False + supports_native_json_serialization = False + supports_native_json_deserialization = False + dialect_injects_custom_json_deserializer = False + _json_serializer: Callable[[_JSON_VALUE], str] | None = None + + _json_deserializer: Callable[[str], _JSON_VALUE] | None = None + non_native_boolean_check_constraint = True supports_simple_order_by_label = True @@ -295,7 +308,7 @@ def __init__( self, paramstyle: Optional[_ParamStyle] = None, isolation_level: Optional[IsolationLevel] = None, - dbapi: Optional[ModuleType] = None, + dbapi: Optional[DBAPIModule] = None, implicit_returning: Literal[True] = True, supports_native_boolean: Optional[bool] = None, max_identifier_length: Optional[int] = None, @@ -306,6 +319,7 @@ def __init__( # Linting.NO_LINTING constant compiler_linting: Linting = int(compiler.NO_LINTING), # type: ignore server_side_cursors: bool = False, + skip_autocommit_rollback: bool = False, **kwargs: Any, ): if server_side_cursors: @@ -330,6 +344,8 @@ def __init__( self.dbapi = dbapi + self.skip_autocommit_rollback = skip_autocommit_rollback + if paramstyle is not None: self.paramstyle = paramstyle elif self.dbapi is not None: @@ -428,7 +444,7 @@ def insert_executemany_returning_sort_by_parameter_order(self): delete_executemany_returning = False @util.memoized_property - def loaded_dbapi(self) -> ModuleType: + def loaded_dbapi(self) -> DBAPIModule: if self.dbapi is None: raise exc.InvalidRequestError( f"Dialect {self} does not have a Python DBAPI established " @@ -440,7 +456,7 @@ def loaded_dbapi(self) -> ModuleType: def _bind_typing_render_casts(self): return self.bind_typing is interfaces.BindTyping.RENDER_CASTS - def _ensure_has_table_connection(self, arg): + def _ensure_has_table_connection(self, arg: Connection) -> None: if not isinstance(arg, Connection): raise exc.ArgumentError( "The argument passed to Dialect.has_table() should be a " @@ -477,7 +493,7 @@ def _type_memos(self): return weakref.WeakKeyDictionary() @property - def dialect_description(self): + def dialect_description(self): # type: ignore[override] return self.name + "+" + self.driver @property @@ -524,7 +540,7 @@ def builtin_connect(dbapi_conn, conn_rec): else: return None - def initialize(self, connection): + def initialize(self, connection: Connection) -> None: try: self.server_version_info = self._get_server_version_info( connection @@ -560,7 +576,7 @@ def initialize(self, connection): % (self.label_length, self.max_identifier_length) ) - def on_connect(self): + def on_connect(self) -> Optional[Callable[[Any], None]]: # inherits the docstring from interfaces.Dialect.on_connect return None @@ -571,8 +587,6 @@ def _check_max_identifier_length(self, connection): If the dialect's class level max_identifier_length should be used, can return None. - .. versionadded:: 1.3.9 - """ return None @@ -587,8 +601,6 @@ def get_default_isolation_level(self, dbapi_conn): By default, calls the :meth:`_engine.Interfaces.get_isolation_level` method, propagating any exceptions raised. - .. versionadded:: 1.3.22 - """ return self.get_isolation_level(dbapi_conn) @@ -619,18 +631,18 @@ def has_schema( ) -> bool: return schema_name in self.get_schema_names(connection, **kw) - def validate_identifier(self, ident): + def validate_identifier(self, ident: str) -> None: if len(ident) > self.max_identifier_length: raise exc.IdentifierError( "Identifier '%s' exceeds maximum length of %d characters" % (ident, self.max_identifier_length) ) - def connect(self, *cargs, **cparams): + def connect(self, *cargs: Any, **cparams: Any) -> DBAPIConnection: # inherits the docstring from interfaces.Dialect.connect - return self.loaded_dbapi.connect(*cargs, **cparams) + return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501 - def create_connect_args(self, url): + def create_connect_args(self, url: URL) -> ConnectArgsType: # inherits the docstring from interfaces.Dialect.create_connect_args opts = url.translate_connect_args() opts.update(url.query) @@ -706,6 +718,10 @@ def do_begin(self, dbapi_connection): pass def do_rollback(self, dbapi_connection): + if self.skip_autocommit_rollback and self.detect_autocommit_setting( + dbapi_connection + ): + return dbapi_connection.rollback() def do_commit(self, dbapi_connection): @@ -745,8 +761,6 @@ def _do_ping_w_event(self, dbapi_connection: DBAPIConnection) -> bool: raise def do_ping(self, dbapi_connection: DBAPIConnection) -> bool: - cursor = None - cursor = dbapi_connection.cursor() try: cursor.execute(self._dialect_specific_select_one) @@ -887,6 +901,7 @@ def _deliver_insertmanyvalues_batches( Dict[Tuple[Any, ...], Any], Dict[Any, Any], ] + if composite_sentinel: rows_by_sentinel = { tuple( @@ -953,7 +968,14 @@ def do_execute(self, cursor, statement, parameters, context=None): def do_execute_no_params(self, cursor, statement, context=None): cursor.execute(statement) - def is_disconnect(self, e, connection, cursor): + def is_disconnect( + self, + e: DBAPIModule.Error, + connection: Union[ + pool.PoolProxiedConnection, interfaces.DBAPIConnection, None + ], + cursor: Optional[interfaces.DBAPICursor], + ) -> bool: return False @util.memoized_instancemethod @@ -1053,7 +1075,7 @@ def denormalize_name(self, name): name = name_upper return name - def get_driver_connection(self, connection): + def get_driver_connection(self, connection: DBAPIConnection) -> Any: return connection def _overrides_default(self, method): @@ -1230,7 +1252,9 @@ class DefaultExecutionContext(ExecutionContext): # a hook for SQLite's translation of # result column names # NOTE: pyhive is using this hook, can't remove it :( - _translate_colname: Optional[Callable[[str], str]] = None + _translate_colname: Optional[ + Callable[[str], Tuple[str, Optional[str]]] + ] = None _expanded_parameters: Mapping[str, List[str]] = util.immutabledict() """used by set_input_sizes(). @@ -1312,6 +1336,7 @@ def _init_compiled( invoked_statement: Executable, extracted_parameters: Optional[Sequence[BindParameter[Any]]], cache_hit: CacheStats = CacheStats.CACHING_DISABLED, + param_dict: _CoreSingleExecuteParams | None = None, ) -> ExecutionContext: """Initialize execution context for a Compiled construct.""" @@ -1400,6 +1425,7 @@ def _init_compiled( compiled.construct_params( extracted_parameters=extracted_parameters, escape_names=False, + _collected_params=param_dict, ) ] else: @@ -1409,6 +1435,7 @@ def _init_compiled( escape_names=False, _group_number=grp, extracted_parameters=extracted_parameters, + _collected_params=param_dict, ) for grp, m in enumerate(parameters) ] @@ -1627,7 +1654,7 @@ def _get_cache_stats(self) -> str: return "unknown" @property - def executemany(self): + def executemany(self): # type: ignore[override] return self.execute_style in ( ExecuteStyle.EXECUTEMANY, ExecuteStyle.INSERTMANYVALUES, @@ -1669,7 +1696,12 @@ def prefetch_cols(self) -> Optional[Sequence[Column[Any]]]: def no_parameters(self): return self.execution_options.get("no_parameters", False) - def _execute_scalar(self, stmt, type_, parameters=None): + def _execute_scalar( + self, + stmt: str, + type_: Optional[TypeEngine[Any]], + parameters: Optional[_DBAPISingleExecuteParams] = None, + ) -> Any: """Execute a string statement on the current cursor, returning a scalar result. @@ -1743,7 +1775,7 @@ def _use_server_side_cursor(self): return use_server_side - def create_cursor(self): + def create_cursor(self) -> DBAPICursor: if ( # inlining initial preference checks for SS cursors self.dialect.supports_server_side_cursors @@ -1764,10 +1796,10 @@ def create_cursor(self): def fetchall_for_returning(self, cursor): return cursor.fetchall() - def create_default_cursor(self): + def create_default_cursor(self) -> DBAPICursor: return self._dbapi_connection.cursor() - def create_server_side_cursor(self): + def create_server_side_cursor(self) -> DBAPICursor: raise NotImplementedError() def pre_exec(self): @@ -1781,7 +1813,9 @@ def get_out_parameter_values(self, names): def post_exec(self): pass - def get_result_processor(self, type_, colname, coltype): + def get_result_processor( + self, type_: TypeEngine[Any], colname: str, coltype: DBAPIType + ) -> Optional[_ResultProcessorType[Any]]: """Return a 'result processor' for a given type as present in cursor.description. @@ -1791,7 +1825,7 @@ def get_result_processor(self, type_, colname, coltype): """ return type_._cached_result_processor(self.dialect, coltype) - def get_lastrowid(self): + def get_lastrowid(self) -> int: """return self.cursor.lastrowid, or equivalent, after an INSERT. This may involve calling special cursor functions, issuing a new SELECT @@ -1836,9 +1870,10 @@ def _setup_result_proxy(self): if self._rowcount is None and exec_opt.get("preserve_rowcount", False): self._rowcount = self.cursor.rowcount + yp: Optional[Union[int, bool]] if self.is_crud or self.is_text: result = self._setup_dml_or_text_result() - yp = sr = False + yp = False else: yp = exec_opt.get("yield_per", None) sr = self._is_server_side or exec_opt.get("stream_results", False) @@ -1945,11 +1980,8 @@ def _setup_dml_or_text_result(self): strategy = _cursor._NO_CURSOR_DML elif self._num_sentinel_cols: assert self.execute_style is ExecuteStyle.INSERTMANYVALUES - # strip out the sentinel columns from cursor description - # a similar logic is done to the rows only in CursorResult - cursor_description = cursor_description[ - 0 : -self._num_sentinel_cols - ] + # the sentinel columns are handled in CursorResult._init_metadata + # using essentially _reduce result: _cursor.CursorResult[Any] = _cursor.CursorResult( self, strategy, cursor_description @@ -2047,7 +2079,7 @@ def _setup_ins_pk_from_implicit_returning(self, result, rows): getter(row, param) for row, param in zip(rows, compiled_params) ] - def lastrow_has_defaults(self): + def lastrow_has_defaults(self) -> bool: return (self.isinsert or self.isupdate) and bool( cast(SQLCompiler, self.compiled).postfetch ) @@ -2258,12 +2290,6 @@ def get_current_parameters(self, isolate_multiinsert_groups=True): raw parameters of the statement are returned including the naming convention used in the case of multi-valued INSERT. - .. versionadded:: 1.2 added - :meth:`.DefaultExecutionContext.get_current_parameters` - which provides more functionality over the existing - :attr:`.DefaultExecutionContext.current_parameters` - attribute. - .. seealso:: :attr:`.DefaultExecutionContext.current_parameters` diff --git a/lib/sqlalchemy/engine/events.py b/lib/sqlalchemy/engine/events.py index dbaac3789e6..a6c3b457435 100644 --- a/lib/sqlalchemy/engine/events.py +++ b/lib/sqlalchemy/engine/events.py @@ -1,5 +1,5 @@ # engine/events.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -11,6 +11,7 @@ import typing from typing import Any from typing import Dict +from typing import Literal from typing import Optional from typing import Tuple from typing import Type @@ -24,7 +25,6 @@ from .interfaces import Dialect from .. import event from .. import exc -from ..util.typing import Literal from ..util.typing import TupleAny from ..util.typing import Unpack @@ -253,7 +253,7 @@ def before_execute(conn, clauseelement, multiparams, params): the connection, and those passed in to the method itself for the 2.0 style of execution. - .. versionadded: 1.4 + .. versionadded:: 1.4 .. seealso:: @@ -296,7 +296,7 @@ def after_execute( the connection, and those passed in to the method itself for the 2.0 style of execution. - .. versionadded: 1.4 + .. versionadded:: 1.4 :param result: :class:`_engine.CursorResult` generated by the execution. @@ -957,8 +957,6 @@ def do_setinputsizes( :ref:`mssql_pyodbc_setinputsizes` - .. versionadded:: 1.2.9 - .. seealso:: :ref:`cx_oracle_setinputsizes` diff --git a/lib/sqlalchemy/engine/interfaces.py b/lib/sqlalchemy/engine/interfaces.py index 35c52ae3b94..986d39d40f8 100644 --- a/lib/sqlalchemy/engine/interfaces.py +++ b/lib/sqlalchemy/engine/interfaces.py @@ -1,5 +1,5 @@ # engine/interfaces.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -10,7 +10,6 @@ from __future__ import annotations from enum import Enum -from types import ModuleType from typing import Any from typing import Awaitable from typing import Callable @@ -20,6 +19,7 @@ from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import Mapping from typing import MutableMapping from typing import Optional @@ -36,14 +36,13 @@ from .. import util from ..event import EventTarget from ..pool import Pool -from ..pool import PoolProxiedConnection +from ..pool import PoolProxiedConnection as PoolProxiedConnection from ..sql.compiler import Compiled as Compiled from ..sql.compiler import Compiled # noqa from ..sql.compiler import TypeCompiler as TypeCompiler from ..sql.compiler import TypeCompiler # noqa from ..util import immutabledict from ..util.concurrency import await_ -from ..util.typing import Literal from ..util.typing import NotRequired if TYPE_CHECKING: @@ -51,11 +50,13 @@ from .base import Engine from .cursor import CursorResult from .url import URL + from ..connectors.asyncio import AsyncIODBAPIConnection from ..event import _ListenerFnType from ..event import dispatcher from ..exc import StatementError from ..sql import Executable from ..sql.compiler import _InsertManyValuesBatch + from ..sql.compiler import AggregateOrderByStyle from ..sql.compiler import DDLCompiler from ..sql.compiler import IdentifierPreparer from ..sql.compiler import InsertmanyvaluesSentinelOpts @@ -67,9 +68,11 @@ from ..sql.schema import DefaultGenerator from ..sql.schema import SchemaItem from ..sql.schema import Sequence as Sequence_SchemaItem + from ..sql.sqltypes import _JSON_VALUE from ..sql.sqltypes import Integer from ..sql.type_api import _TypeMemoDict from ..sql.type_api import TypeEngine + from ..util.langhelpers import generic_fn_descriptor ConnectArgsType = Tuple[Sequence[str], MutableMapping[str, Any]] @@ -106,6 +109,22 @@ class ExecuteStyle(Enum): """ +class DBAPIModule(Protocol): + class Error(Exception): + def __getattr__(self, key: str) -> Any: ... + + class OperationalError(Error): + pass + + class InterfaceError(Error): + pass + + class IntegrityError(Error): + pass + + def __getattr__(self, key: str) -> Any: ... + + class DBAPIConnection(Protocol): """protocol representing a :pep:`249` database connection. @@ -122,11 +141,13 @@ def close(self) -> None: ... def commit(self) -> None: ... - def cursor(self) -> DBAPICursor: ... + def cursor(self, *args: Any, **kwargs: Any) -> DBAPICursor: ... def rollback(self) -> None: ... - autocommit: bool + def __getattr__(self, key: str) -> Any: ... + + def __setattr__(self, key: str, value: Any) -> None: ... class DBAPIType(Protocol): @@ -386,8 +407,6 @@ class ReflectedColumn(TypedDict): computed: NotRequired[ReflectedComputed] """indicates that this column is computed by the database. Only some dialects return this key. - - .. versionadded:: 1.3.16 - added support for computed reflection. """ identity: NotRequired[ReflectedIdentity] @@ -430,8 +449,6 @@ class ReflectedCheckConstraint(ReflectedConstraint): dialect_options: NotRequired[Dict[str, Any]] """Additional dialect-specific options detected for this check constraint - - .. versionadded:: 1.3.8 """ @@ -540,8 +557,6 @@ class ReflectedIndex(TypedDict): """optional dict mapping column names or expressions to tuple of sort keywords, which may include ``asc``, ``desc``, ``nulls_first``, ``nulls_last``. - - .. versionadded:: 1.3.5 """ dialect_options: NotRequired[Dict[str, Any]] @@ -659,7 +674,7 @@ class Dialect(EventTarget): dialect_description: str - dbapi: Optional[ModuleType] + dbapi: Optional[DBAPIModule] """A reference to the DBAPI module object itself. SQLAlchemy dialects import DBAPI modules using the classmethod @@ -683,7 +698,7 @@ class Dialect(EventTarget): """ @util.non_memoized_property - def loaded_dbapi(self) -> ModuleType: + def loaded_dbapi(self) -> DBAPIModule: """same as .dbapi, but is never None; will raise an error if no DBAPI was set up. @@ -761,6 +776,14 @@ def loaded_dbapi(self) -> ModuleType: default_isolation_level: Optional[IsolationLevel] """the isolation that is implicitly present on new connections""" + skip_autocommit_rollback: bool + """Whether or not the :paramref:`.create_engine.skip_autocommit_rollback` + parameter was set. + + .. versionadded:: 2.0.43 + + """ + # create_engine() -> isolation_level currently goes here _on_connect_isolation_level: Optional[IsolationLevel] @@ -780,8 +803,14 @@ def loaded_dbapi(self) -> ModuleType: max_identifier_length: int """The maximum length of identifier names.""" - - supports_server_side_cursors: bool + max_index_name_length: Optional[int] + """The maximum length of index names if different from + ``max_identifier_length``.""" + max_constraint_name_length: Optional[int] + """The maximum length of constraint names if different from + ``max_identifier_length``.""" + + supports_server_side_cursors: Union[generic_fn_descriptor[bool], bool] """indicates if the dialect supports server side cursors""" server_side_cursors: bool @@ -837,6 +866,42 @@ def loaded_dbapi(self) -> ModuleType: """ + _json_serializer: Callable[[_JSON_VALUE], str] | None + + _json_deserializer: Callable[[str], _JSON_VALUE] | None + + supports_native_json_serialization: bool + """target dialect includes a native JSON serializer, eliminating + the need to use json.dumps() for JSON data + + .. versionadded:: 2.1 + + """ + + supports_native_json_deserialization: bool + """target dialect includes a native JSON deserializer, eliminating + the need to use json.loads() for JSON data + + .. versionadded:: 2.1 + + """ + + dialect_injects_custom_json_deserializer: bool + """target dialect, when given a custom _json_deserializer, needs to + inject this handler at the connection/cursor level, rather than + having JSON data returned as a string to be handled by the type + + ..versionadded:: 2.1 + + """ + + aggregate_order_by_style: AggregateOrderByStyle + """Style of ORDER BY supported for arbitrary aggregate functions + + .. versionadded:: 2.1 + + """ + insert_executemany_returning: bool """dialect / driver / database supports some means of providing INSERT...RETURNING support when dialect.do_executemany() is used. @@ -1066,7 +1131,7 @@ def loaded_dbapi(self) -> ModuleType: If the above construct is established on the PostgreSQL dialect, the :class:`.Index` construct will now accept the keyword arguments - ``postgresql_using``, ``postgresql_where``, nad ``postgresql_ops``. + ``postgresql_using``, ``postgresql_where``, and ``postgresql_ops``. Any other argument specified to the constructor of :class:`.Index` which is prefixed with ``postgresql_`` will raise :class:`.ArgumentError`. @@ -1193,6 +1258,13 @@ def loaded_dbapi(self) -> ModuleType: tuple_in_values: bool """target database supports tuple IN, i.e. (x, y) IN ((q, p), (r, z))""" + requires_name_normalize: bool + """Indicates symbol names are returned by the database in + UPPERCASED if they are case insensitive within the database. + If this is True, the methods normalize_name() + and denormalize_name() must be provided. + """ + _bind_typing_render_casts: bool _type_memos: MutableMapping[TypeEngine[Any], _TypeMemoDict] @@ -1234,7 +1306,7 @@ def create_connect_args(self, url): raise NotImplementedError() @classmethod - def import_dbapi(cls) -> ModuleType: + def import_dbapi(cls) -> DBAPIModule: """Import the DBAPI module that is used by this dialect. The Python module object returned here will be assigned as an @@ -1283,8 +1355,6 @@ def initialize(self, connection: Connection) -> None: """ - pass - if TYPE_CHECKING: def _overrides_default(self, method_name: str) -> bool: ... @@ -1750,8 +1820,6 @@ def get_table_comment( :raise: ``NotImplementedError`` for dialects that don't support comments. - .. versionadded:: 1.2 - """ raise NotImplementedError() @@ -2206,7 +2274,7 @@ def do_execute_no_params( def is_disconnect( self, - e: Exception, + e: DBAPIModule.Error, connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]], cursor: Optional[DBAPICursor], ) -> bool: @@ -2310,7 +2378,7 @@ def do_on_connect(connection): """ return self.on_connect() - def on_connect(self) -> Optional[Callable[[Any], Any]]: + def on_connect(self) -> Optional[Callable[[Any], None]]: """return a callable which sets up a newly created DBAPI connection. The callable should accept a single argument "conn" which is the @@ -2459,6 +2527,30 @@ def get_isolation_level( raise NotImplementedError() + def detect_autocommit_setting(self, dbapi_conn: DBAPIConnection) -> bool: + """Detect the current autocommit setting for a DBAPI connection. + + :param dbapi_connection: a DBAPI connection object + :return: True if autocommit is enabled, False if disabled + :rtype: bool + + This method inspects the given DBAPI connection to determine + whether autocommit mode is currently enabled. The specific + mechanism for detecting autocommit varies by database dialect + and DBAPI driver, however it should be done **without** network + round trips. + + .. note:: + + Not all dialects support autocommit detection. Dialects + that do not support this feature will raise + :exc:`NotImplementedError`. + + """ + raise NotImplementedError( + "This dialect cannot detect autocommit on a DBAPI connection" + ) + def get_default_isolation_level( self, dbapi_conn: DBAPIConnection ) -> IsolationLevel: @@ -2476,14 +2568,12 @@ def get_default_isolation_level( The method defaults to using the :meth:`.Dialect.get_isolation_level` method unless overridden by a dialect. - .. versionadded:: 1.3.22 - """ raise NotImplementedError() def get_isolation_level_values( self, dbapi_conn: DBAPIConnection - ) -> List[IsolationLevel]: + ) -> Sequence[IsolationLevel]: """return a sequence of string isolation level names that are accepted by this dialect. @@ -2588,8 +2678,6 @@ def load_provisioning(cls): except ImportError: pass - .. versionadded:: 1.3.14 - """ @classmethod @@ -2657,6 +2745,9 @@ def get_dialect_pool_class(self, url: URL) -> Type[Pool]: """return a Pool class to use for a given URL""" raise NotImplementedError() + def validate_identifier(self, ident: str) -> None: + """Validates an identifier name, raising an exception if invalid""" + class CreateEnginePlugin: """A set of hooks intended to augment the construction of an @@ -2748,9 +2839,6 @@ def _log_event( "mysql+pymysql://scott:tiger@localhost/test", plugins=["myplugin"] ) - .. versionadded:: 1.2.3 plugin names can also be specified - to :func:`_sa.create_engine` as a list - A plugin may consume plugin-specific arguments from the :class:`_engine.URL` object as well as the ``kwargs`` dictionary, which is the dictionary of arguments passed to the :func:`_sa.create_engine` @@ -3364,7 +3452,7 @@ class AdaptedConnection: __slots__ = ("_connection",) - _connection: Any + _connection: AsyncIODBAPIConnection @property def driver_connection(self) -> Any: diff --git a/lib/sqlalchemy/engine/mock.py b/lib/sqlalchemy/engine/mock.py index 08dba5a6456..a5173b9d7d0 100644 --- a/lib/sqlalchemy/engine/mock.py +++ b/lib/sqlalchemy/engine/mock.py @@ -1,5 +1,5 @@ # engine/mock.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -27,10 +27,9 @@ from .interfaces import Dialect from .url import URL from ..sql.base import Executable - from ..sql.ddl import SchemaDropper - from ..sql.ddl import SchemaGenerator + from ..sql.ddl import InvokeDDLBase from ..sql.schema import HasSchemaAttr - from ..sql.schema import SchemaItem + from ..sql.visitors import Visitable class MockConnection: @@ -53,12 +52,14 @@ def execution_options(self, **kw: Any) -> MockConnection: def _run_ddl_visitor( self, - visitorcallable: Type[Union[SchemaGenerator, SchemaDropper]], - element: SchemaItem, + visitorcallable: Type[InvokeDDLBase], + element: Visitable, **kwargs: Any, ) -> None: kwargs["checkfirst"] = False - visitorcallable(self.dialect, self, **kwargs).traverse_single(element) + visitorcallable( + dialect=self.dialect, connection=self, **kwargs + ).traverse_single(element) def execute( self, diff --git a/lib/sqlalchemy/engine/processors.py b/lib/sqlalchemy/engine/processors.py index 32f0de4c6b8..19ab95d3210 100644 --- a/lib/sqlalchemy/engine/processors.py +++ b/lib/sqlalchemy/engine/processors.py @@ -1,5 +1,5 @@ # engine/processors.py -# Copyright (C) 2010-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2010-2026 the SQLAlchemy authors and contributors # # Copyright (C) 2010 Gaetan de Menten gdementen@gmail.com # diff --git a/lib/sqlalchemy/engine/reflection.py b/lib/sqlalchemy/engine/reflection.py index e284cb4009d..21ea03463e7 100644 --- a/lib/sqlalchemy/engine/reflection.py +++ b/lib/sqlalchemy/engine/reflection.py @@ -1,5 +1,5 @@ # engine/reflection.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -1316,8 +1316,6 @@ def get_table_comment( :return: a dictionary, with the table comment. - .. versionadded:: 1.2 - .. seealso:: :meth:`Inspector.get_multi_table_comment` """ @@ -1714,9 +1712,12 @@ def _reflect_pk( if pk in cols_by_orig_name and pk not in exclude_columns ] - # update pk constraint name and comment + # update pk constraint name, comment and dialect_kwargs table.primary_key.name = pk_cons.get("name") table.primary_key.comment = pk_cons.get("comment", None) + dialect_options = pk_cons.get("dialect_options") + if dialect_options: + table.primary_key.dialect_kwargs.update(dialect_options) # tell the PKConstraint to re-initialize # its column collection diff --git a/lib/sqlalchemy/engine/result.py b/lib/sqlalchemy/engine/result.py index d550d8c4416..471e2e4c65e 100644 --- a/lib/sqlalchemy/engine/result.py +++ b/lib/sqlalchemy/engine/result.py @@ -1,5 +1,5 @@ # engine/result.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -9,7 +9,6 @@ from __future__ import annotations -from enum import Enum import functools import itertools import operator @@ -22,39 +21,55 @@ from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import Mapping from typing import NoReturn from typing import Optional from typing import overload from typing import Sequence -from typing import Set from typing import Tuple from typing import TYPE_CHECKING -from typing import TypeVar from typing import Union +from ._result_cy import _InterimRowType +from ._result_cy import _NO_ROW as _NO_ROW +from ._result_cy import _R as _R +from ._result_cy import _RowData +from ._result_cy import _T +from ._result_cy import _UniqueFilterType as _UniqueFilterType +from ._result_cy import BaseResultInternal from ._util_cy import tuplegetter as tuplegetter from .row import Row from .row import RowMapping from .. import exc from .. import util from ..sql.base import _generative -from ..sql.base import HasMemoized from ..sql.base import InPlaceGenerative from ..util import deprecated -from ..util import HasMemoized_ro_memoized_attribute from ..util import NONE_SET -from ..util.typing import Literal +from ..util.typing import Never from ..util.typing import Self from ..util.typing import TupleAny from ..util.typing import TypeVarTuple from ..util.typing import Unpack if typing.TYPE_CHECKING: + from typing import Type + + from .. import inspection + from ..sql import roles + from ..sql._typing import _HasClauseElement from ..sql.elements import SQLCoreOperations from ..sql.type_api import _ResultProcessorType -_KeyType = Union[str, "SQLCoreOperations[Any]"] +_KeyType = Union[ + str, + "SQLCoreOperations[Any]", + "roles.TypedColumnsClauseRole[Any]", + "roles.ColumnsClauseRole", + "Type[Any]", + "inspection.Inspectable[_HasClauseElement[Any]]", +] _KeyIndexType = Union[_KeyType, int] # is overridden in cursor using _CursorKeyMapRecType @@ -63,28 +78,13 @@ _KeyMapType = Mapping[_KeyType, _KeyMapRecType] -_RowData = Union[Row[Unpack[TupleAny]], RowMapping, Any] -"""A generic form of "row" that accommodates for the different kinds of -"rows" that different result objects return, including row, row mapping, and -scalar values""" - - -_R = TypeVar("_R", bound=_RowData) -_T = TypeVar("_T", bound=Any) _Ts = TypeVarTuple("_Ts") -_InterimRowType = Union[_R, TupleAny] -"""a catchall "anything" kind of return type that can be applied -across all the result types - -""" _InterimSupportsScalarsRowType = Union[Row[Unpack[TupleAny]], Any] _ProcessorsType = Sequence[Optional["_ResultProcessorType[Any]"]] _TupleGetterType = Callable[[Sequence[Any]], Sequence[Any]] -_UniqueFilterType = Callable[[Any], Any] -_UniqueFilterStateType = Tuple[Set[Any], Optional[_UniqueFilterType]] class ResultMetaData: @@ -325,7 +325,7 @@ def __setstate__(self, state: Dict[str, Any]) -> None: ) def _index_for_key(self, key: Any, raiseerr: bool = True) -> int: - if int in key.__class__.__mro__: + if isinstance(key, int): key = self._keys[key] try: rec = self._keymap[key] @@ -341,7 +341,7 @@ def _metadata_for_keys( self, keys: Sequence[Any] ) -> Iterator[_KeyMapRecType]: for key in keys: - if int in key.__class__.__mro__: + if isinstance(key, int): key = self._keys[key] try: @@ -354,9 +354,7 @@ def _metadata_for_keys( def _reduce(self, keys: Sequence[Any]) -> ResultMetaData: try: metadata_for_keys = [ - self._keymap[ - self._keys[key] if int in key.__class__.__mro__ else key - ] + self._keymap[self._keys[key] if isinstance(key, int) else key] for key in keys ] except KeyError as ke: @@ -393,449 +391,10 @@ def result_tuple( ) -# a symbol that indicates to internal Result methods that -# "no row is returned". We can't use None for those cases where a scalar -# filter is applied to rows. -class _NoRow(Enum): - _NO_ROW = 0 - - -_NO_ROW = _NoRow._NO_ROW - - -class ResultInternal(InPlaceGenerative, Generic[_R]): +class ResultInternal(InPlaceGenerative, BaseResultInternal[_R]): __slots__ = () - - _real_result: Optional[Result[Unpack[TupleAny]]] = None - _generate_rows: bool = True - _row_logging_fn: Optional[Callable[[Any], Any]] - - _unique_filter_state: Optional[_UniqueFilterStateType] = None - _post_creational_filter: Optional[Callable[[Any], Any]] = None _is_cursor = False - _metadata: ResultMetaData - - _source_supports_scalars: bool - - def _fetchiter_impl( - self, - ) -> Iterator[_InterimRowType[Row[Unpack[TupleAny]]]]: - raise NotImplementedError() - - def _fetchone_impl( - self, hard_close: bool = False - ) -> Optional[_InterimRowType[Row[Unpack[TupleAny]]]]: - raise NotImplementedError() - - def _fetchmany_impl( - self, size: Optional[int] = None - ) -> List[_InterimRowType[Row[Unpack[TupleAny]]]]: - raise NotImplementedError() - - def _fetchall_impl( - self, - ) -> List[_InterimRowType[Row[Unpack[TupleAny]]]]: - raise NotImplementedError() - - def _soft_close(self, hard: bool = False) -> None: - raise NotImplementedError() - - @HasMemoized_ro_memoized_attribute - def _row_getter(self) -> Optional[Callable[..., _R]]: - real_result: Result[Unpack[TupleAny]] = ( - self._real_result - if self._real_result - else cast("Result[Unpack[TupleAny]]", self) - ) - - if real_result._source_supports_scalars: - if not self._generate_rows: - return None - else: - _proc = Row - - def process_row( - metadata: ResultMetaData, - processors: Optional[_ProcessorsType], - key_to_index: Dict[_KeyType, int], - scalar_obj: Any, - ) -> Row[Unpack[TupleAny]]: - return _proc( - metadata, processors, key_to_index, (scalar_obj,) - ) - - else: - process_row = Row # type: ignore - - metadata = self._metadata - - key_to_index = metadata._key_to_index - processors = metadata._effective_processors - tf = metadata._tuplefilter - - if tf and not real_result._source_supports_scalars: - if processors: - processors = tf(processors) - - _make_row_orig: Callable[..., _R] = functools.partial( # type: ignore # noqa E501 - process_row, metadata, processors, key_to_index - ) - - fixed_tf = tf - - def make_row(row: _InterimRowType[Row[Unpack[TupleAny]]]) -> _R: - return _make_row_orig(fixed_tf(row)) - - else: - make_row = functools.partial( # type: ignore - process_row, metadata, processors, key_to_index - ) - - if real_result._row_logging_fn: - _log_row = real_result._row_logging_fn - _make_row = make_row - - def make_row(row: _InterimRowType[Row[Unpack[TupleAny]]]) -> _R: - return _log_row(_make_row(row)) # type: ignore - - return make_row - - @HasMemoized_ro_memoized_attribute - def _iterator_getter(self) -> Callable[..., Iterator[_R]]: - make_row = self._row_getter - - post_creational_filter = self._post_creational_filter - - if self._unique_filter_state: - uniques, strategy = self._unique_strategy - - def iterrows(self: Result[Unpack[TupleAny]]) -> Iterator[_R]: - for raw_row in self._fetchiter_impl(): - obj: _InterimRowType[Any] = ( - make_row(raw_row) if make_row else raw_row - ) - hashed = strategy(obj) if strategy else obj - if hashed in uniques: - continue - uniques.add(hashed) - if post_creational_filter: - obj = post_creational_filter(obj) - yield obj # type: ignore - - else: - - def iterrows(self: Result[Unpack[TupleAny]]) -> Iterator[_R]: - for raw_row in self._fetchiter_impl(): - row: _InterimRowType[Any] = ( - make_row(raw_row) if make_row else raw_row - ) - if post_creational_filter: - row = post_creational_filter(row) - yield row # type: ignore - - return iterrows - - def _raw_all_rows(self) -> List[_R]: - make_row = self._row_getter - assert make_row is not None - rows = self._fetchall_impl() - return [make_row(row) for row in rows] - - def _allrows(self) -> List[_R]: - post_creational_filter = self._post_creational_filter - - make_row = self._row_getter - - rows = self._fetchall_impl() - made_rows: List[_InterimRowType[_R]] - if make_row: - made_rows = [make_row(row) for row in rows] - else: - made_rows = rows # type: ignore - - interim_rows: List[_R] - - if self._unique_filter_state: - uniques, strategy = self._unique_strategy - - interim_rows = [ - made_row # type: ignore - for made_row, sig_row in [ - ( - made_row, - strategy(made_row) if strategy else made_row, - ) - for made_row in made_rows - ] - if sig_row not in uniques and not uniques.add(sig_row) # type: ignore # noqa: E501 - ] - else: - interim_rows = made_rows # type: ignore - - if post_creational_filter: - interim_rows = [ - post_creational_filter(row) for row in interim_rows - ] - return interim_rows - - @HasMemoized_ro_memoized_attribute - def _onerow_getter( - self, - ) -> Callable[..., Union[Literal[_NoRow._NO_ROW], _R]]: - make_row = self._row_getter - - post_creational_filter = self._post_creational_filter - - if self._unique_filter_state: - uniques, strategy = self._unique_strategy - - def onerow(self: Result[Unpack[TupleAny]]) -> Union[_NoRow, _R]: - _onerow = self._fetchone_impl - while True: - row = _onerow() - if row is None: - return _NO_ROW - else: - obj: _InterimRowType[Any] = ( - make_row(row) if make_row else row - ) - hashed = strategy(obj) if strategy else obj - if hashed in uniques: - continue - else: - uniques.add(hashed) - if post_creational_filter: - obj = post_creational_filter(obj) - return obj # type: ignore - - else: - - def onerow(self: Result[Unpack[TupleAny]]) -> Union[_NoRow, _R]: - row = self._fetchone_impl() - if row is None: - return _NO_ROW - else: - interim_row: _InterimRowType[Any] = ( - make_row(row) if make_row else row - ) - if post_creational_filter: - interim_row = post_creational_filter(interim_row) - return interim_row # type: ignore - - return onerow - - @HasMemoized_ro_memoized_attribute - def _manyrow_getter(self) -> Callable[..., List[_R]]: - make_row = self._row_getter - - post_creational_filter = self._post_creational_filter - - if self._unique_filter_state: - uniques, strategy = self._unique_strategy - - def filterrows( - make_row: Optional[Callable[..., _R]], - rows: List[Any], - strategy: Optional[Callable[[List[Any]], Any]], - uniques: Set[Any], - ) -> List[_R]: - if make_row: - rows = [make_row(row) for row in rows] - - if strategy: - made_rows = ( - (made_row, strategy(made_row)) for made_row in rows - ) - else: - made_rows = ((made_row, made_row) for made_row in rows) - return [ - made_row - for made_row, sig_row in made_rows - if sig_row not in uniques and not uniques.add(sig_row) # type: ignore # noqa: E501 - ] - - def manyrows( - self: ResultInternal[_R], num: Optional[int] - ) -> List[_R]: - collect: List[_R] = [] - - _manyrows = self._fetchmany_impl - - if num is None: - # if None is passed, we don't know the default - # manyrows number, DBAPI has this as cursor.arraysize - # different DBAPIs / fetch strategies may be different. - # do a fetch to find what the number is. if there are - # only fewer rows left, then it doesn't matter. - real_result = ( - self._real_result - if self._real_result - else cast("Result[Unpack[TupleAny]]", self) - ) - if real_result._yield_per: - num_required = num = real_result._yield_per - else: - rows = _manyrows(num) - num = len(rows) - assert make_row is not None - collect.extend( - filterrows(make_row, rows, strategy, uniques) - ) - num_required = num - len(collect) - else: - num_required = num - - assert num is not None - - while num_required: - rows = _manyrows(num_required) - if not rows: - break - - collect.extend( - filterrows(make_row, rows, strategy, uniques) - ) - num_required = num - len(collect) - - if post_creational_filter: - collect = [post_creational_filter(row) for row in collect] - return collect - - else: - - def manyrows( - self: ResultInternal[_R], num: Optional[int] - ) -> List[_R]: - if num is None: - real_result = ( - self._real_result - if self._real_result - else cast("Result[Unpack[TupleAny]]", self) - ) - num = real_result._yield_per - - rows: List[_InterimRowType[Any]] = self._fetchmany_impl(num) - if make_row: - rows = [make_row(row) for row in rows] - if post_creational_filter: - rows = [post_creational_filter(row) for row in rows] - return rows # type: ignore - - return manyrows - - @overload - def _only_one_row( - self, - raise_for_second_row: bool, - raise_for_none: Literal[True], - scalar: bool, - ) -> _R: ... - - @overload - def _only_one_row( - self, - raise_for_second_row: bool, - raise_for_none: bool, - scalar: bool, - ) -> Optional[_R]: ... - - def _only_one_row( - self, - raise_for_second_row: bool, - raise_for_none: bool, - scalar: bool, - ) -> Optional[_R]: - onerow = self._fetchone_impl - - row: Optional[_InterimRowType[Any]] = onerow(hard_close=True) - if row is None: - if raise_for_none: - raise exc.NoResultFound( - "No row was found when one was required" - ) - else: - return None - - if scalar and self._source_supports_scalars: - self._generate_rows = False - make_row = None - else: - make_row = self._row_getter - - try: - row = make_row(row) if make_row else row - except: - self._soft_close(hard=True) - raise - - if raise_for_second_row: - if self._unique_filter_state: - # for no second row but uniqueness, need to essentially - # consume the entire result :( - uniques, strategy = self._unique_strategy - - existing_row_hash = strategy(row) if strategy else row - - while True: - next_row: Any = onerow(hard_close=True) - if next_row is None: - next_row = _NO_ROW - break - - try: - next_row = make_row(next_row) if make_row else next_row - - if strategy: - assert next_row is not _NO_ROW - if existing_row_hash == strategy(next_row): - continue - elif row == next_row: - continue - # here, we have a row and it's different - break - except: - self._soft_close(hard=True) - raise - else: - next_row = onerow(hard_close=True) - if next_row is None: - next_row = _NO_ROW - - if next_row is not _NO_ROW: - self._soft_close(hard=True) - raise exc.MultipleResultsFound( - "Multiple rows were found when exactly one was required" - if raise_for_none - else "Multiple rows were found when one or none " - "was required" - ) - else: - next_row = _NO_ROW - # if we checked for second row then that would have - # closed us :) - self._soft_close(hard=True) - - if not scalar: - post_creational_filter = self._post_creational_filter - if post_creational_filter: - row = post_creational_filter(row) - - if scalar and make_row: - return row[0] # type: ignore - else: - return row # type: ignore - - def _iter_impl(self) -> Iterator[_R]: - return self._iterator_getter(self) - - def _next_impl(self) -> _R: - row = self._onerow_getter(self) - if row is _NO_ROW: - raise StopIteration() - else: - return row - @_generative def _column_slices(self, indexes: Sequence[_KeyIndexType]) -> Self: real_result = ( @@ -851,31 +410,6 @@ def _column_slices(self, indexes: Sequence[_KeyIndexType]) -> Self: return self - @HasMemoized.memoized_attribute - def _unique_strategy(self) -> _UniqueFilterStateType: - assert self._unique_filter_state is not None - uniques, strategy = self._unique_filter_state - - real_result = ( - self._real_result - if self._real_result is not None - else cast("Result[Unpack[TupleAny]]", self) - ) - - if not strategy and self._metadata._unique_filters: - if ( - real_result._source_supports_scalars - and not self._generate_rows - ): - strategy = self._metadata._unique_filters[0] - else: - filters = self._metadata._unique_filters - if self._metadata._tuplefilter: - filters = self._metadata._tuplefilter(filters) - - strategy = operator.methodcaller("_filter_on_values", filters) - return uniques, strategy - class _WithKeys: __slots__ = () @@ -951,7 +485,7 @@ def __exit__(self, type_: Any, value: Any, traceback: Any) -> None: self.close() def close(self) -> None: - """close this :class:`_engine.Result`. + """Hard close this :class:`_engine.Result`. The behavior of this method is implementation specific, and is not implemented by default. The method should generally end @@ -978,9 +512,19 @@ def _soft_closed(self) -> bool: @property def closed(self) -> bool: - """return ``True`` if this :class:`_engine.Result` reports .closed + """Return ``True`` if this :class:`_engine.Result` was **hard closed** + by explicitly calling the :meth:`close` method. - .. versionadded:: 1.4.43 + The attribute is **not** True if the :class:`_engine.Result` was only + **soft closed**; a "soft close" is the style of close that takes place + for example when the :class:`.CursorResult` is returned for a DML + only statement without RETURNING, or when all result rows are fetched. + + .. seealso:: + + :attr:`.CursorResult.returns_rows` - attribute specific to + :class:`.CursorResult` which indicates if the result is one that + may return zero or more rows """ raise NotImplementedError() @@ -1464,13 +1008,7 @@ def one_or_none(self) -> Optional[Row[Unpack[_Ts]]]: raise_for_second_row=True, raise_for_none=False, scalar=False ) - @overload - def scalar_one(self: Result[_T]) -> _T: ... - - @overload - def scalar_one(self) -> Any: ... - - def scalar_one(self) -> Any: + def scalar_one(self: Result[_T, Unpack[TupleAny]]) -> _T: """Return exactly one scalar result or raise an exception. This is equivalent to calling :meth:`_engine.Result.scalars` and @@ -1487,13 +1025,7 @@ def scalar_one(self) -> Any: raise_for_second_row=True, raise_for_none=True, scalar=True ) - @overload - def scalar_one_or_none(self: Result[_T]) -> Optional[_T]: ... - - @overload - def scalar_one_or_none(self) -> Optional[Any]: ... - - def scalar_one_or_none(self) -> Optional[Any]: + def scalar_one_or_none(self: Result[_T, Unpack[TupleAny]]) -> Optional[_T]: """Return exactly one scalar result or ``None``. This is equivalent to calling :meth:`_engine.Result.scalars` and @@ -1513,8 +1045,8 @@ def scalar_one_or_none(self) -> Optional[Any]: def one(self) -> Row[Unpack[_Ts]]: """Return exactly one row or raise an exception. - Raises :class:`.NoResultFound` if the result returns no - rows, or :class:`.MultipleResultsFound` if multiple rows + Raises :class:`_exc.NoResultFound` if the result returns no + rows, or :class:`_exc.MultipleResultsFound` if multiple rows would be returned. .. note:: This method returns one **row**, e.g. tuple, by default. @@ -1543,13 +1075,17 @@ def one(self) -> Row[Unpack[_Ts]]: raise_for_second_row=True, raise_for_none=True, scalar=False ) + # special case to handle mypy issue: + # https://github.com/python/mypy/issues/20651 @overload - def scalar(self: Result[_T]) -> Optional[_T]: ... + def scalar(self: Result[Never, Unpack[TupleAny]]) -> Optional[Any]: + pass @overload - def scalar(self) -> Any: ... + def scalar(self: Result[_T, Unpack[TupleAny]]) -> Optional[_T]: + pass - def scalar(self) -> Any: + def scalar(self: Result[_T, Unpack[TupleAny]]) -> Optional[_T]: """Fetch the first column of the first row, and close the result set. Returns ``None`` if there are no rows to fetch. @@ -1668,10 +1204,13 @@ def _soft_closed(self) -> bool: @property def closed(self) -> bool: - """Return ``True`` if the underlying :class:`_engine.Result` reports - closed + """Return ``True`` if this :class:`_engine.Result` being + proxied by this :class:`_engine.FilterResult` was + **hard closed** by explicitly calling the :meth:`_engine.Result.close` + method. - .. versionadded:: 1.4.43 + This is a direct proxy for the :attr:`_engine.Result.closed` attribute; + see that attribute for details. """ return self._real_result.closed @@ -2038,7 +1577,7 @@ def unique(self, strategy: Optional[_UniqueFilterType] = None) -> Self: return self def columns(self, *col_expressions: _KeyIndexType) -> Self: - r"""Establish the columns that should be returned in each row.""" + """Establish the columns that should be returned in each row.""" return self._column_slices(col_expressions) def partitions( @@ -2198,7 +1737,8 @@ def __init__(self, result: Result[Unpack[_Ts]]): else: self.data = result.fetchall() - def rewrite_rows(self) -> Sequence[Sequence[Any]]: + def _rewrite_rows(self) -> Sequence[Sequence[Any]]: + # used only by the orm fn merge_frozen_result if self._source_supports_scalars: return [[elem] for elem in self.data] else: @@ -2252,10 +1792,20 @@ def __init__( @property def closed(self) -> bool: - """Return ``True`` if this :class:`_engine.IteratorResult` has - been closed + """Return ``True`` if this :class:`_engine.IteratorResult` was + **hard closed** by explicitly calling the :meth:`_engine.Result.close` + method. - .. versionadded:: 1.4.43 + The attribute is **not** True if the :class:`_engine.Result` was only + **soft closed**; a "soft close" is the style of close that takes place + for example when the :class:`.CursorResult` is returned for a DML + only statement without RETURNING, or when all result rows are fetched. + + .. seealso:: + + :attr:`.CursorResult.returns_rows` - attribute specific to + :class:`.CursorResult` which indicates if the result is one that + may return zero or more rows """ return self._hard_closed diff --git a/lib/sqlalchemy/engine/row.py b/lib/sqlalchemy/engine/row.py index 6c5db5b49d8..fe21ae7efde 100644 --- a/lib/sqlalchemy/engine/row.py +++ b/lib/sqlalchemy/engine/row.py @@ -1,5 +1,5 @@ # engine/row.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -18,9 +18,7 @@ from typing import Dict from typing import Generic from typing import Iterator -from typing import List from typing import Mapping -from typing import NoReturn from typing import Optional from typing import Sequence from typing import Tuple @@ -45,7 +43,7 @@ _Ts = TypeVarTuple("_Ts") -class Row(BaseRow, _RowBase[Unpack[_Ts]], Generic[Unpack[_Ts]]): +class Row(BaseRow, _RowBase[Unpack[_Ts]], Generic[Unpack[_Ts]]): # type: ignore[misc] # noqa: E501 """Represent a single result row. The :class:`.Row` object represents a row of a database result. It is @@ -75,12 +73,6 @@ class Row(BaseRow, _RowBase[Unpack[_Ts]], Generic[Unpack[_Ts]]): __slots__ = () - def __setattr__(self, name: str, value: Any) -> NoReturn: - raise AttributeError("can't set attribute") - - def __delattr__(self, name: str) -> NoReturn: - raise AttributeError("can't delete attribute") - @deprecated( "2.1.0", "The :meth:`.Row._tuple` method is deprecated, :class:`.Row` " @@ -222,9 +214,6 @@ def meth(*arg: Any, **kw: Any) -> Any: count = _special_name_accessor("count") index = _special_name_accessor("index") - def __contains__(self, key: Any) -> bool: - return key in self._data - def _op(self, other: Any, op: Callable[[Any, Any], bool]) -> bool: return ( op(self._to_tuple_instance(), other._to_tuple_instance()) @@ -374,15 +363,9 @@ def __getitem__(self, key: _KeyType) -> Any: ... else: __getitem__ = BaseRow._get_by_key_impl_mapping - def _values_impl(self) -> List[Any]: - return list(self._data) - def __iter__(self) -> Iterator[str]: return (k for k in self._parent.keys if k is not None) - def __len__(self) -> int: - return len(self._data) - def __contains__(self, key: object) -> bool: return self._parent._has_key(key) diff --git a/lib/sqlalchemy/engine/strategies.py b/lib/sqlalchemy/engine/strategies.py index 5dd7bca9a49..a7eb81b43f7 100644 --- a/lib/sqlalchemy/engine/strategies.py +++ b/lib/sqlalchemy/engine/strategies.py @@ -1,14 +1,11 @@ # engine/strategies.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -"""Deprecated mock engine strategy used by Alembic. - - -""" +"""Deprecated mock engine strategy used by Alembic.""" from __future__ import annotations diff --git a/lib/sqlalchemy/engine/url.py b/lib/sqlalchemy/engine/url.py index f72940d4bd3..c3f1986bc7f 100644 --- a/lib/sqlalchemy/engine/url.py +++ b/lib/sqlalchemy/engine/url.py @@ -1,5 +1,5 @@ # engine/url.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -918,5 +918,5 @@ def _parse_url(name: str) -> URL: else: raise exc.ArgumentError( - "Could not parse SQLAlchemy URL from string '%s'" % name + "Could not parse SQLAlchemy URL from given URL string" ) diff --git a/lib/sqlalchemy/engine/util.py b/lib/sqlalchemy/engine/util.py index b8eae80cbc7..b4cd054db8c 100644 --- a/lib/sqlalchemy/engine/util.py +++ b/lib/sqlalchemy/engine/util.py @@ -1,5 +1,5 @@ # engine/util.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/event/__init__.py b/lib/sqlalchemy/event/__init__.py index 309b7bd33fb..a48af7ef5d0 100644 --- a/lib/sqlalchemy/event/__init__.py +++ b/lib/sqlalchemy/event/__init__.py @@ -1,5 +1,5 @@ # event/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -20,6 +20,7 @@ from .base import dispatcher as dispatcher from .base import Events as Events from .legacy import _legacy_signature as _legacy_signature +from .legacy import _omit_standard_example as _omit_standard_example from .registry import _EventKey as _EventKey from .registry import _ListenerFnType as _ListenerFnType from .registry import EventTarget as EventTarget diff --git a/lib/sqlalchemy/event/api.py b/lib/sqlalchemy/event/api.py index b6ec8f6d32b..38b2716b88b 100644 --- a/lib/sqlalchemy/event/api.py +++ b/lib/sqlalchemy/event/api.py @@ -1,13 +1,11 @@ # event/api.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -"""Public API functions for the event system. - -""" +"""Public API functions for the event system.""" from __future__ import annotations from typing import Any diff --git a/lib/sqlalchemy/event/attr.py b/lib/sqlalchemy/event/attr.py index 7e28a00cb92..08cf350d280 100644 --- a/lib/sqlalchemy/event/attr.py +++ b/lib/sqlalchemy/event/attr.py @@ -1,5 +1,5 @@ # event/attr.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -343,11 +343,20 @@ def for_modify( obj = cast("_Dispatch[_ET]", obj) assert obj._instance_cls is not None - result = _ListenerCollection(self.parent, obj._instance_cls) - if getattr(obj, self.name) is self: - setattr(obj, self.name, result) - else: - assert isinstance(getattr(obj, self.name), _JoinedListener) + existing = getattr(obj, self.name) + + with util.mini_gil: + if existing is self or isinstance(existing, _JoinedListener): + result = _ListenerCollection(self.parent, obj._instance_cls) + else: + # this codepath is an extremely rare race condition + # that has been observed in test_pool.py->test_timeout_race + # with freethreaded. + assert isinstance(existing, _ListenerCollection) + return existing + + if existing is self: + setattr(obj, self.name, result) return result def _needs_modify(self, *args: Any, **kw: Any) -> NoReturn: @@ -409,7 +418,7 @@ class _CompoundListener(_InstanceLevelDispatch[_ET]): "_is_asyncio", ) - _exec_once_mutex: _MutexProtocol + _exec_once_mutex: Optional[_MutexProtocol] parent_listeners: Collection[_ListenerFnType] listeners: Collection[_ListenerFnType] _exec_once: bool @@ -422,16 +431,23 @@ def __init__(self, *arg: Any, **kw: Any): def _set_asyncio(self) -> None: self._is_asyncio = True - def _memoized_attr__exec_once_mutex(self) -> _MutexProtocol: - if self._is_asyncio: - return AsyncAdaptedLock() - else: - return threading.Lock() + def _get_exec_once_mutex(self) -> _MutexProtocol: + with util.mini_gil: + if self._exec_once_mutex is not None: + return self._exec_once_mutex + + if self._is_asyncio: + mutex = AsyncAdaptedLock() + else: + mutex = threading.Lock() # type: ignore[assignment] + self._exec_once_mutex = mutex + + return mutex def _exec_once_impl( self, retry_on_exception: bool, *args: Any, **kw: Any ) -> None: - with self._exec_once_mutex: + with self._get_exec_once_mutex(): if not self._exec_once: try: self(*args, **kw) @@ -459,8 +475,6 @@ def exec_once_unless_exception(self, *args: Any, **kw: Any) -> None: If exec_once was already called, then this method will never run the callable regardless of whether it raised or not. - .. versionadded:: 1.3.8 - """ if not self._exec_once: self._exec_once_impl(True, *args, **kw) @@ -472,13 +486,15 @@ def _exec_w_sync_on_first_run(self, *args: Any, **kw: Any) -> None: raised an exception. If _exec_w_sync_on_first_run was already called and didn't raise an - exception, then a mutex is not used. + exception, then a mutex is not used. It's not guaranteed + the mutex won't be used more than once in the case of very rare + race conditions. .. versionadded:: 1.4.11 """ if not self._exec_w_sync_once: - with self._exec_once_mutex: + with self._get_exec_once_mutex(): try: self(*args, **kw) except: @@ -540,6 +556,7 @@ def __init__(self, parent: _ClsLevelDispatch[_ET], target_cls: Type[_ET]): parent.update_subclass(target_cls) self._exec_once = False self._exec_w_sync_once = False + self._exec_once_mutex = None self.parent_listeners = parent._clslevel[target_cls] self.parent = parent self.name = parent.name @@ -617,6 +634,8 @@ def __init__( local: _EmptyListener[_ET], ): self._exec_once = False + self._exec_w_sync_once = False + self._exec_once_mutex = None self.parent_dispatch = parent_dispatch self.name = name self.local = local diff --git a/lib/sqlalchemy/event/base.py b/lib/sqlalchemy/event/base.py index 66dc12996bc..c4601b9d6ba 100644 --- a/lib/sqlalchemy/event/base.py +++ b/lib/sqlalchemy/event/base.py @@ -1,5 +1,5 @@ # event/base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -24,6 +24,7 @@ from typing import Generic from typing import Iterator from typing import List +from typing import Literal from typing import Mapping from typing import MutableMapping from typing import Optional @@ -40,7 +41,6 @@ from .registry import _ET from .registry import _EventKey from .. import util -from ..util.typing import Literal _registrars: MutableMapping[str, List[Type[_HasEventsDispatch[Any]]]] = ( util.defaultdict(list) diff --git a/lib/sqlalchemy/event/legacy.py b/lib/sqlalchemy/event/legacy.py index e60fd9a5e17..63fc1ee9a9a 100644 --- a/lib/sqlalchemy/event/legacy.py +++ b/lib/sqlalchemy/event/legacy.py @@ -1,5 +1,5 @@ # event/legacy.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -18,6 +18,7 @@ from typing import Optional from typing import Tuple from typing import Type +from typing import TypeVar from .registry import _ET from .registry import _ListenerFnType @@ -29,14 +30,16 @@ from .base import _HasEventsDispatch -_LegacySignatureType = Tuple[str, List[str], Optional[Callable[..., Any]]] +_F = TypeVar("_F", bound=Callable[..., Any]) + +_LegacySignatureType = Tuple[str, List[str], Callable[..., Any]] def _legacy_signature( since: str, argnames: List[str], converter: Optional[Callable[..., Any]] = None, -) -> Callable[[Callable[..., Any]], Callable[..., Any]]: +) -> Callable[[_F], _F]: """legacy sig decorator @@ -48,7 +51,7 @@ def _legacy_signature( """ - def leg(fn: Callable[..., Any]) -> Callable[..., Any]: + def leg(fn: _F) -> _F: if not hasattr(fn, "_legacy_signatures"): fn._legacy_signatures = [] # type: ignore[attr-defined] fn._legacy_signatures.append((since, argnames, converter)) # type: ignore[attr-defined] # noqa: E501 @@ -57,6 +60,11 @@ def leg(fn: Callable[..., Any]) -> Callable[..., Any]: return leg +def _omit_standard_example(fn: _F) -> _F: + fn._omit_standard_example = True # type: ignore[attr-defined] + return fn + + def _wrap_fn_for_legacy( dispatch_collection: _ClsLevelDispatch[_ET], fn: _ListenerFnType, @@ -222,6 +230,10 @@ def _augment_fn_docs( parent_dispatch_cls: Type[_HasEventsDispatch[_ET]], fn: _ListenerFnType, ) -> str: + if getattr(fn, "_omit_standard_example", False): + assert fn.__doc__ + return fn.__doc__ + header = ( ".. container:: event_signatures\n\n" " Example argument forms::\n" diff --git a/lib/sqlalchemy/event/registry.py b/lib/sqlalchemy/event/registry.py index d7e4b321553..22be5628e44 100644 --- a/lib/sqlalchemy/event/registry.py +++ b/lib/sqlalchemy/event/registry.py @@ -1,5 +1,5 @@ # event/registry.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/events.py b/lib/sqlalchemy/events.py index ce832439516..501d79ed63c 100644 --- a/lib/sqlalchemy/events.py +++ b/lib/sqlalchemy/events.py @@ -1,5 +1,5 @@ # events.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/exc.py b/lib/sqlalchemy/exc.py index c66124d6c8d..78e041b95aa 100644 --- a/lib/sqlalchemy/exc.py +++ b/lib/sqlalchemy/exc.py @@ -1,5 +1,5 @@ # exc.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -115,6 +115,44 @@ def __str__(self) -> str: return self._sql_message() +class EmulatedDBAPIException(Exception): + """Serves as the base of the DBAPI ``Error`` class for dialects where + a DBAPI exception hierrchy needs to be emulated. + + The current example is the asyncpg dialect. + + .. versionadded:: 2.1 + + """ + + orig: Exception | None + + def __init__(self, message: str, orig: Exception | None = None): + # we accept None for Exception since all DBAPI.Error objects + # need to support construction with a message alone + super().__init__(message) + self.orig = orig + + @property + def driver_exception(self) -> Exception: + """The original driver exception that was raised. + + This exception object will always originate from outside of + SQLAlchemy. + + """ + + if self.orig is None: + raise ValueError( + "No original exception is present. Was this " + "EmulatedDBAPIException constructed without a driver error?" + ) + return self.orig + + def __reduce__(self) -> Any: + return self.__class__, (self.args[0], self.orig) + + class ArgumentError(SQLAlchemyError): """Raised when an invalid or conflicting function argument is supplied. @@ -139,7 +177,7 @@ class ObjectNotExecutableError(ArgumentError): """ def __init__(self, target: Any): - super().__init__("Not an executable object: %r" % target) + super().__init__(f"Not an executable object: {target!r}") self.target = target def __reduce__(self) -> Union[str, Tuple[Any, ...]]: @@ -277,8 +315,6 @@ class InvalidatePoolError(DisconnectionError): :class:`_exc.DisconnectionError`, allowing three attempts to reconnect before giving up. - .. versionadded:: 1.2 - """ invalidate_pool: bool = True @@ -328,6 +364,18 @@ class NoSuchColumnError(InvalidRequestError, KeyError): """A nonexistent column is requested from a ``Row``.""" +class AmbiguousColumnError(InvalidRequestError): + """Raised when a column/attribute name is ambiguous across multiple + entities. + + This can occur when using :meth:`_sql.Select.filter_by` with multiple + joined tables that have columns with the same name. + + .. versionadded:: 2.1 + + """ + + class NoResultFound(InvalidRequestError): """A database result was required but none was found. @@ -412,11 +460,7 @@ class NoSuchTableError(InvalidRequestError): class UnreflectableTableError(InvalidRequestError): - """Table exists but can't be reflected for some reason. - - .. versionadded:: 1.2 - - """ + """Table exists but can't be reflected for some reason.""" class UnboundExecutionError(InvalidRequestError): @@ -469,6 +513,12 @@ class StatementError(SQLAlchemyError): orig: Optional[BaseException] = None """The original exception that was thrown. + .. seealso:: + + :attr:`.DBAPIError.driver_exception` - a more specific attribute that + is guaranteed to return the exception object raised by the third + party driver in use, even when using asyncio. + """ ismulti: Optional[bool] = None @@ -561,6 +611,8 @@ class DBAPIError(StatementError): code = "dbapi" + orig: Optional[Exception] + @overload @classmethod def instance( @@ -718,6 +770,42 @@ def __init__( ) self.connection_invalidated = connection_invalidated + @property + def driver_exception(self) -> Exception: + """The exception object originating from the driver (DBAPI) outside + of SQLAlchemy. + + In the case of some asyncio dialects, special steps are taken to + resolve the exception to what the third party driver has raised, even + for SQLAlchemy dialects that include an "emulated" DBAPI exception + hierarchy. + + For non-asyncio dialects, this attribute will be the same attribute + as the :attr:`.StatementError.orig` attribute. + + For an asyncio dialect provided by SQLAlchemy, depending on if the + dialect provides an "emulated" exception hierarchy or if the underlying + DBAPI raises DBAPI-style exceptions, it will refer to either the + :attr:`.EmulatedDBAPIException.driver_exception` attribute on the + :class:`.EmulatedDBAPIException` that's thrown (such as when using + asyncpg), or to the actual exception object thrown by the + third party driver. + + .. versionadded:: 2.1 + + """ + + if self.orig is None: + raise ValueError( + "No original exception is present. Was this " + "DBAPIError constructed without a driver error?" + ) + + if isinstance(self.orig, EmulatedDBAPIException): + return self.orig.driver_exception + else: + return self.orig + class InterfaceError(DBAPIError): """Wraps a DB-API InterfaceError.""" diff --git a/lib/sqlalchemy/ext/__init__.py b/lib/sqlalchemy/ext/__init__.py index 2751bcf938a..5d61ac7652e 100644 --- a/lib/sqlalchemy/ext/__init__.py +++ b/lib/sqlalchemy/ext/__init__.py @@ -1,5 +1,5 @@ # ext/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/ext/associationproxy.py b/lib/sqlalchemy/ext/associationproxy.py index c5d85860f20..a8d794f897f 100644 --- a/lib/sqlalchemy/ext/associationproxy.py +++ b/lib/sqlalchemy/ext/associationproxy.py @@ -1,5 +1,5 @@ # ext/associationproxy.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -29,6 +29,7 @@ from typing import Iterator from typing import KeysView from typing import List +from typing import Literal from typing import Mapping from typing import MutableMapping from typing import MutableSequence @@ -61,7 +62,6 @@ from ..sql import operators from ..sql import or_ from ..sql.base import _NoArg -from ..util.typing import Literal from ..util.typing import Self from ..util.typing import SupportsKeysAndGetItem @@ -69,6 +69,7 @@ from ..orm.interfaces import MapperProperty from ..orm.interfaces import PropComparator from ..orm.mapper import Mapper + from ..orm.util import AliasedInsp from ..sql._typing import _ColumnExpressionArgument from ..sql._typing import _InfoType @@ -99,6 +100,7 @@ def association_proxy( compare: Union[_NoArg, bool] = _NoArg.NO_ARG, kw_only: Union[_NoArg, bool] = _NoArg.NO_ARG, hash: Union[_NoArg, bool, None] = _NoArg.NO_ARG, # noqa: A002 + dataclass_metadata: Union[_NoArg, Mapping[Any, Any], None] = _NoArg.NO_ARG, ) -> AssociationProxy[Any]: r"""Return a Python property implementing a view of a target attribute which references an attribute on members of the @@ -152,8 +154,6 @@ def association_proxy( source, as this object may have other state that is still to be kept. - .. versionadded:: 1.3 - .. seealso:: :ref:`cascade_scalar_deletes` - complete usage example @@ -162,7 +162,7 @@ def association_proxy( the proxied value to ``None`` should **create** the source object if it does not exist, using the creator. Only applies to scalar attributes. This is mutually exclusive - vs. the :paramref:`.assocation_proxy.cascade_scalar_deletes`. + vs. the :paramref:`.association_proxy.cascade_scalar_deletes`. .. versionadded:: 2.0.18 @@ -206,6 +206,12 @@ def association_proxy( .. versionadded:: 2.0.36 + :param dataclass_metadata: Specific to + :ref:`orm_declarative_native_dataclasses`, supplies metadata + to be attached to the generated dataclass field. + + .. versionadded:: 2.0.42 + :param info: optional, will be assigned to :attr:`.AssociationProxy.info` if present. @@ -245,7 +251,14 @@ def association_proxy( cascade_scalar_deletes=cascade_scalar_deletes, create_on_none_assignment=create_on_none_assignment, attribute_options=_AttributeOptions( - init, repr, default, default_factory, compare, kw_only, hash + init, + repr, + default, + default_factory, + compare, + kw_only, + hash, + dataclass_metadata, ), ) @@ -477,11 +490,6 @@ class User(Base): to look at the type of the actual destination object to get the complete path. - .. versionadded:: 1.3 - :class:`.AssociationProxy` no longer stores - any state specific to a particular parent class; the state is now - stored in per-class :class:`.AssociationProxyInstance` objects. - - """ return self._as_instance(class_, obj) @@ -589,8 +597,6 @@ class AssociationProxyInstance(SQLORMOperations[_T]): >>> proxy_state.scalar False - .. versionadded:: 1.3 - """ # noqa collection_class: Optional[Type[Any]] @@ -658,7 +664,7 @@ def for_proxy( except Exception as err: raise exc.InvalidRequestError( f"Association proxy received an unexpected error when " - f"trying to retreive attribute " + f"trying to retrieve attribute " f'"{target_class.__name__}.{parent.value_attr}" from ' f'class "{target_class.__name__}": {err}' ) from err @@ -1226,6 +1232,11 @@ class ObjectAssociationProxyInstance(AssociationProxyInstance[_T]): _target_is_object: bool = True _is_canonical = True + def adapt_to_entity( + self, aliased_insp: AliasedInsp[Any] + ) -> AliasedAssociationProxyInstance[_T]: + return AliasedAssociationProxyInstance(self, aliased_insp) + def contains(self, other: Any, **kw: Any) -> ColumnElement[bool]: """Produce a proxied 'contains' expression using EXISTS. @@ -1279,6 +1290,44 @@ def __ne__(self, obj: Any) -> ColumnElement[bool]: # type: ignore[override] # ) +class AliasedAssociationProxyInstance(ObjectAssociationProxyInstance[_T]): + def __init__( + self, + parent_instance: ObjectAssociationProxyInstance[_T], + aliased_insp: AliasedInsp[Any], + ) -> None: + self.parent = parent_instance.parent + self.owning_class = parent_instance.owning_class + self.aliased_insp = aliased_insp + self.target_collection = parent_instance.target_collection + self.collection_class = None + self.target_class = parent_instance.target_class + self.value_attr = parent_instance.value_attr + + @property + def _comparator(self) -> PropComparator[Any]: + return getattr( # type: ignore + self.aliased_insp.entity, self.target_collection + ).comparator + + @property + def local_attr(self) -> SQLORMOperations[Any]: + """The 'local' class attribute referenced by this + :class:`.AssociationProxyInstance`. + + .. seealso:: + + :attr:`.AssociationProxyInstance.attr` + + :attr:`.AssociationProxyInstance.remote_attr` + + """ + return cast( + "SQLORMOperations[Any]", + getattr(self.aliased_insp.entity, self.target_collection), + ) + + class ColumnAssociationProxyInstance(AssociationProxyInstance[_T]): """an :class:`.AssociationProxyInstance` that has a database column as a target. diff --git a/lib/sqlalchemy/ext/asyncio/__init__.py b/lib/sqlalchemy/ext/asyncio/__init__.py index b3452c80887..311b8c1e042 100644 --- a/lib/sqlalchemy/ext/asyncio/__init__.py +++ b/lib/sqlalchemy/ext/asyncio/__init__.py @@ -1,5 +1,5 @@ # ext/asyncio/__init__.py -# Copyright (C) 2020-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2020-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/ext/asyncio/base.py b/lib/sqlalchemy/ext/asyncio/base.py index b53d53b1a4e..40ad7d10a80 100644 --- a/lib/sqlalchemy/ext/asyncio/base.py +++ b/lib/sqlalchemy/ext/asyncio/base.py @@ -1,5 +1,5 @@ # ext/asyncio/base.py -# Copyright (C) 2020-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2020-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -18,6 +18,7 @@ from typing import Dict from typing import Generator from typing import Generic +from typing import Literal from typing import NoReturn from typing import Optional from typing import overload @@ -27,7 +28,6 @@ from . import exc as async_exc from ... import util -from ...util.typing import Literal from ...util.typing import Self _T = TypeVar("_T", bound=Any) @@ -71,26 +71,26 @@ def _target_gced( cls._proxy_objects.pop(ref, None) @classmethod - def _regenerate_proxy_for_target(cls, target: _PT) -> Self: + def _regenerate_proxy_for_target( + cls, target: _PT, **additional_kw: Any + ) -> Self: raise NotImplementedError() @overload @classmethod def _retrieve_proxy_for_target( - cls, - target: _PT, - regenerate: Literal[True] = ..., + cls, target: _PT, regenerate: Literal[True] = ..., **additional_kw: Any ) -> Self: ... @overload @classmethod def _retrieve_proxy_for_target( - cls, target: _PT, regenerate: bool = True + cls, target: _PT, regenerate: bool = True, **additional_kw: Any ) -> Optional[Self]: ... @classmethod def _retrieve_proxy_for_target( - cls, target: _PT, regenerate: bool = True + cls, target: _PT, regenerate: bool = True, **additional_kw: Any ) -> Optional[Self]: try: proxy_ref = cls._proxy_objects[weakref.ref(target)] @@ -102,7 +102,7 @@ def _retrieve_proxy_for_target( return proxy # type: ignore if regenerate: - return cls._regenerate_proxy_for_target(target) + return cls._regenerate_proxy_for_target(target, **additional_kw) else: return None @@ -148,7 +148,7 @@ def __init__( async def start(self, is_ctxmanager: bool = False) -> _T_co: try: - start_value = await util.anext_(self.gen) + start_value = await anext(self.gen) except StopAsyncIteration: raise RuntimeError("generator didn't yield") from None @@ -167,7 +167,7 @@ async def __aexit__( # vendored from contextlib.py if typ is None: try: - await util.anext_(self.gen) + await anext(self.gen) except StopAsyncIteration: return False else: @@ -193,7 +193,7 @@ async def __aexit__( # (see PEP 479 for sync generators; async generators also # have this behavior). But do this only if the exception # wrapped - # by the RuntimeError is actully Stop(Async)Iteration (see + # by the RuntimeError is actually Stop(Async)Iteration (see # issue29692). if ( isinstance(value, (StopIteration, StopAsyncIteration)) @@ -215,7 +215,7 @@ async def __aexit__( def asyncstartablecontext( - func: Callable[..., AsyncIterator[_T_co]] + func: Callable[..., AsyncIterator[_T_co]], ) -> Callable[..., GeneratorStartableContext[_T_co]]: """@asyncstartablecontext decorator. diff --git a/lib/sqlalchemy/ext/asyncio/engine.py b/lib/sqlalchemy/ext/asyncio/engine.py index f8c063a2f4f..722f3d6b3c1 100644 --- a/lib/sqlalchemy/ext/asyncio/engine.py +++ b/lib/sqlalchemy/ext/asyncio/engine.py @@ -1,5 +1,5 @@ # ext/asyncio/engine.py -# Copyright (C) 2020-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2020-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -11,11 +11,13 @@ from typing import Any from typing import AsyncIterator from typing import Callable +from typing import Concatenate from typing import Dict from typing import Generator from typing import NoReturn from typing import Optional from typing import overload +from typing import ParamSpec from typing import Type from typing import TYPE_CHECKING from typing import TypeVar @@ -39,9 +41,9 @@ from ...engine.base import NestedTransaction from ...engine.base import Transaction from ...exc import ArgumentError +from ...util import immutabledict from ...util.concurrency import greenlet_spawn -from ...util.typing import Concatenate -from ...util.typing import ParamSpec +from ...util.typing import Never from ...util.typing import TupleAny from ...util.typing import TypeVarTuple from ...util.typing import Unpack @@ -68,6 +70,7 @@ _P = ParamSpec("_P") _T = TypeVar("_T", bound=Any) _Ts = TypeVarTuple("_Ts") +_stream_results = immutabledict(stream_results=True) def create_async_engine(url: Union[str, URL], **kw: Any) -> AsyncEngine: @@ -189,7 +192,8 @@ def _no_async_engine_events(cls) -> NoReturn: "default_isolation_level", ], ) -class AsyncConnection( +# "Class has incompatible disjoint bases" - no idea +class AsyncConnection( # type:ignore[misc] ProxyComparable[Connection], StartableContext["AsyncConnection"], AsyncConnectable, @@ -258,7 +262,7 @@ def __init__( @classmethod def _regenerate_proxy_for_target( - cls, target: Connection + cls, target: Connection, **additional_kw: Any # noqa: U100 ) -> AsyncConnection: return AsyncConnection( AsyncEngine._retrieve_proxy_for_target(target.engine), target @@ -580,7 +584,7 @@ async def stream( """ if not self.dialect.supports_server_side_cursors: raise exc.InvalidRequestError( - "Cant use `stream` or `stream_scalars` with the current " + "Can't use `stream` or `stream_scalars` with the current " "dialect since it does not support server side cursors." ) @@ -589,7 +593,7 @@ async def stream( statement, parameters, execution_options=util.EMPTY_DICT.merge_with( - execution_options, {"stream_results": True} + execution_options, _stream_results ), _require_await=True, ) @@ -668,6 +672,17 @@ async def execute( ) return await _ensure_sync_result(result, self.execute) + # special case to handle mypy issue: + # https://github.com/python/mypy/issues/20651 + @overload + async def scalar( + self, + statement: TypedReturnsRows[Never], + parameters: Optional[_CoreSingleExecuteParams] = None, + *, + execution_options: Optional[CoreExecuteOptionsParameter] = None, + ) -> Optional[Any]: ... + @overload async def scalar( self, @@ -998,7 +1013,8 @@ def default_isolation_level(self) -> Any: ], attributes=["url", "pool", "dialect", "engine", "name", "driver", "echo"], ) -class AsyncEngine(ProxyComparable[Engine], AsyncConnectable): +# "Class has incompatible disjoint bases" - no idea +class AsyncEngine(ProxyComparable[Engine], AsyncConnectable): # type: ignore[misc] # noqa:E501 """An asyncio proxy for a :class:`_engine.Engine`. :class:`_asyncio.AsyncEngine` is acquired using the @@ -1045,7 +1061,9 @@ def _proxied(self) -> Engine: return self.sync_engine @classmethod - def _regenerate_proxy_for_target(cls, target: Engine) -> AsyncEngine: + def _regenerate_proxy_for_target( + cls, target: Engine, **additional_kw: Any # noqa: U100 + ) -> AsyncEngine: return AsyncEngine(target) @contextlib.asynccontextmanager @@ -1208,8 +1226,6 @@ def get_execution_options(self) -> _ExecuteOptions: Proxied for the :class:`_engine.Engine` class on behalf of the :class:`_asyncio.AsyncEngine` class. - .. versionadded: 1.3 - .. seealso:: :meth:`_engine.Engine.execution_options` @@ -1348,7 +1364,7 @@ def __init__(self, connection: AsyncConnection, nested: bool = False): @classmethod def _regenerate_proxy_for_target( - cls, target: Transaction + cls, target: Transaction, **additional_kw: Any # noqa: U100 ) -> AsyncTransaction: sync_connection = target.connection sync_transaction = target @@ -1433,7 +1449,7 @@ def _get_sync_engine_or_connection( def _get_sync_engine_or_connection( - async_engine: Union[AsyncEngine, AsyncConnection] + async_engine: Union[AsyncEngine, AsyncConnection], ) -> Union[Engine, Connection]: if isinstance(async_engine, AsyncConnection): return async_engine._proxied diff --git a/lib/sqlalchemy/ext/asyncio/exc.py b/lib/sqlalchemy/ext/asyncio/exc.py index 558187c0b41..6f61a71c195 100644 --- a/lib/sqlalchemy/ext/asyncio/exc.py +++ b/lib/sqlalchemy/ext/asyncio/exc.py @@ -1,5 +1,5 @@ # ext/asyncio/exc.py -# Copyright (C) 2020-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2020-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/ext/asyncio/result.py b/lib/sqlalchemy/ext/asyncio/result.py index ab3e23c593e..f3167f93d44 100644 --- a/lib/sqlalchemy/ext/asyncio/result.py +++ b/lib/sqlalchemy/ext/asyncio/result.py @@ -1,5 +1,5 @@ # ext/asyncio/result.py -# Copyright (C) 2020-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2020-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -9,6 +9,7 @@ import operator from typing import Any from typing import AsyncIterator +from typing import Literal from typing import Optional from typing import overload from typing import Sequence @@ -30,7 +31,6 @@ from ...sql.base import _generative from ...util import deprecated from ...util.concurrency import greenlet_spawn -from ...util.typing import Literal from ...util.typing import Self from ...util.typing import TupleAny from ...util.typing import TypeVarTuple @@ -854,7 +854,7 @@ async def all(self) -> Sequence[_R]: # noqa: A001 """ ... - async def __aiter__(self) -> AsyncIterator[_R]: ... + def __aiter__(self) -> AsyncIterator[_R]: ... async def __anext__(self) -> _R: ... @@ -988,4 +988,7 @@ async def _ensure_sync_result(result: _RT, calling_method: Any) -> _RT: calling_method.__self__.__class__.__name__, ) ) + + if is_cursor and cursor_result.cursor is not None: + await cursor_result.cursor._async_soft_close() return result diff --git a/lib/sqlalchemy/ext/asyncio/scoping.py b/lib/sqlalchemy/ext/asyncio/scoping.py index 823c354f3f4..80d7a15a0f8 100644 --- a/lib/sqlalchemy/ext/asyncio/scoping.py +++ b/lib/sqlalchemy/ext/asyncio/scoping.py @@ -1,5 +1,5 @@ # ext/asyncio/scoping.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -31,6 +31,7 @@ from ...util import ScopedRegistry from ...util import warn from ...util import warn_deprecated +from ...util.typing import Never from ...util.typing import TupleAny from ...util.typing import TypeVarTuple from ...util.typing import Unpack @@ -41,7 +42,6 @@ from .result import AsyncScalarResult from .session import AsyncSessionTransaction from ...engine import Connection - from ...engine import CursorResult from ...engine import Engine from ...engine import Result from ...engine import Row @@ -58,7 +58,6 @@ from ...orm.session import _PKIdentityArgument from ...orm.session import _SessionBind from ...sql.base import Executable - from ...sql.dml import UpdateBase from ...sql.elements import ClauseElement from ...sql.selectable import ForUpdateParameter from ...sql.selectable import TypedReturnsRows @@ -116,6 +115,7 @@ "autoflush", "no_autoflush", "info", + "execution_options", ], use_intermediate_variable=["get"], ) @@ -561,18 +561,6 @@ async def execute( _add_event: Optional[Any] = None, ) -> Result[Unpack[_Ts]]: ... - @overload - async def execute( - self, - statement: UpdateBase, - params: Optional[_CoreAnyExecuteParams] = None, - *, - execution_options: OrmExecuteOptionsParameter = util.EMPTY_DICT, - bind_arguments: Optional[_BindArguments] = None, - _parent_execute_state: Optional[Any] = None, - _add_event: Optional[Any] = None, - ) -> CursorResult[Unpack[TupleAny]]: ... - @overload async def execute( self, @@ -1051,6 +1039,17 @@ async def rollback(self) -> None: return await self._proxied.rollback() + @overload + async def scalar( + self, + statement: TypedReturnsRows[Never], + params: Optional[_CoreAnyExecuteParams] = None, + *, + execution_options: OrmExecuteOptionsParameter = util.EMPTY_DICT, + bind_arguments: Optional[_BindArguments] = None, + **kw: Any, + ) -> Optional[Any]: ... + @overload async def scalar( self, @@ -1223,8 +1222,7 @@ async def get_one( Proxied for the :class:`_asyncio.AsyncSession` class on behalf of the :class:`_asyncio.scoping.async_scoped_session` class. - Raises ``sqlalchemy.orm.exc.NoResultFound`` if the query selects - no rows. + Raises :class:`_exc.NoResultFound` if the query selects no rows. ..versionadded: 2.0.22 @@ -1586,6 +1584,25 @@ def info(self) -> Any: return self._proxied.info + @property + def execution_options(self) -> Any: + r"""Proxy for the :attr:`_orm.Session.execution_options` attribute + on behalf of the :class:`_asyncio.AsyncSession` class. + + .. container:: class_bases + + Proxied for the :class:`_asyncio.AsyncSession` class + on behalf of the :class:`_asyncio.scoping.async_scoped_session` class. + + + """ # noqa: E501 + + return self._proxied.execution_options + + @execution_options.setter + def execution_options(self, attr: Any) -> None: + self._proxied.execution_options = attr + @classmethod async def close_all(cls) -> None: r"""Close all :class:`_asyncio.AsyncSession` sessions. diff --git a/lib/sqlalchemy/ext/asyncio/session.py b/lib/sqlalchemy/ext/asyncio/session.py index adb88f53f6e..69c98c57bbe 100644 --- a/lib/sqlalchemy/ext/asyncio/session.py +++ b/lib/sqlalchemy/ext/asyncio/session.py @@ -1,5 +1,5 @@ # ext/asyncio/session.py -# Copyright (C) 2020-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2020-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -11,6 +11,7 @@ from typing import Awaitable from typing import Callable from typing import cast +from typing import Concatenate from typing import Dict from typing import Generic from typing import Iterable @@ -18,6 +19,7 @@ from typing import NoReturn from typing import Optional from typing import overload +from typing import ParamSpec from typing import Sequence from typing import Tuple from typing import Type @@ -38,8 +40,7 @@ from ...orm import SessionTransaction from ...orm import state as _instance_state from ...util.concurrency import greenlet_spawn -from ...util.typing import Concatenate -from ...util.typing import ParamSpec +from ...util.typing import Never from ...util.typing import TupleAny from ...util.typing import TypeVarTuple from ...util.typing import Unpack @@ -49,13 +50,13 @@ from .engine import AsyncConnection from .engine import AsyncEngine from ...engine import Connection - from ...engine import CursorResult from ...engine import Engine from ...engine import Result from ...engine import Row from ...engine import RowMapping from ...engine import ScalarResult from ...engine.interfaces import _CoreAnyExecuteParams + from ...engine.interfaces import _ExecuteOptions from ...engine.interfaces import CoreExecuteOptionsParameter from ...event import dispatcher from ...orm._typing import _IdentityKeyType @@ -70,7 +71,6 @@ from ...orm.session import _SessionBindKey from ...sql._typing import _InfoType from ...sql.base import Executable - from ...sql.dml import UpdateBase from ...sql.elements import ClauseElement from ...sql.selectable import ForUpdateParameter from ...sql.selectable import TypedReturnsRows @@ -205,6 +205,7 @@ def awaitable_attrs(self) -> AsyncAttrs._AsyncAttrGetitem: "autoflush", "no_autoflush", "info", + "execution_options", ], ) class AsyncSession(ReversibleProxy[Session]): @@ -414,18 +415,6 @@ async def execute( _add_event: Optional[Any] = None, ) -> Result[Unpack[_Ts]]: ... - @overload - async def execute( - self, - statement: UpdateBase, - params: Optional[_CoreAnyExecuteParams] = None, - *, - execution_options: OrmExecuteOptionsParameter = util.EMPTY_DICT, - bind_arguments: Optional[_BindArguments] = None, - _parent_execute_state: Optional[Any] = None, - _add_event: Optional[Any] = None, - ) -> CursorResult[Unpack[TupleAny]]: ... - @overload async def execute( self, @@ -473,6 +462,19 @@ async def execute( ) return await _ensure_sync_result(result, self.execute) + # special case to handle mypy issue: + # https://github.com/python/mypy/issues/20651 + @overload + async def scalar( + self, + statement: TypedReturnsRows[Never], + params: Optional[_CoreAnyExecuteParams] = None, + *, + execution_options: OrmExecuteOptionsParameter = util.EMPTY_DICT, + bind_arguments: Optional[_BindArguments] = None, + **kw: Any, + ) -> Optional[Any]: ... + @overload async def scalar( self, @@ -631,8 +633,7 @@ async def get_one( """Return an instance based on the given primary key identifier, or raise an exception if not found. - Raises ``sqlalchemy.orm.exc.NoResultFound`` if the query selects - no rows. + Raises :class:`_exc.NoResultFound` if the query selects no rows. ..versionadded: 2.0.22 @@ -843,7 +844,9 @@ def get_transaction(self) -> Optional[AsyncSessionTransaction]: """ trans = self.sync_session.get_transaction() if trans is not None: - return AsyncSessionTransaction._retrieve_proxy_for_target(trans) + return AsyncSessionTransaction._retrieve_proxy_for_target( + trans, async_session=self + ) else: return None @@ -859,7 +862,9 @@ def get_nested_transaction(self) -> Optional[AsyncSessionTransaction]: trans = self.sync_session.get_nested_transaction() if trans is not None: - return AsyncSessionTransaction._retrieve_proxy_for_target(trans) + return AsyncSessionTransaction._retrieve_proxy_for_target( + trans, async_session=self + ) else: return None @@ -1598,6 +1603,19 @@ def info(self) -> Any: return self._proxied.info + @property + def execution_options(self) -> _ExecuteOptions: + r"""Proxy for the :attr:`_orm.Session.execution_options` attribute + on behalf of the :class:`_asyncio.AsyncSession` class. + + """ # noqa: E501 + + return self._proxied.execution_options + + @execution_options.setter + def execution_options(self, attr: _ExecuteOptions) -> None: + self._proxied.execution_options = attr + @classmethod def object_session(cls, instance: object) -> Optional[Session]: r"""Return the :class:`.Session` to which an object belongs. @@ -1896,6 +1914,21 @@ async def commit(self) -> None: await greenlet_spawn(self._sync_transaction().commit) + @classmethod + def _regenerate_proxy_for_target( # type: ignore[override] + cls, + target: SessionTransaction, + async_session: AsyncSession, + **additional_kw: Any, # noqa: U100 + ) -> AsyncSessionTransaction: + sync_transaction = target + nested = target.nested + obj = cls.__new__(cls) + obj.session = async_session + obj.sync_transaction = obj._assign_proxied(sync_transaction) + obj.nested = nested + return obj + async def start( self, is_ctxmanager: bool = False ) -> AsyncSessionTransaction: diff --git a/lib/sqlalchemy/ext/automap.py b/lib/sqlalchemy/ext/automap.py index 169bebfbf3f..35af5ed8d2b 100644 --- a/lib/sqlalchemy/ext/automap.py +++ b/lib/sqlalchemy/ext/automap.py @@ -1,5 +1,5 @@ # ext/automap.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -229,7 +229,7 @@ class name. :attr:`.AutomapBase.by_module` when explicit ``__module__`` conventions are present. -.. versionadded: 2.0 +.. versionadded:: 2.0 Added the :attr:`.AutomapBase.by_module` collection, which stores classes within a named hierarchy based on dot-separated module names, @@ -736,7 +736,7 @@ def column_reflect(inspector, table, column_info): from ..orm import exc as orm_exc from ..orm import interfaces from ..orm import relationship -from ..orm.decl_base import _DeferredMapperConfig +from ..orm.decl_base import _DeferredDeclarativeConfig from ..orm.mapper import _CONFIGURE_MUTEX from ..schema import ForeignKeyConstraint from ..sql import and_ @@ -1266,11 +1266,11 @@ def prepare( with _CONFIGURE_MUTEX: table_to_map_config: Union[ - Dict[Optional[Table], _DeferredMapperConfig], - Dict[Table, _DeferredMapperConfig], + Dict[Optional[Table], _DeferredDeclarativeConfig], + Dict[Table, _DeferredDeclarativeConfig], ] = { cast("Table", m.local_table): m - for m in _DeferredMapperConfig.classes_for_base( + for m in _DeferredDeclarativeConfig.classes_for_base( cls, sort=False ) } @@ -1324,7 +1324,7 @@ def prepare( (automap_base,), clsdict, ) - map_config = _DeferredMapperConfig.config_for_cls( + map_config = _DeferredDeclarativeConfig.config_for_cls( mapped_cls ) assert map_config.cls.__name__ == newname @@ -1374,7 +1374,7 @@ def prepare( generate_relationship, ) - for map_config in _DeferredMapperConfig.classes_for_base( + for map_config in _DeferredDeclarativeConfig.classes_for_base( automap_base ): map_config.map() @@ -1490,10 +1490,10 @@ def _is_many_to_many( def _relationships_for_fks( automap_base: Type[Any], - map_config: _DeferredMapperConfig, + map_config: _DeferredDeclarativeConfig, table_to_map_config: Union[ - Dict[Optional[Table], _DeferredMapperConfig], - Dict[Table, _DeferredMapperConfig], + Dict[Optional[Table], _DeferredDeclarativeConfig], + Dict[Table, _DeferredDeclarativeConfig], ], collection_class: type, name_for_scalar_relationship: NameForScalarRelationshipType, @@ -1605,8 +1605,8 @@ def _m2m_relationship( m2m_const: List[ForeignKeyConstraint], table: Table, table_to_map_config: Union[ - Dict[Optional[Table], _DeferredMapperConfig], - Dict[Table, _DeferredMapperConfig], + Dict[Optional[Table], _DeferredDeclarativeConfig], + Dict[Table, _DeferredDeclarativeConfig], ], collection_class: type, name_for_scalar_relationship: NameForCollectionRelationshipType, diff --git a/lib/sqlalchemy/ext/baked.py b/lib/sqlalchemy/ext/baked.py index cd3e087931e..fd88556c836 100644 --- a/lib/sqlalchemy/ext/baked.py +++ b/lib/sqlalchemy/ext/baked.py @@ -1,5 +1,5 @@ # ext/baked.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -39,9 +39,6 @@ class Bakery: :meth:`.BakedQuery.bakery`. It exists as an object so that the "cache" can be easily inspected. - .. versionadded:: 1.2 - - """ __slots__ = "cls", "cache" @@ -277,10 +274,6 @@ def to_query(self, query_or_session): :class:`.Session` object, that is assumed to be within the context of an enclosing :class:`.BakedQuery` callable. - - .. versionadded:: 1.3 - - """ # noqa: E501 if isinstance(query_or_session, Session): @@ -360,10 +353,6 @@ def with_post_criteria(self, fn): :meth:`_query.Query.execution_options` methods should be used. - - .. versionadded:: 1.2 - - """ return self._using_post_criteria([fn]) diff --git a/lib/sqlalchemy/ext/compiler.py b/lib/sqlalchemy/ext/compiler.py index cc64477ed47..2de5001ee46 100644 --- a/lib/sqlalchemy/ext/compiler.py +++ b/lib/sqlalchemy/ext/compiler.py @@ -1,5 +1,5 @@ # ext/compiler.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/ext/declarative/__init__.py b/lib/sqlalchemy/ext/declarative/__init__.py index 0383f9d34f8..e134641685e 100644 --- a/lib/sqlalchemy/ext/declarative/__init__.py +++ b/lib/sqlalchemy/ext/declarative/__init__.py @@ -1,5 +1,5 @@ # ext/declarative/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/ext/declarative/extensions.py b/lib/sqlalchemy/ext/declarative/extensions.py index 3dc6bf698c4..7b5c81b3919 100644 --- a/lib/sqlalchemy/ext/declarative/extensions.py +++ b/lib/sqlalchemy/ext/declarative/extensions.py @@ -1,5 +1,5 @@ # ext/declarative/extensions.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -24,7 +24,7 @@ from ...orm import relationships from ...orm.base import _mapper_or_none from ...orm.clsregistry import _resolver -from ...orm.decl_base import _DeferredMapperConfig +from ...orm.decl_base import _DeferredDeclarativeConfig from ...orm.util import polymorphic_union from ...schema import Table from ...util import OrderedDict @@ -40,7 +40,7 @@ class ConcreteBase: function automatically, against all tables mapped as a subclass to this class. The function is called via the ``__declare_last__()`` function, which is essentially - a hook for the :meth:`.after_configured` event. + a hook for the :meth:`.MapperEvents.after_configured` event. :class:`.ConcreteBase` produces a mapped table for the class itself. Compare to :class:`.AbstractConcreteBase`, @@ -80,10 +80,6 @@ class Manager(Employee): class Employee(ConcreteBase, Base): _concrete_discriminator_name = "_concrete_discriminator" - .. versionadded:: 1.3.19 Added the ``_concrete_discriminator_name`` - attribute to :class:`_declarative.ConcreteBase` so that the - virtual discriminator column name can be customized. - .. versionchanged:: 1.4.2 The ``_concrete_discriminator_name`` attribute need only be placed on the basemost class to take correct effect for all subclasses. An explicit error message is now raised if the @@ -133,7 +129,7 @@ class AbstractConcreteBase(ConcreteBase): function automatically, against all tables mapped as a subclass to this class. The function is called via the ``__declare_first__()`` function, which is essentially - a hook for the :meth:`.before_configured` event. + a hook for the :meth:`.MapperEvents.before_configured` event. :class:`.AbstractConcreteBase` applies :class:`_orm.Mapper` for its immediately inheriting class, as would occur for any other @@ -274,7 +270,7 @@ def _sa_decl_prepare_nocascade(cls): if getattr(cls, "__mapper__", None): return - to_map = _DeferredMapperConfig.config_for_cls(cls) + to_map = _DeferredDeclarativeConfig.config_for_cls(cls) # can't rely on 'self_and_descendants' here # since technically an immediate subclass @@ -455,7 +451,7 @@ def prepare( """ - to_map = _DeferredMapperConfig.classes_for_base(cls) + to_map = _DeferredDeclarativeConfig.classes_for_base(cls) metadata_to_table = collections.defaultdict(set) diff --git a/lib/sqlalchemy/ext/horizontal_shard.py b/lib/sqlalchemy/ext/horizontal_shard.py index 7ada621226c..02cdd66d698 100644 --- a/lib/sqlalchemy/ext/horizontal_shard.py +++ b/lib/sqlalchemy/ext/horizontal_shard.py @@ -1,5 +1,5 @@ # ext/horizontal_shard.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -16,7 +16,7 @@ .. deepalchemy:: The horizontal sharding extension is an advanced feature, involving a complex statement -> database interaction as well as use of semi-public APIs for non-trivial cases. Simpler approaches to - refering to multiple database "shards", most commonly using a distinct + referring to multiple database "shards", most commonly using a distinct :class:`_orm.Session` per "shard", should always be considered first before using this more complex and less-production-tested system. diff --git a/lib/sqlalchemy/ext/hybrid.py b/lib/sqlalchemy/ext/hybrid.py index 6a22fb614d2..f0efee6b76f 100644 --- a/lib/sqlalchemy/ext/hybrid.py +++ b/lib/sqlalchemy/ext/hybrid.py @@ -1,5 +1,5 @@ # ext/hybrid.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -253,7 +253,7 @@ def radius(cls): In order to produce a reasonable syntax while remaining typing compliant, the :attr:`.hybrid_property.inplace` decorator allows the same -decorator to be re-used with different method names, while still producing +decorator to be reused with different method names, while still producing a single decorator under one name:: # correct use which is also accepted by pep-484 tooling @@ -320,59 +320,140 @@ def _length_setter(self, value: int) -> None: .. _hybrid_bulk_update: -Allowing Bulk ORM Update ------------------------- +Supporting ORM Bulk INSERT and UPDATE +------------------------------------- -A hybrid can define a custom "UPDATE" handler for when using -ORM-enabled updates, allowing the hybrid to be used in the -SET clause of the update. +Hybrids have support for use in ORM Bulk INSERT/UPDATE operations described +at :ref:`orm_expression_update_delete`. There are two distinct hooks +that may be used supply a hybrid value within a DML operation: -Normally, when using a hybrid with :func:`_sql.update`, the SQL -expression is used as the column that's the target of the SET. If our -``Interval`` class had a hybrid ``start_point`` that linked to -``Interval.start``, this could be substituted directly:: +1. The :meth:`.hybrid_property.update_expression` hook indicates a method that + can provide one or more expressions to render in the SET clause of an + UPDATE or INSERT statement, in response to when a hybrid attribute is referenced + directly in the :meth:`.UpdateBase.values` method; i.e. the use shown + in :ref:`orm_queryguide_update_delete_where` and :ref:`orm_queryguide_insert_values` - from sqlalchemy import update +2. The :meth:`.hybrid_property.bulk_dml` hook indicates a method that + can intercept individual parameter dictionaries sent to :meth:`_orm.Session.execute`, + i.e. the use shown at :ref:`orm_queryguide_bulk_insert` as well + as :ref:`orm_queryguide_bulk_update`. - stmt = update(Interval).values({Interval.start_point: 10}) +Using update_expression with update.values() and insert.values() +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -However, when using a composite hybrid like ``Interval.length``, this -hybrid represents more than one column. We can set up a handler that will -accommodate a value passed in the VALUES expression which can affect -this, using the :meth:`.hybrid_property.update_expression` decorator. -A handler that works similarly to our setter would be:: +The :meth:`.hybrid_property.update_expression` decorator indicates a method +that is invoked when a hybrid is used in the :meth:`.ValuesBase.values` clause +of an :func:`_sql.update` or :func:`_sql.insert` statement. It returns a list +of tuple pairs ``[(x1, y1), (x2, y2), ...]`` which will expand into the SET +clause of an UPDATE statement as ``SET x1=y1, x2=y2, ...``. - from typing import List, Tuple, Any +The :func:`_sql.from_dml_column` construct is often useful as it can create a +SQL expression that refers to another column that may also present in the same +INSERT or UPDATE statement, alternatively falling back to referring to the +original column if such an expression is not present. +In the example below, the ``total_price`` hybrid will derive the ``price`` +column, by taking the given "total price" value and dividing it by a +``tax_rate`` value that is also present in the :meth:`.ValuesBase.values` call:: - class Interval(Base): - # ... + from sqlalchemy import from_dml_column - @hybrid_property - def length(self) -> int: - return self.end - self.start - @length.inplace.setter - def _length_setter(self, value: int) -> None: - self.end = self.start + value + class Product(Base): + __tablename__ = "product" - @length.inplace.update_expression - def _length_update_expression( + id: Mapped[int] = mapped_column(primary_key=True) + price: Mapped[float] + tax_rate: Mapped[float] + + @hybrid_property + def total_price(self) -> float: + return self.price * (1 + self.tax_rate) + + @total_price.inplace.update_expression + @classmethod + def _total_price_update_expression( cls, value: Any ) -> List[Tuple[Any, Any]]: - return [(cls.end, cls.start + value)] + return [(cls.price, value / (1 + from_dml_column(cls.tax_rate)))] -Above, if we use ``Interval.length`` in an UPDATE expression, we get -a hybrid SET expression: +When used in an UPDATE statement, :func:`_sql.from_dml_column` creates a +reference to the ``tax_rate`` column that will use the value passed to +the :meth:`.ValuesBase.values` method, rather than the existing value on the column +in the database. This allows the hybrid to access other values being +updated in the same statement: .. sourcecode:: pycon+sql + >>> from sqlalchemy import update + >>> print( + ... update(Product).values( + ... {Product.tax_rate: 0.08, Product.total_price: 125.00} + ... ) + ... ) + {printsql}UPDATE product SET tax_rate=:tax_rate, price=(:total_price / (:tax_rate + :param_1)) + +When the column referenced by :func:`_sql.from_dml_column` (in this case ``product.tax_rate``) +is omitted from :meth:`.ValuesBase.values`, the rendered expression falls back to +using the original column: + +.. sourcecode:: pycon+sql >>> from sqlalchemy import update - >>> print(update(Interval).values({Interval.length: 25})) - {printsql}UPDATE interval SET "end"=(interval.start + :start_1) + >>> print(update(Product).values({Product.total_price: 125.00})) + {printsql}UPDATE product SET price=(:total_price / (tax_rate + :param_1)) + + + +Using bulk_dml to intercept bulk parameter dictionaries +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.1 + +For bulk operations that pass a list of parameter dictionaries to +methods like :meth:`.Session.execute`, the +:meth:`.hybrid_property.bulk_dml` decorator provides a hook that can +receive each dictionary and populate it with new values. + +The implementation for the :meth:`.hybrid_property.bulk_dml` hook can retrieve +other column values from the parameter dictionary:: + + from typing import MutableMapping + + + class Product(Base): + __tablename__ = "product" + + id: Mapped[int] = mapped_column(primary_key=True) + price: Mapped[float] + tax_rate: Mapped[float] + + @hybrid_property + def total_price(self) -> float: + return self.price * (1 + self.tax_rate) -This SET expression is accommodated by the ORM automatically. + @total_price.inplace.bulk_dml + @classmethod + def _total_price_bulk_dml( + cls, mapping: MutableMapping[str, Any], value: float + ) -> None: + mapping["price"] = value / (1 + mapping["tax_rate"]) + +This allows for bulk INSERT/UPDATE with derived values:: + + # Bulk INSERT + session.execute( + insert(Product), + [ + {"tax_rate": 0.08, "total_price": 125.00}, + {"tax_rate": 0.05, "total_price": 110.00}, + ], + ) + +Note that the method decorated by :meth:`.hybrid_property.bulk_dml` is invoked +only with parameter dictionaries and does not have the ability to use +SQL expressions in the given dictionaries, only literal Python values that will +be passed to parameters in the INSERT or UPDATE statement. .. seealso:: @@ -465,7 +546,7 @@ def _balance_expression(cls) -> SQLColumnExpression[Optional[Decimal]]: list available on ``self``. .. tip:: The ``User.balance`` getter in the above example accesses the - ``self.acccounts`` collection, which will normally be loaded via the + ``self.accounts`` collection, which will normally be loaded via the :func:`.selectinload` loader strategy configured on the ``User.balance`` :func:`_orm.relationship`. The default loader strategy when not otherwise stated on :func:`_orm.relationship` is :func:`.lazyload`, which emits SQL on @@ -731,31 +812,36 @@ class FirstNameLastName(FirstNameOnly): def name(cls): return func.concat(cls.first_name, " ", cls.last_name) +.. _hybrid_value_objects: + Hybrid Value Objects -------------------- -Note in our previous example, if we were to compare the ``word_insensitive`` +In the example shown previously at :ref:`hybrid_custom_comparators`, +if we were to compare the ``word_insensitive`` attribute of a ``SearchWord`` instance to a plain Python string, the plain Python string would not be coerced to lower case - the ``CaseInsensitiveComparator`` we built, being returned by ``@word_insensitive.comparator``, only applies to the SQL side. -A more comprehensive form of the custom comparator is to construct a *Hybrid -Value Object*. This technique applies the target value or expression to a value +A more comprehensive form of the custom comparator is to construct a **Hybrid +Value Object**. This technique applies the target value or expression to a value object which is then returned by the accessor in all cases. The value object allows control of all operations upon the value as well as how compared values are treated, both on the SQL expression side as well as the Python value side. Replacing the previous ``CaseInsensitiveComparator`` class with a new ``CaseInsensitiveWord`` class:: + from sqlalchemy import func + from sqlalchemy.ext.hybrid import Comparator + + class CaseInsensitiveWord(Comparator): "Hybrid value representing a lower case representation of a word." def __init__(self, word): - if isinstance(word, basestring): + if isinstance(word, str): self.word = word.lower() - elif isinstance(word, CaseInsensitiveWord): - self.word = word.word else: self.word = func.lower(word) @@ -774,11 +860,50 @@ def __str__(self): "Label to apply to Query tuple results" Above, the ``CaseInsensitiveWord`` object represents ``self.word``, which may -be a SQL function, or may be a Python native. By overriding ``operate()`` and -``__clause_element__()`` to work in terms of ``self.word``, all comparison -operations will work against the "converted" form of ``word``, whether it be -SQL side or Python side. Our ``SearchWord`` class can now deliver the -``CaseInsensitiveWord`` object unconditionally from a single hybrid call:: +be a SQL function, or may be a Python native string. The hybrid value object should +implement ``__clause_element__()``, which allows the object to be coerced into +a SQL-capable value when used in SQL expression constructs, as well as Python +comparison methods such as ``__eq__()``, which is accomplished in the above +example by subclassing :class:`.hybrid.Comparator` and overriding the +``operate()`` method. + +.. topic:: Building the Value object with dataclasses + + Hybrid value objects may also be implemented as Python dataclasses. If + modification to values upon construction is needed, use the + ``__post_init__()`` dataclasses method. Instance variables that work in + a "hybrid" fashion may be instance of a plain Python value, or an instance + of :class:`.SQLColumnExpression` genericized against that type. Also make sure to disable + dataclass comparison features, as the :class:`.hybrid.Comparator` class + provides these:: + + from sqlalchemy import func + from sqlalchemy.ext.hybrid import Comparator + from dataclasses import dataclass + + + @dataclass(eq=False) + class CaseInsensitiveWord(Comparator): + word: str | SQLColumnExpression[str] + + def __post_init__(self): + if isinstance(self.word, str): + self.word = self.word.lower() + else: + self.word = func.lower(self.word) + + def operate(self, op, other, **kwargs): + if not isinstance(other, CaseInsensitiveWord): + other = CaseInsensitiveWord(other) + return op(self.word, other.word, **kwargs) + + def __clause_element__(self): + return self.word + +With ``__clause_element__()`` provided, our ``SearchWord`` class +can now deliver the ``CaseInsensitiveWord`` object unconditionally from a +single hybrid method, returning an object that behaves appropriately +in both value-based and SQL contexts:: class SearchWord(Base): __tablename__ = "searchword" @@ -789,18 +914,20 @@ class SearchWord(Base): def word_insensitive(self) -> CaseInsensitiveWord: return CaseInsensitiveWord(self.word) -The ``word_insensitive`` attribute now has case-insensitive comparison behavior -universally, including SQL expression vs. Python expression (note the Python -value is converted to lower case on the Python side here): +The class-level version of ``CaseInsensitiveWord`` will work in SQL +constructs: .. sourcecode:: pycon+sql - >>> print(select(SearchWord).filter_by(word_insensitive="Trucks")) + >>> print(select(SearchWord).filter(SearchWord.word_insensitive == "Trucks")) {printsql}SELECT searchword.id AS searchword_id, searchword.word AS searchword_word FROM searchword WHERE lower(searchword.word) = :lower_1 -SQL expression versus SQL expression: +By also subclassing :class:`.hybrid.Comparator` and providing an implementation +for ``operate()``, the ``word_insensitive`` attribute also has case-insensitive +comparison behavior universally, including SQL expression and Python expression +(note the Python value is converted to lower case on the Python side here): .. sourcecode:: pycon+sql @@ -841,6 +968,176 @@ def word_insensitive(self) -> CaseInsensitiveWord: `_ - on the techspot.zzzeek.org blog +.. _composite_hybrid_value_objects: + +Composite Hybrid Value Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The functionality of :ref:`hybrid_value_objects` may also be expanded to +support "composite" forms; in this pattern, SQLAlchemy hybrids begin to +approximate most (though not all) the same functionality that is available from +the ORM natively via the :ref:`mapper_composite` feature. We can imitate the +example of ``Point`` and ``Vertex`` from that section using hybrids, where +``Point`` is modified to become a Hybrid Value Object:: + + from dataclasses import dataclass + + from sqlalchemy import tuple_ + from sqlalchemy.ext.hybrid import Comparator + from sqlalchemy import SQLColumnExpression + + + @dataclass(eq=False) + class Point(Comparator): + x: int | SQLColumnExpression[int] + y: int | SQLColumnExpression[int] + + def operate(self, op, other, **kwargs): + return op(self.x, other.x) & op(self.y, other.y) + + def __clause_element__(self): + return tuple_(self.x, self.y) + +Above, the ``operate()`` method is where the most "hybrid" behavior takes +place, making use of ``op()`` (the Python operator function in use) along +with the the bitwise ``&`` operator provides us with the SQL AND operator +in a SQL context, and boolean "and" in a Python boolean context. + +Following from there, the owning ``Vertex`` class now uses hybrids to +represent ``start`` and ``end``:: + + from sqlalchemy.orm import DeclarativeBase, Mapped + from sqlalchemy.orm import mapped_column + from sqlalchemy.ext.hybrid import hybrid_property + + + class Base(DeclarativeBase): + pass + + + class Vertex(Base): + __tablename__ = "vertices" + + id: Mapped[int] = mapped_column(primary_key=True) + + x1: Mapped[int] + y1: Mapped[int] + x2: Mapped[int] + y2: Mapped[int] + + @hybrid_property + def start(self) -> Point: + return Point(self.x1, self.y1) + + @start.inplace.setter + def _set_start(self, value: Point) -> None: + self.x1 = value.x + self.y1 = value.y + + @hybrid_property + def end(self) -> Point: + return Point(self.x2, self.y2) + + @end.inplace.setter + def _set_end(self, value: Point) -> None: + self.x2 = value.x + self.y2 = value.y + + def __repr__(self) -> str: + return f"Vertex(start={self.start}, end={self.end})" + +Using the above mapping, we can use expressions at the Python or SQL level +using ``Vertex.start`` and ``Vertex.end``:: + + >>> v1 = Vertex(start=Point(3, 4), end=Point(15, 10)) + >>> v1.end == Point(15, 10) + True + >>> stmt = ( + ... select(Vertex) + ... .where(Vertex.start == Point(3, 4)) + ... .where(Vertex.end < Point(7, 8)) + ... ) + >>> print(stmt) + SELECT vertices.id, vertices.x1, vertices.y1, vertices.x2, vertices.y2 + FROM vertices + WHERE vertices.x1 = :x1_1 AND vertices.y1 = :y1_1 AND vertices.x2 < :x2_1 AND vertices.y2 < :y2_1 + +DML Support for Composite Value Objects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Composite value objects like ``Point`` can also be used with the ORM's +DML features. The :meth:`.hybrid_property.update_expression` decorator allows +the hybrid to expand a composite value into multiple column assignments +in UPDATE and INSERT statements:: + + class Location(Base): + __tablename__ = "location" + + id: Mapped[int] = mapped_column(primary_key=True) + x: Mapped[int] + y: Mapped[int] + + @hybrid_property + def coordinates(self) -> Point: + return Point(self.x, self.y) + + @coordinates.inplace.update_expression + @classmethod + def _coordinates_update_expression( + cls, value: Any + ) -> List[Tuple[Any, Any]]: + assert isinstance(value, Point) + return [(cls.x, value.x), (cls.y, value.y)] + +This allows UPDATE statements to work with the composite value: + +.. sourcecode:: pycon+sql + + >>> from sqlalchemy import update + >>> print( + ... update(Location) + ... .where(Location.id == 5) + ... .values({Location.coordinates: Point(25, 17)}) + ... ) + {printsql}UPDATE location SET x=:x, y=:y WHERE location.id = :id_1 + +For bulk operations that use parameter dictionaries, the +:meth:`.hybrid_property.bulk_dml` decorator provides a hook to +convert composite values into individual column values:: + + from typing import MutableMapping + + + class Location(Base): + # ... (same as above) + + @coordinates.inplace.bulk_dml + @classmethod + def _coordinates_bulk_dml( + cls, mapping: MutableMapping[str, Any], value: Point + ) -> None: + mapping["x"] = value.x + mapping["y"] = value.y + +This enables bulk operations with composite values:: + + # Bulk INSERT + session.execute( + insert(Location), + [ + {"id": 1, "coordinates": Point(10, 20)}, + {"id": 2, "coordinates": Point(30, 40)}, + ], + ) + + # Bulk UPDATE + session.execute( + update(Location), + [ + {"id": 1, "coordinates": Point(15, 25)}, + {"id": 2, "coordinates": Point(35, 45)}, + ], + ) """ # noqa @@ -849,10 +1146,14 @@ def word_insensitive(self) -> CaseInsensitiveWord: from typing import Any from typing import Callable from typing import cast +from typing import Concatenate from typing import Generic from typing import List +from typing import Literal +from typing import MutableMapping from typing import Optional from typing import overload +from typing import ParamSpec from typing import Protocol from typing import Sequence from typing import Tuple @@ -861,6 +1162,7 @@ def word_insensitive(self) -> CaseInsensitiveWord: from typing import TypeVar from typing import Union +from .. import exc from .. import util from ..orm import attributes from ..orm import InspectionAttrExtensionType @@ -871,9 +1173,6 @@ def word_insensitive(self) -> CaseInsensitiveWord: from ..sql._typing import is_has_clause_element from ..sql.elements import ColumnElement from ..sql.elements import SQLCoreOperations -from ..util.typing import Concatenate -from ..util.typing import Literal -from ..util.typing import ParamSpec from ..util.typing import Self if TYPE_CHECKING: @@ -923,11 +1222,11 @@ class HybridExtensionType(InspectionAttrExtensionType): class _HybridGetterType(Protocol[_T_co]): - def __call__(s, self: Any) -> _T_co: ... + def __call__(s, self: Any, /) -> _T_co: ... class _HybridSetterType(Protocol[_T_con]): - def __call__(s, self: Any, value: _T_con) -> None: ... + def __call__(s, self: Any, value: _T_con, /) -> None: ... class _HybridUpdaterType(Protocol[_T_con]): @@ -938,13 +1237,22 @@ def __call__( ) -> List[Tuple[_DMLColumnArgument, Any]]: ... +class _HybridBulkDMLType(Protocol[_T_co]): + def __call__( + s, + cls: Any, + mapping: MutableMapping[str, Any], + value: Any, + ) -> Any: ... + + class _HybridDeleterType(Protocol[_T_co]): - def __call__(s, self: Any) -> None: ... + def __call__(s, self: Any, /) -> None: ... class _HybridExprCallableType(Protocol[_T_co]): def __call__( - s, cls: Any + s, cls: Any, / ) -> Union[_HasClauseElement[_T_co], SQLColumnExpression[_T_co]]: ... @@ -979,6 +1287,10 @@ def update_expression( self, meth: _HybridUpdaterType[_T] ) -> hybrid_property[_T]: ... + def bulk_dml( + self, meth: _HybridBulkDMLType[_T] + ) -> hybrid_property[_T]: ... + class hybrid_method(interfaces.InspectionAttrInfo, Generic[_P, _R]): """A decorator which allows definition of a Python object method with both @@ -1093,6 +1405,7 @@ def __init__( expr: Optional[_HybridExprCallableType[_T]] = None, custom_comparator: Optional[Comparator[_T]] = None, update_expr: Optional[_HybridUpdaterType[_T]] = None, + bulk_dml_setter: Optional[_HybridBulkDMLType[_T]] = None, ): """Create a new :class:`.hybrid_property`. @@ -1117,6 +1430,7 @@ def value(self, value): self.expr = _unwrap_classmethod(expr) self.custom_comparator = _unwrap_classmethod(custom_comparator) self.update_expr = _unwrap_classmethod(update_expr) + self.bulk_dml_setter = _unwrap_classmethod(bulk_dml_setter) util.update_wrapper(self, fget) # type: ignore[arg-type] @overload @@ -1140,10 +1454,12 @@ def __get__( else: return self.fget(instance) - def __set__(self, instance: object, value: Any) -> None: + def __set__( + self, instance: object, value: Union[SQLCoreOperations[_T], _T] + ) -> None: if self.fset is None: raise AttributeError("can't set attribute") - self.fset(instance, value) + self.fset(instance, value) # type: ignore[arg-type] def __delete__(self, instance: object) -> None: if self.fdel is None: @@ -1187,8 +1503,6 @@ class SubClass(SuperClass): def foobar(cls): return func.subfoobar(self._foobar) - .. versionadded:: 1.2 - .. seealso:: :ref:`hybrid_reuse_subclass` @@ -1239,12 +1553,17 @@ def update_expression( ) -> hybrid_property[_TE]: return self._set(update_expr=meth) + def bulk_dml( + self, meth: _HybridBulkDMLType[_TE] + ) -> hybrid_property[_TE]: + return self._set(bulk_dml_setter=meth) + @property def inplace(self) -> _InPlace[_T]: """Return the inplace mutator for this :class:`.hybrid_property`. This is to allow in-place mutation of the hybrid, allowing the first - hybrid method of a certain name to be re-used in order to add + hybrid method of a certain name to be reused in order to add more methods without having to name those methods the same, e.g.:: class Interval(Base): @@ -1272,11 +1591,7 @@ def _radius_expression(cls) -> ColumnElement[float]: return hybrid_property._InPlace(self) def getter(self, fget: _HybridGetterType[_T]) -> hybrid_property[_T]: - """Provide a modifying decorator that defines a getter method. - - .. versionadded:: 1.2 - - """ + """Provide a modifying decorator that defines a getter method.""" return self._copy(fget=fget) @@ -1391,11 +1706,17 @@ def fullname(cls, value): fname, lname = value.split(" ", 1) return [(cls.first_name, fname), (cls.last_name, lname)] - .. versionadded:: 1.2 - """ return self._copy(update_expr=meth) + def bulk_dml(self, meth: _HybridBulkDMLType[_T]) -> hybrid_property[_T]: + """Define a setter for bulk dml. + + .. versionadded:: 2.1 + + """ + return self._copy(bulk_dml=meth) + @util.memoized_property def _expr_comparator( self, @@ -1434,7 +1755,7 @@ def expr_comparator( name = self.__name__ break else: - name = attributes._UNKNOWN_ATTR_KEY # type: ignore[assignment] + name = attributes._UNKNOWN_ATTR_KEY # type: ignore[assignment,unused-ignore] # noqa: E501 return cast( "_HybridClassLevelAccessor[_T]", @@ -1506,7 +1827,8 @@ def info(self) -> _InfoType: return self.hybrid.info def _bulk_update_tuples( - self, value: Any + self, + value: Any, ) -> Sequence[Tuple[_DMLColumnArgument, Any]]: if isinstance(self.expression, attributes.QueryableAttribute): return self.expression._bulk_update_tuples(value) @@ -1515,6 +1837,28 @@ def _bulk_update_tuples( else: return [(self.expression, value)] + def _bulk_dml_setter(self, key: str) -> Optional[Callable[..., Any]]: + """return a callable that will process a bulk INSERT value""" + + meth = None + + def prop(mapping: MutableMapping[str, Any]) -> None: + nonlocal meth + value = mapping[key] + + if meth is None: + if self.hybrid.bulk_dml_setter is None: + raise exc.InvalidRequestError( + "Can't evaluate bulk DML statement; please " + "supply a bulk_dml decorated function" + ) + + meth = self.hybrid.bulk_dml_setter + + meth(self.cls, mapping, value) + + return prop + @util.non_memoized_property def property(self) -> MapperProperty[_T]: # this accessor is not normally used, however is accessed by things diff --git a/lib/sqlalchemy/ext/indexable.py b/lib/sqlalchemy/ext/indexable.py index 886069ce000..0cdbb99cc46 100644 --- a/lib/sqlalchemy/ext/indexable.py +++ b/lib/sqlalchemy/ext/indexable.py @@ -1,10 +1,9 @@ # ext/indexable.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors """Define attributes on ORM-mapped classes that have "index" attributes for columns with :class:`_types.Indexable` types. @@ -216,6 +215,7 @@ class Person(Base): >>> query = session.query(Person).filter(Person.age < 20) The above query will render: + .. sourcecode:: sql SELECT person.id, person.data @@ -223,15 +223,32 @@ class Person(Base): WHERE CAST(person.data ->> %(data_1)s AS INTEGER) < %(param_1)s """ # noqa + +from __future__ import annotations + +from typing import Any +from typing import Callable +from typing import cast +from typing import Optional +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union + from .. import inspect from ..ext.hybrid import hybrid_property from ..orm.attributes import flag_modified +if TYPE_CHECKING: + from ..sql import SQLColumnExpression + from ..sql._typing import _HasClauseElement + __all__ = ["index_property"] +_T = TypeVar("_T") + -class index_property(hybrid_property): # noqa +class index_property(hybrid_property[_T]): """A property generator. The generated property describes an object attribute that corresponds to an :class:`_types.Indexable` column. @@ -242,16 +259,16 @@ class index_property(hybrid_property): # noqa """ - _NO_DEFAULT_ARGUMENT = object() + _NO_DEFAULT_ARGUMENT = cast(_T, object()) def __init__( self, - attr_name, - index, - default=_NO_DEFAULT_ARGUMENT, - datatype=None, - mutable=True, - onebased=True, + attr_name: str, + index: Union[int, str], + default: _T = _NO_DEFAULT_ARGUMENT, + datatype: Optional[Callable[[], Any]] = None, + mutable: bool = True, + onebased: bool = True, ): """Create a new :class:`.index_property`. @@ -290,18 +307,18 @@ def __init__( self.datatype = datatype else: if is_numeric: - self.datatype = lambda: [None for x in range(index + 1)] + self.datatype = lambda: [None for x in range(index + 1)] # type: ignore[operator] # noqa: E501 else: self.datatype = dict self.onebased = onebased - def _fget_default(self, err=None): + def _fget_default(self, err: Optional[BaseException] = None) -> _T: if self.default == self._NO_DEFAULT_ARGUMENT: raise AttributeError(self.attr_name) from err else: return self.default - def fget(self, instance): + def fget(self, instance: Any, /) -> _T: attr_name = self.attr_name column_value = getattr(instance, attr_name) if column_value is None: @@ -311,9 +328,9 @@ def fget(self, instance): except (KeyError, IndexError) as err: return self._fget_default(err) else: - return value + return value # type: ignore[no-any-return] - def fset(self, instance, value): + def fset(self, instance: Any, value: _T) -> None: attr_name = self.attr_name column_value = getattr(instance, attr_name, None) if column_value is None: @@ -324,7 +341,7 @@ def fset(self, instance, value): if attr_name in inspect(instance).mapper.attrs: flag_modified(instance, attr_name) - def fdel(self, instance): + def fdel(self, instance: Any) -> None: attr_name = self.attr_name column_value = getattr(instance, attr_name) if column_value is None: @@ -337,9 +354,11 @@ def fdel(self, instance): setattr(instance, attr_name, column_value) flag_modified(instance, attr_name) - def expr(self, model): + def expr( + self, model: Any + ) -> Union[_HasClauseElement[_T], SQLColumnExpression[_T]]: column = getattr(model, self.attr_name) index = self.index if self.onebased: - index += 1 - return column[index] + index += 1 # type: ignore[operator] + return column[index] # type: ignore[no-any-return] diff --git a/lib/sqlalchemy/ext/instrumentation.py b/lib/sqlalchemy/ext/instrumentation.py index a5d991fef6f..392201dbcd3 100644 --- a/lib/sqlalchemy/ext/instrumentation.py +++ b/lib/sqlalchemy/ext/instrumentation.py @@ -1,5 +1,5 @@ # ext/instrumentation.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/ext/mutable.py b/lib/sqlalchemy/ext/mutable.py index 9ead5959be0..c2b0344d890 100644 --- a/lib/sqlalchemy/ext/mutable.py +++ b/lib/sqlalchemy/ext/mutable.py @@ -1,5 +1,5 @@ # ext/mutable.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -398,7 +398,6 @@ def __setstate__(self, state): from .. import event from .. import inspect from .. import types -from .. import util from ..orm import Mapper from ..orm._typing import _ExternalEntityType from ..orm._typing import _O @@ -416,7 +415,6 @@ def __setstate__(self, state): from ..sql.schema import Column from ..sql.type_api import TypeEngine from ..util import memoized_property -from ..util.typing import TypeGuard _KT = TypeVar("_KT") # Key type. _VT = TypeVar("_VT") # Value type. @@ -524,6 +522,7 @@ def load(state: InstanceState[_O], *args: Any) -> None: if val is not None: if coerce: val = cls.coerce(key, val) + assert val is not None state.dict[key] = val val._parents[state] = key @@ -649,8 +648,6 @@ def associate_with(cls, sqltype: type) -> None: """ def listen_for_type(mapper: Mapper[_O], class_: type) -> None: - if mapper.non_primary: - return for prop in mapper.column_attrs: if isinstance(prop.columns[0].type, sqltype): cls.associate_with_attribute(getattr(class_, prop.key)) @@ -714,8 +711,6 @@ def listen_for_type( mapper: Mapper[_T], class_: Union[DeclarativeAttributeIntercept, type], ) -> None: - if mapper.non_primary: - return _APPLIED_KEY = "_ext_mutable_listener_applied" for prop in mapper.column_attrs: @@ -813,7 +808,7 @@ class MutableDict(Mutable, Dict[_KT, _VT]): def __setitem__(self, key: _KT, value: _VT) -> None: """Detect dictionary set events and emit change events.""" - super().__setitem__(key, value) + dict.__setitem__(self, key, value) self.changed() if TYPE_CHECKING: @@ -832,17 +827,17 @@ def setdefault(self, key: _KT, value: object = None) -> object: ... else: def setdefault(self, *arg): # noqa: F811 - result = super().setdefault(*arg) + result = dict.setdefault(self, *arg) self.changed() return result def __delitem__(self, key: _KT) -> None: """Detect dictionary del events and emit change events.""" - super().__delitem__(key) + dict.__delitem__(self, key) self.changed() def update(self, *a: Any, **kw: _VT) -> None: - super().update(*a, **kw) + dict.update(self, *a, **kw) self.changed() if TYPE_CHECKING: @@ -860,17 +855,17 @@ def pop( else: def pop(self, *arg): # noqa: F811 - result = super().pop(*arg) + result = dict.pop(self, *arg) self.changed() return result def popitem(self) -> Tuple[_KT, _VT]: - result = super().popitem() + result = dict.popitem(self) self.changed() return result def clear(self) -> None: - super().clear() + dict.clear(self) self.changed() @classmethod @@ -925,38 +920,29 @@ def __reduce_ex__( def __setstate__(self, state: Iterable[_T]) -> None: self[:] = state - def is_scalar(self, value: _T | Iterable[_T]) -> TypeGuard[_T]: - return not util.is_non_string_iterable(value) - - def is_iterable(self, value: _T | Iterable[_T]) -> TypeGuard[Iterable[_T]]: - return util.is_non_string_iterable(value) - def __setitem__( self, index: SupportsIndex | slice, value: _T | Iterable[_T] ) -> None: """Detect list set events and emit change events.""" - if isinstance(index, SupportsIndex) and self.is_scalar(value): - super().__setitem__(index, value) - elif isinstance(index, slice) and self.is_iterable(value): - super().__setitem__(index, value) + list.__setitem__(self, index, value) self.changed() def __delitem__(self, index: SupportsIndex | slice) -> None: """Detect list del events and emit change events.""" - super().__delitem__(index) + list.__delitem__(self, index) self.changed() def pop(self, *arg: SupportsIndex) -> _T: - result = super().pop(*arg) + result = list.pop(self, *arg) self.changed() return result def append(self, x: _T) -> None: - super().append(x) + list.append(self, x) self.changed() def extend(self, x: Iterable[_T]) -> None: - super().extend(x) + list.extend(self, x) self.changed() def __iadd__(self, x: Iterable[_T]) -> MutableList[_T]: # type: ignore[override,misc] # noqa: E501 @@ -964,23 +950,23 @@ def __iadd__(self, x: Iterable[_T]) -> MutableList[_T]: # type: ignore[override return self def insert(self, i: SupportsIndex, x: _T) -> None: - super().insert(i, x) + list.insert(self, i, x) self.changed() def remove(self, i: _T) -> None: - super().remove(i) + list.remove(self, i) self.changed() def clear(self) -> None: - super().clear() + list.clear(self) self.changed() def sort(self, **kw: Any) -> None: - super().sort(**kw) + list.sort(self, **kw) self.changed() def reverse(self) -> None: - super().reverse() + list.reverse(self) self.changed() @classmethod @@ -1021,19 +1007,19 @@ class MutableSet(Mutable, Set[_T]): """ def update(self, *arg: Iterable[_T]) -> None: - super().update(*arg) + set.update(self, *arg) self.changed() def intersection_update(self, *arg: Iterable[Any]) -> None: - super().intersection_update(*arg) + set.intersection_update(self, *arg) self.changed() def difference_update(self, *arg: Iterable[Any]) -> None: - super().difference_update(*arg) + set.difference_update(self, *arg) self.changed() def symmetric_difference_update(self, *arg: Iterable[_T]) -> None: - super().symmetric_difference_update(*arg) + set.symmetric_difference_update(self, *arg) self.changed() def __ior__(self, other: AbstractSet[_T]) -> MutableSet[_T]: # type: ignore[override,misc] # noqa: E501 @@ -1048,29 +1034,29 @@ def __ixor__(self, other: AbstractSet[_T]) -> MutableSet[_T]: # type: ignore[ov self.symmetric_difference_update(other) return self - def __isub__(self, other: AbstractSet[object]) -> MutableSet[_T]: # type: ignore[misc] # noqa: E501 + def __isub__(self, other: AbstractSet[object]) -> MutableSet[_T]: # type: ignore[misc,unused-ignore] # noqa: E501 self.difference_update(other) return self def add(self, elem: _T) -> None: - super().add(elem) + set.add(self, elem) self.changed() def remove(self, elem: _T) -> None: - super().remove(elem) + set.remove(self, elem) self.changed() - def discard(self, elem: _T) -> None: - super().discard(elem) + def discard(self, elem: _T) -> None: # type: ignore[override,unused-ignore] # noqa: E501 + set.discard(self, elem) self.changed() def pop(self, *arg: Any) -> _T: - result = super().pop(*arg) + result = set.pop(self, *arg) self.changed() return result def clear(self) -> None: - super().clear() + set.clear(self) self.changed() @classmethod diff --git a/lib/sqlalchemy/ext/orderinglist.py b/lib/sqlalchemy/ext/orderinglist.py index 3cc67b18964..83a3e5c4629 100644 --- a/lib/sqlalchemy/ext/orderinglist.py +++ b/lib/sqlalchemy/ext/orderinglist.py @@ -1,10 +1,9 @@ # ext/orderinglist.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -# mypy: ignore-errors """A custom list that manages index/position information for contained elements. @@ -129,17 +128,24 @@ class Bullet(Base): """ from __future__ import annotations +from typing import Any from typing import Callable +from typing import Dict +from typing import Iterable from typing import List from typing import Optional +from typing import overload from typing import Sequence +from typing import SupportsIndex +from typing import Type from typing import TypeVar +from typing import Union from ..orm.collections import collection from ..orm.collections import collection_adapter _T = TypeVar("_T") -OrderingFunc = Callable[[int, Sequence[_T]], int] +OrderingFunc = Callable[[int, Sequence[_T]], object] __all__ = ["ordering_list"] @@ -148,9 +154,9 @@ class Bullet(Base): def ordering_list( attr: str, count_from: Optional[int] = None, - ordering_func: Optional[OrderingFunc] = None, + ordering_func: Optional[OrderingFunc[_T]] = None, reorder_on_append: bool = False, -) -> Callable[[], OrderingList]: +) -> Callable[[], OrderingList[_T]]: """Prepares an :class:`OrderingList` factory for use in mapper definitions. Returns an object suitable for use as an argument to a Mapper @@ -196,22 +202,22 @@ class Slide(Base): # Ordering utility functions -def count_from_0(index, collection): +def count_from_0(index: int, collection: object) -> int: """Numbering function: consecutive integers starting at 0.""" return index -def count_from_1(index, collection): +def count_from_1(index: int, collection: object) -> int: """Numbering function: consecutive integers starting at 1.""" return index + 1 -def count_from_n_factory(start): +def count_from_n_factory(start: int) -> OrderingFunc[Any]: """Numbering function: consecutive integers starting at arbitrary start.""" - def f(index, collection): + def f(index: int, collection: object) -> int: return index + start try: @@ -221,7 +227,7 @@ def f(index, collection): return f -def _unsugar_count_from(**kw): +def _unsugar_count_from(**kw: Any) -> Dict[str, Any]: """Builds counting functions from keyword arguments. Keyword argument filter, prepares a simple ``ordering_func`` from a @@ -249,13 +255,13 @@ class OrderingList(List[_T]): """ ordering_attr: str - ordering_func: OrderingFunc + ordering_func: OrderingFunc[_T] reorder_on_append: bool def __init__( self, - ordering_attr: Optional[str] = None, - ordering_func: Optional[OrderingFunc] = None, + ordering_attr: str, + ordering_func: Optional[OrderingFunc[_T]] = None, reorder_on_append: bool = False, ): """A custom list that manages position information for its children. @@ -315,10 +321,10 @@ def __init__( # More complex serialization schemes (multi column, e.g.) are possible by # subclassing and reimplementing these two methods. - def _get_order_value(self, entity): + def _get_order_value(self, entity: _T) -> Any: return getattr(entity, self.ordering_attr) - def _set_order_value(self, entity, value): + def _set_order_value(self, entity: _T, value: Any) -> None: setattr(entity, self.ordering_attr, value) def reorder(self) -> None: @@ -334,7 +340,9 @@ def reorder(self) -> None: # As of 0.5, _reorder is no longer semi-private _reorder = reorder - def _order_entity(self, index, entity, reorder=True): + def _order_entity( + self, index: int, entity: _T, reorder: bool = True + ) -> None: have = self._get_order_value(entity) # Don't disturb existing ordering if reorder is False @@ -345,34 +353,44 @@ def _order_entity(self, index, entity, reorder=True): if have != should_be: self._set_order_value(entity, should_be) - def append(self, entity): + def append(self, entity: _T) -> None: super().append(entity) self._order_entity(len(self) - 1, entity, self.reorder_on_append) - def _raw_append(self, entity): + def _raw_append(self, entity: _T) -> None: """Append without any ordering behavior.""" super().append(entity) _raw_append = collection.adds(1)(_raw_append) - def insert(self, index, entity): + def insert(self, index: SupportsIndex, entity: _T) -> None: super().insert(index, entity) self._reorder() - def remove(self, entity): + def remove(self, entity: _T) -> None: super().remove(entity) adapter = collection_adapter(self) if adapter and adapter._referenced_by_owner: self._reorder() - def pop(self, index=-1): + def pop(self, index: SupportsIndex = -1) -> _T: entity = super().pop(index) self._reorder() return entity - def __setitem__(self, index, entity): + @overload + def __setitem__(self, index: SupportsIndex, entity: _T) -> None: ... + + @overload + def __setitem__(self, index: slice, entity: Iterable[_T]) -> None: ... + + def __setitem__( + self, + index: Union[SupportsIndex, slice], + entity: Union[_T, Iterable[_T]], + ) -> None: if isinstance(index, slice): step = index.step or 1 start = index.start or 0 @@ -381,26 +399,18 @@ def __setitem__(self, index, entity): stop = index.stop or len(self) if stop < 0: stop += len(self) - + entities = list(entity) # type: ignore[arg-type] for i in range(start, stop, step): - self.__setitem__(i, entity[i]) + self.__setitem__(i, entities[i]) else: - self._order_entity(index, entity, True) - super().__setitem__(index, entity) + self._order_entity(int(index), entity, True) # type: ignore[arg-type] # noqa: E501 + super().__setitem__(index, entity) # type: ignore[assignment] - def __delitem__(self, index): + def __delitem__(self, index: Union[SupportsIndex, slice]) -> None: super().__delitem__(index) self._reorder() - def __setslice__(self, start, end, values): - super().__setslice__(start, end, values) - self._reorder() - - def __delslice__(self, start, end): - super().__delslice__(start, end) - self._reorder() - - def __reduce__(self): + def __reduce__(self) -> Any: return _reconstitute, (self.__class__, self.__dict__, list(self)) for func_name, func in list(locals().items()): @@ -414,7 +424,9 @@ def __reduce__(self): del func_name, func -def _reconstitute(cls, dict_, items): +def _reconstitute( + cls: Type[OrderingList[_T]], dict_: Dict[str, Any], items: List[_T] +) -> OrderingList[_T]: """Reconstitute an :class:`.OrderingList`. This is the adjoint to :meth:`.OrderingList.__reduce__`. It is used for diff --git a/lib/sqlalchemy/ext/serializer.py b/lib/sqlalchemy/ext/serializer.py index b7032b65959..68126dad2a1 100644 --- a/lib/sqlalchemy/ext/serializer.py +++ b/lib/sqlalchemy/ext/serializer.py @@ -1,5 +1,5 @@ # ext/serializer.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -90,9 +90,9 @@ class Serializer(pickle.Pickler): def persistent_id(self, obj): # print "serializing:", repr(obj) - if isinstance(obj, Mapper) and not obj.non_primary: + if isinstance(obj, Mapper): id_ = "mapper:" + b64encode(pickle.dumps(obj.class_)) - elif isinstance(obj, MapperProperty) and not obj.parent.non_primary: + elif isinstance(obj, MapperProperty): id_ = ( "mapperprop:" + b64encode(pickle.dumps(obj.parent.class_)) diff --git a/lib/sqlalchemy/future/__init__.py b/lib/sqlalchemy/future/__init__.py index ef9afb1a52b..3a06418649e 100644 --- a/lib/sqlalchemy/future/__init__.py +++ b/lib/sqlalchemy/future/__init__.py @@ -1,5 +1,5 @@ # future/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/future/engine.py b/lib/sqlalchemy/future/engine.py index 0449c3d9f31..e53286c455c 100644 --- a/lib/sqlalchemy/future/engine.py +++ b/lib/sqlalchemy/future/engine.py @@ -1,5 +1,5 @@ # future/engine.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/inspection.py b/lib/sqlalchemy/inspection.py index 71911671660..03dbb594127 100644 --- a/lib/sqlalchemy/inspection.py +++ b/lib/sqlalchemy/inspection.py @@ -1,5 +1,5 @@ # inspection.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -34,6 +34,7 @@ from typing import Callable from typing import Dict from typing import Generic +from typing import Literal from typing import Optional from typing import overload from typing import Protocol @@ -42,7 +43,6 @@ from typing import Union from . import exc -from .util.typing import Literal _T = TypeVar("_T", bound=Any) _TCov = TypeVar("_TCov", bound=Any, covariant=True) diff --git a/lib/sqlalchemy/log.py b/lib/sqlalchemy/log.py index b9627d879c0..7417dc80306 100644 --- a/lib/sqlalchemy/log.py +++ b/lib/sqlalchemy/log.py @@ -1,5 +1,5 @@ # log.py -# Copyright (C) 2006-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2006-2026 the SQLAlchemy authors and contributors # # Includes alterations by Vinay Sajip vinay_sajip@yahoo.co.uk # @@ -22,6 +22,7 @@ import logging import sys from typing import Any +from typing import Literal from typing import Optional from typing import overload from typing import Set @@ -30,7 +31,6 @@ from typing import Union from .util import py311 -from .util.typing import Literal STACKLEVEL = True diff --git a/lib/sqlalchemy/orm/__init__.py b/lib/sqlalchemy/orm/__init__.py index 7771de47eb2..43e1d980a1b 100644 --- a/lib/sqlalchemy/orm/__init__.py +++ b/lib/sqlalchemy/orm/__init__.py @@ -1,5 +1,5 @@ # orm/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -56,6 +56,7 @@ from .context import QueryContext as QueryContext from .decl_api import add_mapped_attribute as add_mapped_attribute from .decl_api import as_declarative as as_declarative +from .decl_api import as_typed_table as as_typed_table from .decl_api import declarative_base as declarative_base from .decl_api import declarative_mixin as declarative_mixin from .decl_api import DeclarativeBase as DeclarativeBase @@ -63,9 +64,12 @@ from .decl_api import DeclarativeMeta as DeclarativeMeta from .decl_api import declared_attr as declared_attr from .decl_api import has_inherited_table as has_inherited_table +from .decl_api import mapped_as_dataclass as mapped_as_dataclass from .decl_api import MappedAsDataclass as MappedAsDataclass from .decl_api import registry as registry from .decl_api import synonym_for as synonym_for +from .decl_api import TypeResolve as TypeResolve +from .decl_api import unmapped_dataclass as unmapped_dataclass from .decl_base import MappedClassProtocol as MappedClassProtocol from .descriptor_props import Composite as Composite from .descriptor_props import CompositeProperty as CompositeProperty @@ -77,6 +81,7 @@ from .events import InstrumentationEvents as InstrumentationEvents from .events import MapperEvents as MapperEvents from .events import QueryEvents as QueryEvents +from .events import RegistryEvents as RegistryEvents from .events import SessionEvents as SessionEvents from .identity import IdentityMap as IdentityMap from .instrumentation import ClassManager as ClassManager @@ -153,6 +158,7 @@ from .unitofwork import UOWTransaction as UOWTransaction from .util import Bundle as Bundle from .util import CascadeOptions as CascadeOptions +from .util import DictBundle as DictBundle from .util import LoaderCriteriaOption as LoaderCriteriaOption from .util import object_mapper as object_mapper from .util import polymorphic_union as polymorphic_union diff --git a/lib/sqlalchemy/orm/_orm_constructors.py b/lib/sqlalchemy/orm/_orm_constructors.py index b2acc93b43c..97a1d36ce24 100644 --- a/lib/sqlalchemy/orm/_orm_constructors.py +++ b/lib/sqlalchemy/orm/_orm_constructors.py @@ -1,5 +1,5 @@ # orm/_orm_constructors.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -8,10 +8,13 @@ from __future__ import annotations import typing +from typing import Annotated from typing import Any from typing import Callable from typing import Collection from typing import Iterable +from typing import Literal +from typing import Mapping from typing import NoReturn from typing import Optional from typing import overload @@ -46,8 +49,6 @@ from ..sql.schema import _InsertSentinelColumnDefault from ..sql.schema import SchemaConst from ..sql.selectable import FromClause -from ..util.typing import Annotated -from ..util.typing import Literal if TYPE_CHECKING: from ._typing import _EntityType @@ -136,6 +137,7 @@ def mapped_column( system: bool = False, comment: Optional[str] = None, sort_order: Union[_NoArg, int] = _NoArg.NO_ARG, + dataclass_metadata: Union[_NoArg, Mapping[Any, Any], None] = _NoArg.NO_ARG, **kw: Any, ) -> MappedColumn[Any]: r"""declare a new ORM-mapped :class:`_schema.Column` construct @@ -189,9 +191,9 @@ def mapped_column( :class:`_schema.Column`. :param nullable: Optional bool, whether the column should be "NULL" or "NOT NULL". If omitted, the nullability is derived from the type - annotation based on whether or not ``typing.Optional`` is present. - ``nullable`` defaults to ``True`` otherwise for non-primary key columns, - and ``False`` for primary key columns. + annotation based on whether or not ``typing.Optional`` (or its equivalent) + is present. ``nullable`` defaults to ``True`` otherwise for non-primary + key columns, and ``False`` for primary key columns. :param primary_key: optional bool, indicates the :class:`_schema.Column` would be part of the table's primary key or not. :param deferred: Optional bool - this keyword argument is consumed by the @@ -341,6 +343,12 @@ def mapped_column( .. versionadded:: 2.0.36 + :param dataclass_metadata: Specific to + :ref:`orm_declarative_native_dataclasses`, supplies metadata + to be attached to the generated dataclass field. + + .. versionadded:: 2.0.42 + :param \**kw: All remaining keyword arguments are passed through to the constructor for the :class:`_schema.Column`. @@ -355,7 +363,14 @@ def mapped_column( autoincrement=autoincrement, insert_default=insert_default, attribute_options=_AttributeOptions( - init, repr, default, default_factory, compare, kw_only, hash + init, + repr, + default, + default_factory, + compare, + kw_only, + hash, + dataclass_metadata, ), doc=doc, key=key, @@ -461,6 +476,7 @@ def column_property( expire_on_flush: bool = True, info: Optional[_InfoType] = None, doc: Optional[str] = None, + dataclass_metadata: Union[_NoArg, Mapping[Any, Any], None] = _NoArg.NO_ARG, ) -> MappedSQLExpression[_T]: r"""Provide a column-level property for use with a mapping. @@ -583,6 +599,12 @@ def column_property( .. versionadded:: 2.0.36 + :param dataclass_metadata: Specific to + :ref:`orm_declarative_native_dataclasses`, supplies metadata + to be attached to the generated dataclass field. + + .. versionadded:: 2.0.42 + """ return MappedSQLExpression( column, @@ -595,6 +617,7 @@ def column_property( compare, kw_only, hash, + dataclass_metadata, ), group=group, deferred=deferred, @@ -616,6 +639,7 @@ def composite( group: Optional[str] = None, deferred: bool = False, raiseload: bool = False, + return_none_on: Union[_NoArg, None, Callable[..., bool]] = _NoArg.NO_ARG, comparator_factory: Optional[Type[Composite.Comparator[_T]]] = None, active_history: bool = False, init: Union[_NoArg, bool] = _NoArg.NO_ARG, @@ -627,6 +651,7 @@ def composite( hash: Union[_NoArg, bool, None] = _NoArg.NO_ARG, # noqa: A002 info: Optional[_InfoType] = None, doc: Optional[str] = None, + dataclass_metadata: Union[_NoArg, Mapping[Any, Any], None] = _NoArg.NO_ARG, **__kw: Any, ) -> Composite[Any]: ... @@ -639,6 +664,7 @@ def composite( group: Optional[str] = None, deferred: bool = False, raiseload: bool = False, + return_none_on: Union[_NoArg, None, Callable[..., bool]] = _NoArg.NO_ARG, comparator_factory: Optional[Type[Composite.Comparator[_T]]] = None, active_history: bool = False, init: Union[_NoArg, bool] = _NoArg.NO_ARG, @@ -662,6 +688,7 @@ def composite( group: Optional[str] = None, deferred: bool = False, raiseload: bool = False, + return_none_on: Union[_NoArg, None, Callable[..., bool]] = _NoArg.NO_ARG, comparator_factory: Optional[Type[Composite.Comparator[_T]]] = None, active_history: bool = False, init: Union[_NoArg, bool] = _NoArg.NO_ARG, @@ -686,6 +713,7 @@ def composite( group: Optional[str] = None, deferred: bool = False, raiseload: bool = False, + return_none_on: Union[_NoArg, None, Callable[..., bool]] = _NoArg.NO_ARG, comparator_factory: Optional[Type[Composite.Comparator[_T]]] = None, active_history: bool = False, init: Union[_NoArg, bool] = _NoArg.NO_ARG, @@ -697,6 +725,7 @@ def composite( hash: Union[_NoArg, bool, None] = _NoArg.NO_ARG, # noqa: A002 info: Optional[_InfoType] = None, doc: Optional[str] = None, + dataclass_metadata: Union[_NoArg, Mapping[Any, Any], None] = _NoArg.NO_ARG, **__kw: Any, ) -> Composite[Any]: r"""Return a composite column-based property for use with a Mapper. @@ -726,6 +755,23 @@ def composite( scalar attribute should be loaded when replaced, if not already loaded. See the same flag on :func:`.column_property`. + :param return_none_on=None: A callable that will be evaluated when the + composite object is to be constructed, which upon returning the boolean + value ``True`` will instead bypass the construction and cause the + resulting value to be None. This typically may be assigned a lambda + that will evaluate to True when all the columns within the composite + are themselves None, e.g.:: + + composite( + MyComposite, return_none_on=lambda *cols: all(x is None for x in cols) + ) + + The above lambda for :paramref:`.composite.return_none_on` is used + automatically when using ORM Annotated Declarative along with an optional + value within the :class:`.Mapped` annotation. + + .. versionadded:: 2.1 + :param group: A group name for this property when marked as deferred. @@ -775,15 +821,31 @@ def composite( class. .. versionadded:: 2.0.36 - """ + + :param dataclass_metadata: Specific to + :ref:`orm_declarative_native_dataclasses`, supplies metadata + to be attached to the generated dataclass field. + + .. versionadded:: 2.0.42 + + """ # noqa: E501 + if __kw: raise _no_kw() return Composite( _class_or_attr, *attrs, + return_none_on=return_none_on, attribute_options=_AttributeOptions( - init, repr, default, default_factory, compare, kw_only, hash + init, + repr, + default, + default_factory, + compare, + kw_only, + hash, + dataclass_metadata, ), group=group, deferred=deferred, @@ -1037,6 +1099,7 @@ def relationship( info: Optional[_InfoType] = None, omit_join: Literal[None, False] = None, sync_backref: Optional[bool] = None, + dataclass_metadata: Union[_NoArg, Mapping[Any, Any], None] = _NoArg.NO_ARG, **kw: Any, ) -> _RelationshipDeclared[Any]: """Provide a relationship between two mapped classes. @@ -1795,8 +1858,6 @@ class that will be synchronized with this one. It is usually default, changes in state will be back-populated only if neither sides of a relationship is viewonly. - .. versionadded:: 1.3.17 - .. versionchanged:: 1.4 - A relationship that specifies :paramref:`_orm.relationship.viewonly` automatically implies that :paramref:`_orm.relationship.sync_backref` is ``False``. @@ -1816,10 +1877,16 @@ class that will be synchronized with this one. It is usually automatically detected; if it is not detected, then the optimization is not supported. - .. versionchanged:: 1.3.11 setting ``omit_join`` to True will now - emit a warning as this was not the intended use of this flag. + :param default: Specific to :ref:`orm_declarative_native_dataclasses`, + specifies an immutable scalar default value for the relationship that + will behave as though it is the default value for the parameter in the + ``__init__()`` method. This is only supported for a ``uselist=False`` + relationship, that is many-to-one or one-to-one, and only supports the + scalar value ``None``, since no other immutable value is valid for such a + relationship. - .. versionadded:: 1.3 + .. versionchanged:: 2.1 the :paramref:`_orm.relationship.default` + parameter only supports a value of ``None``. :param init: Specific to :ref:`orm_declarative_native_dataclasses`, specifies if the mapped attribute should be part of the ``__init__()`` @@ -1849,6 +1916,13 @@ class that will be synchronized with this one. It is usually class. .. versionadded:: 2.0.36 + + :param dataclass_metadata: Specific to + :ref:`orm_declarative_native_dataclasses`, supplies metadata + to be attached to the generated dataclass field. + + .. versionadded:: 2.0.42 + """ return _RelationshipDeclared( @@ -1866,7 +1940,14 @@ class that will be synchronized with this one. It is usually cascade=cascade, viewonly=viewonly, attribute_options=_AttributeOptions( - init, repr, default, default_factory, compare, kw_only, hash + init, + repr, + default, + default_factory, + compare, + kw_only, + hash, + dataclass_metadata, ), lazy=lazy, passive_deletes=passive_deletes, @@ -1904,6 +1985,7 @@ def synonym( hash: Union[_NoArg, bool, None] = _NoArg.NO_ARG, # noqa: A002 info: Optional[_InfoType] = None, doc: Optional[str] = None, + dataclass_metadata: Union[_NoArg, Mapping[Any, Any], None] = _NoArg.NO_ARG, ) -> Synonym[Any]: """Denote an attribute name as a synonym to a mapped property, in that the attribute will mirror the value and expression behavior @@ -2017,7 +2099,14 @@ def _job_status_descriptor(self): descriptor=descriptor, comparator_factory=comparator_factory, attribute_options=_AttributeOptions( - init, repr, default, default_factory, compare, kw_only, hash + init, + repr, + default, + default_factory, + compare, + kw_only, + hash, + dataclass_metadata, ), doc=doc, info=info, @@ -2152,6 +2241,7 @@ def deferred( expire_on_flush: bool = True, info: Optional[_InfoType] = None, doc: Optional[str] = None, + dataclass_metadata: Union[_NoArg, Mapping[Any, Any], None] = _NoArg.NO_ARG, ) -> MappedSQLExpression[_T]: r"""Indicate a column-based mapped attribute that by default will not load unless accessed. @@ -2182,7 +2272,14 @@ def deferred( column, *additional_columns, attribute_options=_AttributeOptions( - init, repr, default, default_factory, compare, kw_only, hash + init, + repr, + default, + default_factory, + compare, + kw_only, + hash, + dataclass_metadata, ), group=group, deferred=True, @@ -2209,8 +2306,6 @@ def query_expression( :param default_expr: Optional SQL expression object that will be used in all cases if not assigned later with :func:`_orm.with_expression`. - .. versionadded:: 1.2 - .. seealso:: :ref:`orm_queryguide_with_expression` - background and usage examples @@ -2226,6 +2321,7 @@ def query_expression( compare, _NoArg.NO_ARG, _NoArg.NO_ARG, + _NoArg.NO_ARG, ), expire_on_flush=expire_on_flush, info=info, @@ -2254,7 +2350,7 @@ def clear_mappers() -> None: are never discarded independently of their class. If a mapped class itself is garbage collected, its mapper is automatically disposed of as well. As such, :func:`.clear_mappers` is only for usage in test suites - that re-use the same classes with different mappings, which is itself an + that reuse the same classes with different mappings, which is itself an extremely rare use case - the only such use case is in fact SQLAlchemy's own test suite, and possibly the test suites of other ORM extension libraries which intend to test various combinations of mapper construction diff --git a/lib/sqlalchemy/orm/_typing.py b/lib/sqlalchemy/orm/_typing.py index 8cf5335d67d..ab9518beb48 100644 --- a/lib/sqlalchemy/orm/_typing.py +++ b/lib/sqlalchemy/orm/_typing.py @@ -1,5 +1,5 @@ # orm/_typing.py -# Copyright (C) 2022-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2022-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -16,6 +16,7 @@ from typing import Tuple from typing import Type from typing import TYPE_CHECKING +from typing import TypeGuard from typing import TypeVar from typing import Union @@ -27,7 +28,6 @@ ) from ..sql._typing import _HasClauseElement from ..sql.elements import ColumnElement -from ..util.typing import TypeGuard if TYPE_CHECKING: from .attributes import _AttributeImpl @@ -50,9 +50,6 @@ _T = TypeVar("_T", bound=Any) - -_T_co = TypeVar("_T_co", bound=Any, covariant=True) - _O = TypeVar("_O", bound=object) """The 'ORM mapped object' type. diff --git a/lib/sqlalchemy/orm/attributes.py b/lib/sqlalchemy/orm/attributes.py index 85ef9746fda..97c8eb72a4f 100644 --- a/lib/sqlalchemy/orm/attributes.py +++ b/lib/sqlalchemy/orm/attributes.py @@ -1,5 +1,5 @@ # orm/attributes.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -26,6 +26,7 @@ from typing import Dict from typing import Iterable from typing import List +from typing import Literal from typing import NamedTuple from typing import Optional from typing import overload @@ -33,6 +34,7 @@ from typing import Tuple from typing import Type from typing import TYPE_CHECKING +from typing import TypeGuard from typing import TypeVar from typing import Union @@ -45,6 +47,7 @@ from .base import ATTR_WAS_SET from .base import CALLABLES_OK from .base import DEFERRED_HISTORY_LOAD +from .base import DONT_SET from .base import INCLUDE_PENDING_MUTATIONS # noqa from .base import INIT_OK from .base import instance_dict as instance_dict @@ -89,9 +92,7 @@ from ..sql.cache_key import HasCacheKey from ..sql.visitors import _TraverseInternalsType from ..sql.visitors import InternalTraversal -from ..util.typing import Literal from ..util.typing import Self -from ..util.typing import TypeGuard if TYPE_CHECKING: from ._typing import _EntityType @@ -391,6 +392,11 @@ def _bulk_update_tuples( return self.comparator._bulk_update_tuples(value) + def _bulk_dml_setter(self, key: str) -> Optional[Callable[..., Any]]: + """return a callable that will process a bulk INSERT value""" + + return self.comparator._bulk_dml_setter(key) + def adapt_to_entity(self, adapt_to_entity: AliasedInsp[Any]) -> Self: assert not self._of_type return self.__class__( @@ -462,6 +468,9 @@ def hasparent( ) -> bool: return self.impl.hasparent(state, optimistic=optimistic) is not False + def _column_strategy_attrs(self) -> Sequence[QueryableAttribute[Any]]: + return (self,) + def __getattr__(self, key: str) -> Any: try: return util.MemoizedSlots.__getattr__(self, key) @@ -595,7 +604,7 @@ def _create_proxied_attribute( # TODO: can move this to descriptor_props if the need for this # function is removed from ext/hybrid.py - class Proxy(QueryableAttribute[Any]): + class Proxy(QueryableAttribute[_T_co]): """Presents the :class:`.QueryableAttribute` interface as a proxy on top of a Python descriptor / :class:`.PropComparator` combination. @@ -610,13 +619,13 @@ class Proxy(QueryableAttribute[Any]): def __init__( self, - class_, - key, - descriptor, - comparator, - adapt_to_entity=None, - doc=None, - original_property=None, + class_: _ExternalEntityType[Any], + key: str, + descriptor: Any, + comparator: interfaces.PropComparator[_T_co], + adapt_to_entity: Optional[AliasedInsp[Any]] = None, + doc: Optional[str] = None, + original_property: Optional[QueryableAttribute[_T_co]] = None, ): self.class_ = class_ self.key = key @@ -627,11 +636,11 @@ def __init__( self._doc = self.__doc__ = doc @property - def _parententity(self): + def _parententity(self): # type: ignore[override] return inspection.inspect(self.class_, raiseerr=False) @property - def parent(self): + def parent(self): # type: ignore[override] return inspection.inspect(self.class_, raiseerr=False) _is_internal_proxy = True @@ -641,6 +650,13 @@ def parent(self): ("_parententity", visitors.ExtendedInternalTraversal.dp_multi), ] + def _column_strategy_attrs(self) -> Sequence[QueryableAttribute[Any]]: + prop = self.original_property + if prop is None: + return () + else: + return prop._column_strategy_attrs() + @property def _impl_uses_objects(self): return ( @@ -1045,20 +1061,9 @@ def get_all_pending( def _default_value( self, state: InstanceState[Any], dict_: _InstanceDict ) -> Any: - """Produce an empty value for an uninitialized scalar attribute.""" + """Produce an empty value for an uninitialized attribute.""" - assert self.key not in dict_, ( - "_default_value should only be invoked for an " - "uninitialized or expired attribute" - ) - - value = None - for fn in self.dispatch.init_scalar: - ret = fn(state, value, dict_) - if ret is not ATTR_EMPTY: - value = ret - - return value + raise NotImplementedError() def get( self, @@ -1211,15 +1216,38 @@ class _ScalarAttributeImpl(_AttributeImpl): collection = False dynamic = False - __slots__ = "_replace_token", "_append_token", "_remove_token" + __slots__ = ( + "_default_scalar_value", + "_replace_token", + "_append_token", + "_remove_token", + ) - def __init__(self, *arg, **kw): + def __init__(self, *arg, default_scalar_value=None, **kw): super().__init__(*arg, **kw) + self._default_scalar_value = default_scalar_value self._replace_token = self._append_token = AttributeEventToken( self, OP_REPLACE ) self._remove_token = AttributeEventToken(self, OP_REMOVE) + def _default_value( + self, state: InstanceState[Any], dict_: _InstanceDict + ) -> Any: + """Produce an empty value for an uninitialized scalar attribute.""" + + assert self.key not in dict_, ( + "_default_value should only be invoked for an " + "uninitialized or expired attribute" + ) + value = self._default_scalar_value + for fn in self.dispatch.init_scalar: + ret = fn(state, value, dict_) + if ret is not ATTR_EMPTY: + value = ret + + return value + def delete(self, state: InstanceState[Any], dict_: _InstanceDict) -> None: if self.dispatch._active_history: old = self.get(state, dict_, PASSIVE_RETURN_NO_VALUE) @@ -1268,6 +1296,9 @@ def set( check_old: Optional[object] = None, pop: bool = False, ) -> None: + if value is DONT_SET: + return + if self.dispatch._active_history: old = self.get(state, dict_, PASSIVE_RETURN_NO_VALUE) else: @@ -1434,6 +1465,9 @@ def set( ) -> None: """Set a value on the given InstanceState.""" + if value is DONT_SET: + return + if self.dispatch._active_history: old = self.get( state, @@ -1918,6 +1952,10 @@ def set( pop: bool = False, _adapt: bool = True, ) -> None: + + if value is DONT_SET: + return + iterable = orig_iterable = value new_keys = None @@ -1925,33 +1963,32 @@ def set( # not trigger a lazy load of the old collection. new_collection, user_data = self._initialize_collection(state) if _adapt: - if new_collection._converter is not None: - iterable = new_collection._converter(iterable) - else: - setting_type = util.duck_type_collection(iterable) - receiving_type = self._duck_typed_as - - if setting_type is not receiving_type: - given = ( - iterable is None - and "None" - or iterable.__class__.__name__ - ) - wanted = self._duck_typed_as.__name__ - raise TypeError( - "Incompatible collection type: %s is not %s-like" - % (given, wanted) - ) + setting_type = util.duck_type_collection(iterable) + receiving_type = self._duck_typed_as - # If the object is an adapted collection, return the (iterable) - # adapter. - if hasattr(iterable, "_sa_iterator"): - iterable = iterable._sa_iterator() - elif setting_type is dict: - new_keys = list(iterable) - iterable = iterable.values() - else: - iterable = iter(iterable) + if setting_type is not receiving_type: + given = ( + "None" if iterable is None else iterable.__class__.__name__ + ) + wanted = ( + "None" + if self._duck_typed_as is None + else self._duck_typed_as.__name__ + ) + raise TypeError( + "Incompatible collection type: %s is not %s-like" + % (given, wanted) + ) + + # If the object is an adapted collection, return the (iterable) + # adapter. + if hasattr(iterable, "_sa_iterator"): + iterable = iterable._sa_iterator() + elif setting_type is dict: + new_keys = list(iterable) + iterable = iterable.values() + else: + iterable = iter(iterable) elif util.duck_type_collection(iterable) is dict: new_keys = list(value) @@ -2707,7 +2744,7 @@ def init_state_collection( return adapter -def set_committed_value(instance, key, value): +def set_committed_value(instance: object, key: str, value: Any) -> None: """Set the value of an attribute with no history events. Cancels any previous history present. The value should be @@ -2753,8 +2790,6 @@ def set_attribute( is being supplied; the object may be used to track the origin of the chain of events. - .. versionadded:: 1.2.3 - """ state, dict_ = instance_state(instance), instance_dict(instance) state.manager[key].impl.set(state, dict_, value, initiator) @@ -2823,8 +2858,6 @@ def flag_dirty(instance: object) -> None: may establish changes on it, which will then be included in the SQL emitted. - .. versionadded:: 1.2 - .. seealso:: :func:`.attributes.flag_modified` diff --git a/lib/sqlalchemy/orm/base.py b/lib/sqlalchemy/orm/base.py index ae0ba1029d1..5f4268f7059 100644 --- a/lib/sqlalchemy/orm/base.py +++ b/lib/sqlalchemy/orm/base.py @@ -1,13 +1,11 @@ # orm/base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -"""Constants and rudimental functions used throughout the ORM. - -""" +"""Constants and rudimental functions used throughout the ORM.""" from __future__ import annotations @@ -18,6 +16,7 @@ from typing import Callable from typing import Dict from typing import Generic +from typing import Literal from typing import no_type_check from typing import Optional from typing import overload @@ -28,16 +27,18 @@ from typing import Union from . import exc +from ._typing import _O from ._typing import insp_is_mapper from .. import exc as sa_exc from .. import inspection from .. import util from ..sql import roles +from ..sql._typing import _T +from ..sql._typing import _T_co from ..sql.elements import SQLColumnExpression from ..sql.elements import SQLCoreOperations from ..util import FastIntFlag from ..util.langhelpers import TypingOnly -from ..util.typing import Literal if typing.TYPE_CHECKING: from ._typing import _EntityType @@ -48,18 +49,16 @@ from .instrumentation import ClassManager from .interfaces import PropComparator from .mapper import Mapper + from .properties import MappedColumn from .state import InstanceState from .util import AliasedClass from .writeonly import WriteOnlyCollection + from ..sql._annotated_cols import TypedColumns from ..sql._typing import _ColumnExpressionArgument from ..sql._typing import _InfoType from ..sql.elements import ColumnElement from ..sql.operators import OperatorType - -_T = TypeVar("_T", bound=Any) -_T_co = TypeVar("_T_co", bound=Any, covariant=True) - -_O = TypeVar("_O", bound=object) + from ..sql.schema import Column class LoaderCallableStatus(Enum): @@ -97,6 +96,8 @@ class LoaderCallableStatus(Enum): """ + DONT_SET = 5 + ( PASSIVE_NO_RESULT, @@ -104,6 +105,7 @@ class LoaderCallableStatus(Enum): ATTR_WAS_SET, ATTR_EMPTY, NO_VALUE, + DONT_SET, ) = tuple(LoaderCallableStatus) NEVER_SET = NO_VALUE @@ -435,7 +437,7 @@ def _inspect_mapped_object(instance: _T) -> Optional[InstanceState[_T]]: def _class_to_mapper( - class_or_mapper: Union[Mapper[_T], Type[_T]] + class_or_mapper: Union[Mapper[_T], Type[_T]], ) -> Mapper[_T]: # can't get mypy to see an overload for this insp = inspection.inspect(class_or_mapper, False) @@ -447,7 +449,7 @@ def _class_to_mapper( def _mapper_or_none( - entity: Union[Type[_T], _InternalEntityType[_T]] + entity: Union[Type[_T], _InternalEntityType[_T]], ) -> Optional[Mapper[_T]]: """Return the :class:`_orm.Mapper` for the given class or None if the class is not mapped. @@ -620,11 +622,7 @@ class InspectionAttr: """ _is_internal_proxy = False - """True if this object is an internal proxy object. - - .. versionadded:: 1.2.12 - - """ + """True if this object is an internal proxy object.""" is_clause_element = False """True if this object is an instance of @@ -807,6 +805,11 @@ class Mapped( if typing.TYPE_CHECKING: + @overload + def __get__( # type: ignore[misc] + self: MappedColumn[_T_co], instance: TypedColumns, owner: Any + ) -> Column[_T_co]: ... + @overload def __get__( self, instance: None, owner: Any @@ -817,7 +820,7 @@ def __get__(self, instance: object, owner: Any) -> _T_co: ... def __get__( self, instance: Optional[object], owner: Any - ) -> Union[InstrumentedAttribute[_T_co], _T_co]: ... + ) -> Union[InstrumentedAttribute[_T_co], Column[_T_co], _T_co]: ... @classmethod def _empty_constructor(cls, arg1: Any) -> Mapped[_T_co]: ... diff --git a/lib/sqlalchemy/orm/bulk_persistence.py b/lib/sqlalchemy/orm/bulk_persistence.py index ce2efcebce7..11d3e8c9b53 100644 --- a/lib/sqlalchemy/orm/bulk_persistence.py +++ b/lib/sqlalchemy/orm/bulk_persistence.py @@ -1,5 +1,5 @@ # orm/bulk_persistence.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -18,6 +18,7 @@ from typing import cast from typing import Dict from typing import Iterable +from typing import Literal from typing import Optional from typing import overload from typing import TYPE_CHECKING @@ -35,6 +36,7 @@ from .context import _ORMFromStatementCompileState from .context import FromStatement from .context import QueryContext +from .interfaces import PropComparator from .. import exc as sa_exc from .. import util from ..engine import Dialect @@ -52,7 +54,6 @@ from ..sql.dml import InsertDMLState from ..sql.dml import UpdateDMLState from ..util import EMPTY_DICT -from ..util.typing import Literal from ..util.typing import TupleAny from ..util.typing import Unpack @@ -150,7 +151,7 @@ def _bulk_insert( # for all other cases we need to establish a local dictionary # so that the incoming dictionaries aren't mutated mappings = [dict(m) for m in mappings] - _expand_composites(mapper, mappings) + _expand_other_attrs(mapper, mappings) connection = session_transaction.connection(base_mapper) @@ -309,7 +310,7 @@ def _changed_dict(mapper, state): mappings = [state.dict for state in mappings] else: mappings = [dict(m) for m in mappings] - _expand_composites(mapper, mappings) + _expand_other_attrs(mapper, mappings) if session_transaction.session.connection_callable: raise NotImplementedError( @@ -371,19 +372,32 @@ def _changed_dict(mapper, state): return _result.null_result() -def _expand_composites(mapper, mappings): - composite_attrs = mapper.composites - if not composite_attrs: - return +def _expand_other_attrs( + mapper: Mapper[Any], mappings: Iterable[Dict[str, Any]] +) -> None: + all_attrs = mapper.all_orm_descriptors + + attr_keys = set(all_attrs.keys()) - composite_keys = set(composite_attrs.keys()) - populators = { - key: composite_attrs[key]._populate_composite_bulk_save_mappings_fn() - for key in composite_keys + bulk_dml_setters = { + key: setter + for key, setter in ( + (key, attr._bulk_dml_setter(key)) + for key, attr in ( + (key, _entity_namespace_key(mapper, key, default=NO_VALUE)) + for key in attr_keys + ) + if attr is not NO_VALUE and isinstance(attr, PropComparator) + ) + if setter is not None } + setters_todo = set(bulk_dml_setters) + if not setters_todo: + return + for mapping in mappings: - for key in composite_keys.intersection(mapping): - populators[key](mapping) + for key in setters_todo.intersection(mapping): + bulk_dml_setters[key](mapping) class _ORMDMLState(_AbstractORMCompileState): @@ -401,7 +415,7 @@ def _get_orm_crud_kv_pairs( if isinstance(k, str): desc = _entity_namespace_key(mapper, k, default=NO_VALUE) - if desc is NO_VALUE: + if not isinstance(desc, PropComparator): yield ( coercions.expect(roles.DMLColumnRole, k), ( @@ -426,6 +440,7 @@ def _get_orm_crud_kv_pairs( attr = _entity_namespace_key( k_anno["entity_namespace"], k_anno["proxy_key"] ) + assert isinstance(attr, PropComparator) yield from core_get_crud_kv_pairs( statement, attr._bulk_update_tuples(v), @@ -446,11 +461,24 @@ def _get_orm_crud_kv_pairs( ), ) + @classmethod + def _get_dml_plugin_subject(cls, statement): + plugin_subject = statement.table._propagate_attrs.get("plugin_subject") + + if ( + not plugin_subject + or not plugin_subject.mapper + or plugin_subject + is not statement._propagate_attrs["plugin_subject"] + ): + return None + return plugin_subject + @classmethod def _get_multi_crud_kv_pairs(cls, statement, kv_iterator): - plugin_subject = statement._propagate_attrs["plugin_subject"] + plugin_subject = cls._get_dml_plugin_subject(statement) - if not plugin_subject or not plugin_subject.mapper: + if not plugin_subject: return UpdateDMLState._get_multi_crud_kv_pairs( statement, kv_iterator ) @@ -470,13 +498,12 @@ def _get_crud_kv_pairs(cls, statement, kv_iterator, needs_to_be_cacheable): needs_to_be_cacheable ), "no test coverage for needs_to_be_cacheable=False" - plugin_subject = statement._propagate_attrs["plugin_subject"] + plugin_subject = cls._get_dml_plugin_subject(statement) - if not plugin_subject or not plugin_subject.mapper: + if not plugin_subject: return UpdateDMLState._get_crud_kv_pairs( statement, kv_iterator, needs_to_be_cacheable ) - return list( cls._get_orm_crud_kv_pairs( plugin_subject.mapper, @@ -782,6 +809,7 @@ def orm_pre_session_exec( util.immutabledict(execution_options).union( {"_sa_orm_update_options": update_options} ), + params, ) @classmethod @@ -1046,8 +1074,6 @@ def _do_pre_synchronize_evaluate( def _get_resolved_values(cls, mapper, statement): if statement._multi_values: return [] - elif statement._ordered_values: - return list(statement._ordered_values) elif statement._values: return list(statement._values.items()) else: @@ -1099,7 +1125,7 @@ def _do_pre_synchronize_fetch( # call can_use_returning() before invoking the statement and get # answer?, why does this go through the whole execute phase using an # event? Answer: because we are integrating with extensions such - # as the horizontal sharding extention that "multiplexes" an individual + # as the horizontal sharding extension that "multiplexes" an individual # statement run through multiple engines, and it uses # do_orm_execute() to do that. @@ -1210,7 +1236,7 @@ def orm_pre_session_exec( # for ORM object loading, like ORMContext, we have to disable # result set adapt_to_context, because we will be generating a # new statement with specific columns that's cached inside of - # an ORMFromStatementCompileState, which we will re-use for + # an ORMFromStatementCompileState, which we will reuse for # each result. if not execution_options: execution_options = context._orm_load_exec_options @@ -1231,6 +1257,7 @@ def orm_pre_session_exec( util.immutabledict(execution_options).union( {"_sa_orm_insert_options": insert_options} ), + params, ) @classmethod @@ -1468,9 +1495,7 @@ def _setup_for_orm_update(self, statement, compiler, **kw): # are passed through to the new statement, which will then raise # InvalidRequestError because UPDATE doesn't support multi_values # right now. - if statement._ordered_values: - new_stmt._ordered_values = self._resolved_values - elif statement._values: + if statement._values: new_stmt._values = self._resolved_values new_crit = self._adjust_for_extra_criteria( @@ -1557,7 +1582,7 @@ def _setup_for_bulk_update(self, statement, compiler, **kw): UpdateDMLState.__init__(self, statement, compiler, **kw) - if self._ordered_values: + if self._maintain_values_ordering: raise sa_exc.InvalidRequestError( "bulk ORM UPDATE does not support ordered_values() for " "custom UPDATE statements with bulk parameter sets. Use a " diff --git a/lib/sqlalchemy/orm/clsregistry.py b/lib/sqlalchemy/orm/clsregistry.py index 9dd2ab954a2..14c888fb9a9 100644 --- a/lib/sqlalchemy/orm/clsregistry.py +++ b/lib/sqlalchemy/orm/clsregistry.py @@ -1,5 +1,5 @@ # orm/clsregistry.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -72,7 +72,7 @@ def _add_class( # class already exists. existing = decl_class_registry[classname] if not isinstance(existing, _MultipleClassMarker): - existing = decl_class_registry[classname] = _MultipleClassMarker( + decl_class_registry[classname] = _MultipleClassMarker( [cls, cast("Type[Any]", existing)] ) else: @@ -317,7 +317,7 @@ def add_class(self, name: str, cls: Type[Any]) -> None: else: raise else: - existing = self.contents[name] = _MultipleClassMarker( + self.contents[name] = _MultipleClassMarker( [cls], on_remove=lambda: self._remove_item(name) ) diff --git a/lib/sqlalchemy/orm/collections.py b/lib/sqlalchemy/orm/collections.py index c765f59d3cf..4425962379d 100644 --- a/lib/sqlalchemy/orm/collections.py +++ b/lib/sqlalchemy/orm/collections.py @@ -1,5 +1,5 @@ # orm/collections.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -179,7 +179,6 @@ class _AdaptedCollectionProtocol(Protocol): _sa_appender: Callable[..., Any] _sa_remover: Callable[..., Any] _sa_iterator: Callable[..., Iterable[Any]] - _sa_converter: _CollectionConverterProtocol class collection: @@ -187,7 +186,7 @@ class collection: The decorators fall into two groups: annotations and interception recipes. - The annotating decorators (appender, remover, iterator, converter, + The annotating decorators (appender, remover, iterator, internally_instrumented) indicate the method's purpose and take no arguments. They are not written with parens:: @@ -319,47 +318,7 @@ def extend(self, items): ... return fn @staticmethod - @util.deprecated( - "1.3", - "The :meth:`.collection.converter` handler is deprecated and will " - "be removed in a future release. Please refer to the " - ":class:`.AttributeEvents.bulk_replace` listener interface in " - "conjunction with the :func:`.event.listen` function.", - ) - def converter(fn): - """Tag the method as the collection converter. - - This optional method will be called when a collection is being - replaced entirely, as in:: - - myobj.acollection = [newvalue1, newvalue2] - - The converter method will receive the object being assigned and should - return an iterable of values suitable for use by the ``appender`` - method. A converter must not assign values or mutate the collection, - its sole job is to adapt the value the user provides into an iterable - of values for the ORM's use. - - The default converter implementation will use duck-typing to do the - conversion. A dict-like collection will be convert into an iterable - of dictionary values, and other types will simply be iterated:: - - @collection.converter - def convert(self, other): ... - - If the duck-typing of the object does not match the type of this - collection, a TypeError is raised. - - Supply an implementation of this method if you want to expand the - range of possible types that can be assigned in bulk or perform - validation on the values about to be assigned. - - """ - fn._sa_instrument_role = "converter" - return fn - - @staticmethod - def adds(arg): + def adds(arg: int) -> Callable[[_FN], _FN]: """Mark the method as adding an entity to the collection. Adds "add to collection" handling to the method. The decorator @@ -478,7 +437,6 @@ class CollectionAdapter: "_key", "_data", "owner_state", - "_converter", "invalidated", "empty", ) @@ -490,7 +448,6 @@ class CollectionAdapter: _data: Callable[..., _AdaptedCollectionProtocol] owner_state: InstanceState[Any] - _converter: _CollectionConverterProtocol invalidated: bool empty: bool @@ -512,7 +469,6 @@ def __init__( self.owner_state = owner_state data._sa_adapter = self - self._converter = data._sa_converter self.invalidated = False self.empty = False @@ -770,7 +726,6 @@ def __setstate__(self, d): # see note in constructor regarding this type: ignore self._data = weakref.ref(d["data"]) # type: ignore - self._converter = d["data"]._sa_converter d["data"]._sa_adapter = self self.invalidated = d["invalidated"] self.attr = getattr(d["owner_cls"], self._key).impl @@ -905,12 +860,7 @@ def _locate_roles_and_methods(cls): # note role declarations if hasattr(method, "_sa_instrument_role"): role = method._sa_instrument_role - assert role in ( - "appender", - "remover", - "iterator", - "converter", - ) + assert role in ("appender", "remover", "iterator") roles.setdefault(role, name) # transfer instrumentation requests from decorated function @@ -1009,8 +959,6 @@ def _set_collection_attributes(cls, roles, methods): cls._sa_adapter = None - if not hasattr(cls, "_sa_converter"): - cls._sa_converter = None cls._sa_instrumented = id(cls) diff --git a/lib/sqlalchemy/orm/context.py b/lib/sqlalchemy/orm/context.py index bc25eff636b..d5c20f0d008 100644 --- a/lib/sqlalchemy/orm/context.py +++ b/lib/sqlalchemy/orm/context.py @@ -1,5 +1,5 @@ # orm/context.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -54,6 +54,7 @@ from ..sql.base import CacheableOptions from ..sql.base import CompileState from ..sql.base import Executable +from ..sql.base import ExecutableStatement from ..sql.base import Generative from ..sql.base import Options from ..sql.dml import UpdateBase @@ -72,7 +73,6 @@ from ..util.typing import TypeVarTuple from ..util.typing import Unpack - if TYPE_CHECKING: from ._typing import _InternalEntityType from ._typing import OrmExecuteOptionsParameter @@ -98,9 +98,6 @@ _Ts = TypeVarTuple("_Ts") _path_registry = PathRegistry.root -_EMPTY_DICT = util.immutabledict() - - LABEL_STYLE_LEGACY_ORM = SelectLabelStyle.LABEL_STYLE_LEGACY_ORM @@ -173,8 +170,8 @@ def __init__( bind_arguments: Optional[_BindArguments] = None, ): self.load_options = load_options - self.execution_options = execution_options or _EMPTY_DICT - self.bind_arguments = bind_arguments or _EMPTY_DICT + self.execution_options = execution_options or util.EMPTY_DICT + self.bind_arguments = bind_arguments or util.EMPTY_DICT self.compile_state = compile_state self.query = statement @@ -240,7 +237,7 @@ def _init_global_attributes( if compiler is None: # this is the legacy / testing only ORM _compile_state() use case. # there is no need to apply criteria options for this. - self.global_attributes = ga = {} + self.global_attributes = {} assert toplevel return else: @@ -367,7 +364,7 @@ def orm_pre_session_exec( if not is_pre_event and load_options._autoflush: session._autoflush() - return statement, execution_options + return statement, execution_options, params @classmethod def orm_setup_cursor_result( @@ -589,7 +586,7 @@ def orm_pre_session_exec( if not is_pre_event and load_options._autoflush: session._autoflush() - return statement, execution_options + return statement, execution_options, params @classmethod def orm_setup_cursor_result( @@ -783,8 +780,8 @@ class _ORMFromStatementCompileState(_ORMCompileState): eager_adding_joins = False compound_eager_adapter = None - extra_criteria_entities = _EMPTY_DICT - eager_joins = _EMPTY_DICT + extra_criteria_entities = util.EMPTY_DICT + eager_joins = util.EMPTY_DICT @classmethod def _create_orm_context( @@ -879,10 +876,11 @@ def _create_orm_context( self.order_by = None - if isinstance(self.statement, expression.TextClause): - # TextClause has no "column" objects at all. for this case, - # we generate columns from our _QueryEntity objects, then - # flip on all the "please match no matter what" parameters. + if self.statement._is_text_clause: + # AbstractTextClause (TextClause, TString) has no "column" + # objects at all. for this case, we generate columns from our + # _QueryEntity objects, then flip on all the + # "please match no matter what" parameters. self.extra_criteria_entities = {} for entity in self._entities: @@ -981,7 +979,7 @@ class FromStatement(GroupedElement, Generative, TypedReturnsRows[Unpack[_Ts]]): _traverse_internals = [ ("_raw_columns", InternalTraversal.dp_clauseelement_list), ("element", InternalTraversal.dp_clauseelement), - ] + Executable._executable_traverse_internals + ] + ExecutableStatement._executable_traverse_internals _cache_key_traversal = _traverse_internals + [ ("_compile_options", InternalTraversal.dp_has_cache_key) @@ -1087,7 +1085,7 @@ class _CompoundSelectCompileState( class _ORMSelectCompileState(_ORMCompileState, SelectState): _already_joined_edges = () - _memoized_entities = _EMPTY_DICT + _memoized_entities = util.EMPTY_DICT _from_obj_alias = None _has_mapper_entities = False @@ -1127,7 +1125,7 @@ def _create_orm_context( # query, and at the moment subqueryloader is putting some things # in here that we explicitly don't want stuck in a cache. self.select_statement = select_statement._clone() - self.select_statement._execution_options = util.immutabledict() + self.select_statement._execution_options = util.EMPTY_DICT else: self.select_statement = select_statement @@ -1405,11 +1403,7 @@ def _setup_for_generate(self): if self.order_by is False: self.order_by = None - if ( - self.multi_row_eager_loaders - and self.eager_adding_joins - and self._should_nest_selectable - ): + if self._should_nest_selectable: self.statement = self._compound_eager_statement() else: self.statement = self._simple_statement() @@ -1450,10 +1444,62 @@ def _create_entities_collection(cls, query, legacy): return self @classmethod - def determine_last_joined_entity(cls, statement): - setup_joins = statement._setup_joins + def _get_filter_by_entities(cls, statement): + """Return all ORM entities for filter_by() searches. + + the ORM version for Select is special vs. update/delete since it needs + to navigate along select.join() paths which have ORM specific + directives. + + beyond that, it delivers other entities as the Mapper or Aliased + object rather than the Table or Alias, which mostly affects + how error messages regarding ambiguous entities or entity not + found are rendered; class-specific attributes like hybrid, + column_property() etc. work either way since + _entity_namespace_key_search_all() uses _entity_namespace(). + + DML Update and Delete objects, even though they also have filter_by() + and also accept ORM objects, don't use this routine since they + typically just have a single table, and if they have multiple tables + it's only via WHERE clause, which interestingly do not maintain ORM + annotations when used (that is, (User.name == + 'foo').left.table._annotations is empty; the ORMness of User.name is + lost in the expression construction process, since we don't annotate + (copy) Column objects with ORM entities the way we do for Table. + + .. versionadded:: 2.1 + """ + + def _setup_join_targets(collection): + for (target, *_) in collection: + if isinstance(target, attributes.QueryableAttribute): + yield target.entity + elif "_no_filter_by" not in target._annotations: + yield target - return _determine_last_joined_entity(setup_joins, None) + entities = set(_setup_join_targets(statement._setup_joins)) + + for memoized in statement._memoized_select_entities: + entities.update(_setup_join_targets(memoized._setup_joins)) + + entities.update( + ( + from_obj._annotations["parententity"] + if "parententity" in from_obj._annotations + else from_obj + ) + for from_obj in statement._from_obj + if "_no_filter_by" not in from_obj._annotations + ) + + for element in statement._raw_columns: + if "entity_namespace" in element._annotations: + ens = element._annotations["entity_namespace"] + entities.add(ens) + elif "_no_filter_by" not in element._annotations: + entities.update(element._from_objects) + + return entities @classmethod def all_selected_columns(cls, statement): @@ -1750,9 +1796,10 @@ def _select_statement( statement._order_by_clauses += tuple(order_by) if distinct_on: - statement.distinct.non_generative(statement, *distinct_on) + statement._distinct = True + statement._distinct_on = distinct_on elif distinct: - statement.distinct.non_generative(statement) + statement._distinct = True if group_by: statement._group_by_clauses += tuple(group_by) @@ -1827,17 +1874,14 @@ def _get_current_adapter(self): # subquery of itself, i.e. _from_selectable(), apply adaption # to all SQL constructs. adapters.append( - ( - True, - self._from_obj_alias.replace, - ) + self._from_obj_alias.replace, ) # this was *hopefully* the only adapter we were going to need # going forward...however, we unfortunately need _from_obj_alias # for query.union(), which we can't drop if self._polymorphic_adapters: - adapters.append((False, self._adapt_polymorphic_element)) + adapters.append(self._adapt_polymorphic_element) if not adapters: return None @@ -1847,15 +1891,10 @@ def _adapt_clause(clause, as_filter): # tagged as 'ORM' constructs ? def replace(elem): - is_orm_adapt = ( - "_orm_adapt" in elem._annotations - or "parententity" in elem._annotations - ) - for always_adapt, adapter in adapters: - if is_orm_adapt or always_adapt: - e = adapter(elem) - if e is not None: - return e + for adapter in adapters: + e = adapter(elem) + if e is not None: + return e return visitors.replacement_traverse(clause, {}, replace) @@ -1889,8 +1928,6 @@ def _join(self, args, entities_collection): "selectable/table as join target" ) - of_type = None - if isinstance(onclause, interfaces.PropComparator): # descriptor/property given (or determined); this tells us # explicitly what the expected "left" side of the join is. @@ -1972,6 +2009,7 @@ def _join_left_to_right( """ + explicit_left = left if left is None: # left not given (e.g. no relationship object/name specified) # figure out the best "left" side based on our existing froms / @@ -2017,6 +2055,9 @@ def _join_left_to_right( # self._from_obj list left_clause = self.from_clauses[replace_from_obj_index] + if explicit_left is not None and onclause is None: + onclause = _ORMJoin._join_condition(explicit_left, right) + self.from_clauses = ( self.from_clauses[:replace_from_obj_index] + [ @@ -2440,9 +2481,19 @@ def _select_args(self): @property def _should_nest_selectable(self): kwargs = self._select_args + + if not self.eager_adding_joins: + return False + return ( - kwargs.get("limit_clause") is not None - or kwargs.get("offset_clause") is not None + ( + kwargs.get("limit_clause") is not None + and self.multi_row_eager_loaders + ) + or ( + kwargs.get("offset_clause") is not None + and self.multi_row_eager_loaders + ) or kwargs.get("distinct", False) or kwargs.get("distinct_on", ()) or kwargs.get("group_by", False) @@ -2477,9 +2528,16 @@ def _adjust_for_extra_criteria(self): associated with the global context. """ + ext_infos = [ + fromclause._annotations.get("parententity", None) + for fromclause in self.from_clauses + ] + [ + elem._annotations.get("parententity", None) + for where_crit in self.select_statement._where_criteria + for elem in sql_util.surface_expressions(where_crit) + ] - for fromclause in self.from_clauses: - ext_info = fromclause._annotations.get("parententity", None) + for ext_info in ext_infos: if ( ext_info @@ -2566,7 +2624,6 @@ def _adjust_for_extra_criteria(self): # finally run all the criteria through the "main" adapter, if we # have one, and concatenate to final WHERE criteria for crit in _where_criteria_to_add: - crit = sql_util._deep_annotate(crit, {"_orm_adapt": True}) crit = current_adapter(crit, False) self._where_criteria += (crit,) else: diff --git a/lib/sqlalchemy/orm/decl_api.py b/lib/sqlalchemy/orm/decl_api.py index e01ad61362c..0df31d236a0 100644 --- a/lib/sqlalchemy/orm/decl_api.py +++ b/lib/sqlalchemy/orm/decl_api.py @@ -1,5 +1,5 @@ # orm/decl_api.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -9,7 +9,6 @@ from __future__ import annotations -import itertools import re import typing from typing import Any @@ -20,9 +19,11 @@ from typing import Generic from typing import Iterable from typing import Iterator +from typing import Literal from typing import Mapping from typing import Optional from typing import overload +from typing import Protocol from typing import Set from typing import Tuple from typing import Type @@ -47,12 +48,12 @@ from .base import Mapped from .base import ORMDescriptor from .decl_base import _add_attribute -from .decl_base import _as_declarative -from .decl_base import _ClassScanMapperConfig from .decl_base import _declarative_constructor -from .decl_base import _DeferredMapperConfig +from .decl_base import _DeclarativeMapperConfig +from .decl_base import _DeferredDeclarativeConfig from .decl_base import _del_attribute -from .decl_base import _mapper +from .decl_base import _ORMClassConfigurator +from .decl_base import MappedClassProtocol from .descriptor_props import Composite from .descriptor_props import Synonym from .descriptor_props import Synonym as _orm_synonym @@ -63,7 +64,10 @@ from .. import exc from .. import inspection from .. import util +from ..event import dispatcher +from ..event import EventTarget from ..sql import sqltypes +from ..sql._annotated_cols import _TC from ..sql.base import _NoArg from ..sql.elements import SQLCoreOperations from ..sql.schema import MetaData @@ -71,23 +75,25 @@ from ..util import hybridmethod from ..util import hybridproperty from ..util import typing as compat_typing +from ..util import TypingOnly from ..util.typing import CallableReference from ..util.typing import de_optionalize_union_types +from ..util.typing import GenericProtocol from ..util.typing import is_generic from ..util.typing import is_literal -from ..util.typing import Literal from ..util.typing import LITERAL_TYPES from ..util.typing import Self +from ..util.typing import TypeAliasType if TYPE_CHECKING: from ._typing import _O from ._typing import _RegistryType - from .decl_base import _DataclassArguments from .instrumentation import ClassManager + from .interfaces import _DataclassArguments from .interfaces import MapperProperty from .state import InstanceState # noqa from ..sql._typing import _TypeEngineArgument - from ..sql.type_api import _MatchedOnType + from ..util.typing import _MatchedOnType _T = TypeVar("_T", bound=Any) @@ -191,7 +197,7 @@ def __init__( cls._sa_registry = reg if not cls.__dict__.get("__abstract__", False): - _as_declarative(reg, cls, dict_) + _ORMClassConfigurator._as_declarative(reg, cls, dict_) type.__init__(cls, classname, bases, dict_) @@ -244,7 +250,7 @@ def __init__( cascading: bool = False, quiet: bool = False, ): - # suppport + # support # @declared_attr # @classmethod # def foo(cls) -> Mapped[thing]: @@ -476,6 +482,11 @@ def __call__(self, fn: _DeclaredAttrDecorated[_T]) -> declared_attr[_T]: return declared_attr(fn, **self.kw) +@util.deprecated( + "2.1", + "The declarative_mixin decorator was used only by the now removed " + "mypy plugin so it has no longer any use and can be safely removed.", +) def declarative_mixin(cls: Type[_T]) -> Type[_T]: """Mark a class as providing the feature of "declarative mixin". @@ -518,17 +529,10 @@ class MyModel(MyMixin, Base): def _setup_declarative_base(cls: Type[Any]) -> None: - if "metadata" in cls.__dict__: - metadata = cls.__dict__["metadata"] - else: - metadata = None - - if "type_annotation_map" in cls.__dict__: - type_annotation_map = cls.__dict__["type_annotation_map"] - else: - type_annotation_map = None + metadata = getattr(cls, "metadata", None) + type_annotation_map = getattr(cls, "type_annotation_map", None) + reg = getattr(cls, "registry", None) - reg = cls.__dict__.get("registry", None) if reg is not None: if not isinstance(reg, registry): raise exc.InvalidRequestError( @@ -558,6 +562,43 @@ def _setup_declarative_base(cls: Type[Any]) -> None: cls.__init__ = cls.registry.constructor +def _generate_dc_transforms( + cls_: Type[_O], + init: Union[_NoArg, bool] = _NoArg.NO_ARG, + repr: Union[_NoArg, bool] = _NoArg.NO_ARG, # noqa: A002 + eq: Union[_NoArg, bool] = _NoArg.NO_ARG, + order: Union[_NoArg, bool] = _NoArg.NO_ARG, + unsafe_hash: Union[_NoArg, bool] = _NoArg.NO_ARG, + match_args: Union[_NoArg, bool] = _NoArg.NO_ARG, + kw_only: Union[_NoArg, bool] = _NoArg.NO_ARG, + dataclass_callable: Union[ + _NoArg, Callable[..., Type[Any]] + ] = _NoArg.NO_ARG, +) -> None: + apply_dc_transforms: _DataclassArguments = { + "init": init, + "repr": repr, + "eq": eq, + "order": order, + "unsafe_hash": unsafe_hash, + "match_args": match_args, + "kw_only": kw_only, + "dataclass_callable": dataclass_callable, + } + + if hasattr(cls_, "_sa_apply_dc_transforms"): + current = cls_._sa_apply_dc_transforms # type: ignore[attr-defined] + + _DeclarativeMapperConfig._assert_dc_arguments(current) + + cls_._sa_apply_dc_transforms = { # type: ignore # noqa: E501 + k: current.get(k, _NoArg.NO_ARG) if v is _NoArg.NO_ARG else v + for k, v in apply_dc_transforms.items() + } + else: + setattr(cls_, "_sa_apply_dc_transforms", apply_dc_transforms) + + class MappedAsDataclass(metaclass=DCTransformDeclarative): """Mixin class to indicate when mapping this class, also convert it to be a dataclass. @@ -565,7 +606,14 @@ class MappedAsDataclass(metaclass=DCTransformDeclarative): .. seealso:: :ref:`orm_declarative_native_dataclasses` - complete background - on SQLAlchemy native dataclass mapping + on SQLAlchemy native dataclass mapping with + :class:`_orm.MappedAsDataclass`. + + :ref:`orm_declarative_dc_mixins` - examples specific to using + :class:`_orm.MappedAsDataclass` to create mixins + + :func:`_orm.mapped_as_dataclass` / :func:`_orm.unmapped_dataclass` - + decorator versions with equivalent functionality .. versionadded:: 2.0 @@ -585,47 +633,223 @@ def __init_subclass__( ] = _NoArg.NO_ARG, **kw: Any, ) -> None: - apply_dc_transforms: _DataclassArguments = { - "init": init, - "repr": repr, - "eq": eq, - "order": order, - "unsafe_hash": unsafe_hash, - "match_args": match_args, - "kw_only": kw_only, - "dataclass_callable": dataclass_callable, - } + _generate_dc_transforms( + init=init, + repr=repr, + eq=eq, + order=order, + unsafe_hash=unsafe_hash, + match_args=match_args, + kw_only=kw_only, + dataclass_callable=dataclass_callable, + cls_=cls, + ) + super().__init_subclass__(**kw) - current_transforms: _DataclassArguments + if not _is_mapped_class(cls): + # turn unmapped classes into "good enough" dataclasses to serve + # as a base or a mixin + _ORMClassConfigurator._as_unmapped_dataclass(cls, cls.__dict__) - if hasattr(cls, "_sa_apply_dc_transforms"): - current = cls._sa_apply_dc_transforms - _ClassScanMapperConfig._assert_dc_arguments(current) +class _DeclarativeTyping(TypingOnly): + """Common typing annotations shared by the DeclarativeBase and + DeclarativeBaseNoMeta classes. + """ - cls._sa_apply_dc_transforms = current_transforms = { # type: ignore # noqa: E501 - k: current.get(k, _NoArg.NO_ARG) if v is _NoArg.NO_ARG else v - for k, v in apply_dc_transforms.items() - } - else: - cls._sa_apply_dc_transforms = current_transforms = ( - apply_dc_transforms - ) + __slots__ = () - super().__init_subclass__(**kw) + if typing.TYPE_CHECKING: + # protocols for inspection + def _sa_inspect_type(self) -> Mapper[Self]: ... - if not _is_mapped_class(cls): - new_anno = ( - _ClassScanMapperConfig._update_annotations_for_non_mapped_class - )(cls) - _ClassScanMapperConfig._apply_dataclasses_to_any_class( - current_transforms, cls, new_anno - ) + def _sa_inspect_instance(self) -> InstanceState[Self]: ... + + # internal stuff + _sa_registry: ClassVar[_RegistryType] + + # public interface + registry: ClassVar[_RegistryType] + """Refers to the :class:`_orm.registry` in use where new + :class:`_orm.Mapper` objects will be associated.""" + + metadata: ClassVar[MetaData] + """Refers to the :class:`_schema.MetaData` collection that will be used + for new :class:`_schema.Table` objects. + + .. seealso:: + + :ref:`orm_declarative_metadata` + + """ + + __name__: ClassVar[str] + + # this ideally should be Mapper[Self], but mypy as of 1.4.1 does not + # like it, and breaks the declared_attr_one test. Pyright/pylance is + # ok with it. + __mapper__: ClassVar[Mapper[Any]] + """The :class:`_orm.Mapper` object to which a particular class is + mapped. + + May also be acquired using :func:`_sa.inspect`, e.g. + ``inspect(klass)``. + + """ + + __table__: ClassVar[FromClause] + """The :class:`_sql.FromClause` to which a particular subclass is + mapped. + + This is usually an instance of :class:`_schema.Table` but may also + refer to other kinds of :class:`_sql.FromClause` such as + :class:`_sql.Subquery`, depending on how the class is mapped. + + .. seealso:: + + :ref:`orm_declarative_metadata` + + """ + + # pyright/pylance do not consider a classmethod a ClassVar so use Any + # https://github.com/microsoft/pylance-release/issues/3484 + __tablename__: Any + """String name to assign to the generated + :class:`_schema.Table` object, if not specified directly via + :attr:`_orm.DeclarativeBase.__table__`. + + .. seealso:: + + :ref:`orm_declarative_table` + + """ + + __mapper_args__: Any + """Dictionary of arguments which will be passed to the + :class:`_orm.Mapper` constructor. + + .. seealso:: + + :ref:`orm_declarative_mapper_options` + + """ + + __table_args__: Any + """A dictionary or tuple of arguments that will be passed to the + :class:`_schema.Table` constructor. See + :ref:`orm_declarative_table_configuration` + for background on the specific structure of this collection. + + .. seealso:: + + :ref:`orm_declarative_table_configuration` + + """ + + def __init__(self, **kw: Any): ... + + +class MappedClassWithTypedColumnsProtocol(Protocol[_TC]): + """An ORM mapped class that also defines in the ``__typed_cols__`` + attribute its typed columns. + + .. versionadded:: 2.1.0b2 + """ + + __typed_cols__: _TC + """The :class:`_schema.TypedColumns` of this ORM mapped class.""" + + __name__: ClassVar[str] + __mapper__: ClassVar[Mapper[Any]] + __table__: ClassVar[FromClause] + + +@overload +def as_typed_table( + cls: type[MappedClassWithTypedColumnsProtocol[_TC]], / +) -> FromClause[_TC]: ... + + +@overload +def as_typed_table( + cls: MappedClassProtocol[Any], typed_columns_cls: type[_TC], / +) -> FromClause[_TC]: ... + + +def as_typed_table( + cls: ( + MappedClassProtocol[Any] + | type[MappedClassWithTypedColumnsProtocol[Any]] + ), + typed_columns_cls: Any = None, + /, +) -> FromClause[Any]: + """Return a typed :class:`_sql.FromClause` from the give ORM model. + + This function is just a typing help, at runtime it just returns the + ``__table__`` attribute of the provided ORM model. + + It's usually called providing both the ORM model and the + :class:`_schema.TypedColumns` class. Single argument calls are supported + if the ORM model class provides an annotation pointing to its + :class:`_schema.TypedColumns` in the ``__typed_cols__`` attribute. + + + Example usage:: + + from sqlalchemy import TypedColumns + from sqlalchemy.orm import DeclarativeBase, mapped_column + from sqlalchemy.orm import MappedColumn, as_typed_table + + + class Base(DeclarativeBase): + pass + + + class A(Base): + __tablename__ = "a" + + id: MappedColumn[int] = mapped_column(primary_key=True) + data: MappedColumn[str] + + + class a_cols(A, TypedColumns): + pass + + + # table_a is annotated as FromClause[a_cols] + table_a = as_typed_table(A, a_cols) + + + class B(Base): + __tablename__ = "b" + __typed_cols__: "b_cols" + + a: Mapped[int] = mapped_column(primary_key=True) + b: Mapped[str] + + + class b_cols(B, TypedColumns): + pass + + + # table_b is a FromClause[b_cols], can call with just B since it + # provides the __typed_cols__ annotation + table_b = as_typed_table(B) + + For proper typing integration :class:`_orm.MappedColumn` should be used + to annotate the single columns, since it's a more specific annotation than + the usual :class:`_orm.Mapped` used for ORM attributes. + + .. versionadded:: 2.1.0b2 + """ + return cls.__table__ class DeclarativeBase( # Inspectable is used only by the mypy plugin inspection.Inspectable[InstanceState[Any]], + _DeclarativeTyping, metaclass=DeclarativeAttributeIntercept, ): """Base class used for declarative class definitions. @@ -675,7 +899,7 @@ class Base(DeclarativeBase): :param metadata: optional :class:`_schema.MetaData` collection. If a :class:`_orm.registry` is constructed automatically, this :class:`_schema.MetaData` collection will be used to construct it. - Otherwise, the local :class:`_schema.MetaData` collection will supercede + Otherwise, the local :class:`_schema.MetaData` collection will supersede that used by an existing :class:`_orm.registry` passed using the :paramref:`_orm.DeclarativeBase.registry` parameter. :param type_annotation_map: optional type annotation map that will be @@ -740,99 +964,14 @@ def __init__(self, id=None, name=None): """ - if typing.TYPE_CHECKING: - - def _sa_inspect_type(self) -> Mapper[Self]: ... - - def _sa_inspect_instance(self) -> InstanceState[Self]: ... - - _sa_registry: ClassVar[_RegistryType] - - registry: ClassVar[_RegistryType] - """Refers to the :class:`_orm.registry` in use where new - :class:`_orm.Mapper` objects will be associated.""" - - metadata: ClassVar[MetaData] - """Refers to the :class:`_schema.MetaData` collection that will be used - for new :class:`_schema.Table` objects. - - .. seealso:: - - :ref:`orm_declarative_metadata` - - """ - - __name__: ClassVar[str] - - # this ideally should be Mapper[Self], but mypy as of 1.4.1 does not - # like it, and breaks the declared_attr_one test. Pyright/pylance is - # ok with it. - __mapper__: ClassVar[Mapper[Any]] - """The :class:`_orm.Mapper` object to which a particular class is - mapped. - - May also be acquired using :func:`_sa.inspect`, e.g. - ``inspect(klass)``. - - """ - - __table__: ClassVar[FromClause] - """The :class:`_sql.FromClause` to which a particular subclass is - mapped. - - This is usually an instance of :class:`_schema.Table` but may also - refer to other kinds of :class:`_sql.FromClause` such as - :class:`_sql.Subquery`, depending on how the class is mapped. - - .. seealso:: - - :ref:`orm_declarative_metadata` - - """ - - # pyright/pylance do not consider a classmethod a ClassVar so use Any - # https://github.com/microsoft/pylance-release/issues/3484 - __tablename__: Any - """String name to assign to the generated - :class:`_schema.Table` object, if not specified directly via - :attr:`_orm.DeclarativeBase.__table__`. - - .. seealso:: - - :ref:`orm_declarative_table` - - """ - - __mapper_args__: Any - """Dictionary of arguments which will be passed to the - :class:`_orm.Mapper` constructor. - - .. seealso:: - - :ref:`orm_declarative_mapper_options` - - """ - - __table_args__: Any - """A dictionary or tuple of arguments that will be passed to the - :class:`_schema.Table` constructor. See - :ref:`orm_declarative_table_configuration` - for background on the specific structure of this collection. - - .. seealso:: - - :ref:`orm_declarative_table_configuration` - - """ - - def __init__(self, **kw: Any): ... - def __init_subclass__(cls, **kw: Any) -> None: if DeclarativeBase in cls.__bases__: _check_not_declarative(cls, DeclarativeBase) _setup_declarative_base(cls) else: - _as_declarative(cls._sa_registry, cls, cls.__dict__) + _ORMClassConfigurator._as_declarative( + cls._sa_registry, cls, cls.__dict__ + ) super().__init_subclass__(**kw) @@ -853,7 +992,8 @@ def _check_not_declarative(cls: Type[Any], base: Type[Any]) -> None: class DeclarativeBaseNoMeta( # Inspectable is used only by the mypy plugin - inspection.Inspectable[InstanceState[Any]] + inspection.Inspectable[InstanceState[Any]], + _DeclarativeTyping, ): """Same as :class:`_orm.DeclarativeBase`, but does not use a metaclass to intercept new attributes. @@ -866,95 +1006,14 @@ class DeclarativeBaseNoMeta( """ - _sa_registry: ClassVar[_RegistryType] - - registry: ClassVar[_RegistryType] - """Refers to the :class:`_orm.registry` in use where new - :class:`_orm.Mapper` objects will be associated.""" - - metadata: ClassVar[MetaData] - """Refers to the :class:`_schema.MetaData` collection that will be used - for new :class:`_schema.Table` objects. - - .. seealso:: - - :ref:`orm_declarative_metadata` - - """ - - # this ideally should be Mapper[Self], but mypy as of 1.4.1 does not - # like it, and breaks the declared_attr_one test. Pyright/pylance is - # ok with it. - __mapper__: ClassVar[Mapper[Any]] - """The :class:`_orm.Mapper` object to which a particular class is - mapped. - - May also be acquired using :func:`_sa.inspect`, e.g. - ``inspect(klass)``. - - """ - - __table__: Optional[FromClause] - """The :class:`_sql.FromClause` to which a particular subclass is - mapped. - - This is usually an instance of :class:`_schema.Table` but may also - refer to other kinds of :class:`_sql.FromClause` such as - :class:`_sql.Subquery`, depending on how the class is mapped. - - .. seealso:: - - :ref:`orm_declarative_metadata` - - """ - - if typing.TYPE_CHECKING: - - def _sa_inspect_type(self) -> Mapper[Self]: ... - - def _sa_inspect_instance(self) -> InstanceState[Self]: ... - - __tablename__: Any - """String name to assign to the generated - :class:`_schema.Table` object, if not specified directly via - :attr:`_orm.DeclarativeBase.__table__`. - - .. seealso:: - - :ref:`orm_declarative_table` - - """ - - __mapper_args__: Any - """Dictionary of arguments which will be passed to the - :class:`_orm.Mapper` constructor. - - .. seealso:: - - :ref:`orm_declarative_mapper_options` - - """ - - __table_args__: Any - """A dictionary or tuple of arguments that will be passed to the - :class:`_schema.Table` constructor. See - :ref:`orm_declarative_table_configuration` - for background on the specific structure of this collection. - - .. seealso:: - - :ref:`orm_declarative_table_configuration` - - """ - - def __init__(self, **kw: Any): ... - def __init_subclass__(cls, **kw: Any) -> None: if DeclarativeBaseNoMeta in cls.__bases__: _check_not_declarative(cls, DeclarativeBaseNoMeta) _setup_declarative_base(cls) else: - _as_declarative(cls._sa_registry, cls, cls.__dict__) + _ORMClassConfigurator._as_declarative( + cls._sa_registry, cls, cls.__dict__ + ) super().__init_subclass__(**kw) @@ -1100,7 +1159,7 @@ class that has no ``__init__`` of its own. Defaults to an ) -class registry: +class registry(EventTarget): """Generalized registry for mapping classes. The :class:`_orm.registry` serves as the basis for maintaining a collection @@ -1135,13 +1194,13 @@ class registry: _class_registry: clsregistry._ClsRegistryType _managers: weakref.WeakKeyDictionary[ClassManager[Any], Literal[True]] - _non_primary_mappers: weakref.WeakKeyDictionary[Mapper[Any], Literal[True]] metadata: MetaData constructor: CallableReference[Callable[..., None]] type_annotation_map: _MutableTypeAnnotationMapType _dependents: Set[_RegistryType] _dependencies: Set[_RegistryType] _new_mappers: bool + dispatch: dispatcher["registry"] def __init__( self, @@ -1197,7 +1256,6 @@ class that has no ``__init__`` of its own. Defaults to an self._class_registry = class_registry self._managers = weakref.WeakKeyDictionary() - self._non_primary_mappers = weakref.WeakKeyDictionary() self.metadata = lcl_metadata self.constructor = constructor self.type_annotation_map = {} @@ -1225,6 +1283,65 @@ def update_type_annotation_map( } ) + def _resolve_type_with_events( + self, + cls: Any, + key: str, + raw_annotation: _MatchedOnType, + extracted_type: _MatchedOnType, + *, + raw_pep_593_type: Optional[GenericProtocol[Any]] = None, + pep_593_resolved_argument: Optional[_MatchedOnType] = None, + raw_pep_695_type: Optional[TypeAliasType] = None, + pep_695_resolved_value: Optional[_MatchedOnType] = None, + ) -> Optional[sqltypes.TypeEngine[Any]]: + """Resolve type with event support for custom type mapping. + + This method fires the resolve_type_annotation event first to allow + custom resolution, then falls back to normal resolution. + + """ + + if self.dispatch.resolve_type_annotation: + type_resolve = TypeResolve( + self, + cls, + key, + raw_annotation, + ( + pep_593_resolved_argument + if pep_593_resolved_argument is not None + else ( + pep_695_resolved_value + if pep_695_resolved_value is not None + else extracted_type + ) + ), + raw_pep_593_type, + pep_593_resolved_argument, + raw_pep_695_type, + pep_695_resolved_value, + ) + + for fn in self.dispatch.resolve_type_annotation: + result = fn(type_resolve) + if result is not None: + return sqltypes.to_instance(result) # type: ignore[no-any-return] # noqa: E501 + + if raw_pep_695_type is not None: + sqltype = self._resolve_type(raw_pep_695_type) + if sqltype is not None: + return sqltype + + sqltype = self._resolve_type(extracted_type) + if sqltype is not None: + return sqltype + + if pep_593_resolved_argument is not None: + sqltype = self._resolve_type(pep_593_resolved_argument) + + return sqltype + def _resolve_type( self, python_type: _MatchedOnType ) -> Optional[sqltypes.TypeEngine[Any]]: @@ -1237,7 +1354,7 @@ def _resolve_type( search = ( (python_type, python_type_type), - *((lt, python_type_type) for lt in LITERAL_TYPES), # type: ignore[arg-type] # noqa: E501 + *((lt, python_type_type) for lt in LITERAL_TYPES), ) else: python_type_type = python_type.__origin__ @@ -1277,9 +1394,7 @@ def _resolve_type( def mappers(self) -> FrozenSet[Mapper[Any]]: """read only collection of all :class:`_orm.Mapper` objects.""" - return frozenset(manager.mapper for manager in self._managers).union( - self._non_primary_mappers - ) + return frozenset(manager.mapper for manager in self._managers) def _set_depends_on(self, registry: RegistryType) -> None: if registry is self: @@ -1335,24 +1450,14 @@ def _recurse_with_dependencies( todo.update(reg._dependencies.difference(done)) def _mappers_to_configure(self) -> Iterator[Mapper[Any]]: - return itertools.chain( - ( - manager.mapper - for manager in list(self._managers) - if manager.is_mapped - and not manager.mapper.configured - and manager.mapper._ready_for_configure - ), - ( - npm - for npm in list(self._non_primary_mappers) - if not npm.configured and npm._ready_for_configure - ), + return ( + manager.mapper + for manager in list(self._managers) + if manager.is_mapped + and not manager.mapper.configured + and manager.mapper._ready_for_configure ) - def _add_non_primary_mapper(self, np_mapper: Mapper[Any]) -> None: - self._non_primary_mappers[np_mapper] = True - def _dispose_cls(self, cls: Type[_O]) -> None: clsregistry._remove_class(cls.__name__, cls, self._class_registry) @@ -1605,29 +1710,25 @@ def mapped_as_dataclass( :ref:`orm_declarative_native_dataclasses` - complete background on SQLAlchemy native dataclass mapping + :func:`_orm.mapped_as_dataclass` - functional version that may + provide better compatibility with mypy .. versionadded:: 2.0 """ - def decorate(cls: Type[_O]) -> Type[_O]: - setattr( - cls, - "_sa_apply_dc_transforms", - { - "init": init, - "repr": repr, - "eq": eq, - "order": order, - "unsafe_hash": unsafe_hash, - "match_args": match_args, - "kw_only": kw_only, - "dataclass_callable": dataclass_callable, - }, - ) - _as_declarative(self, cls, cls.__dict__) - return cls + decorate = mapped_as_dataclass( + self, + init=init, + repr=repr, + eq=eq, + order=order, + unsafe_hash=unsafe_hash, + match_args=match_args, + kw_only=kw_only, + dataclass_callable=dataclass_callable, + ) if __cls: return decorate(__cls) @@ -1672,7 +1773,7 @@ class Foo: :meth:`_orm.registry.mapped_as_dataclass` """ - _as_declarative(self, cls, cls.__dict__) + _ORMClassConfigurator._as_declarative(self, cls, cls.__dict__) return cls def as_declarative_base(self, **kw: Any) -> Callable[[Type[_T]], Type[_T]]: @@ -1759,7 +1860,7 @@ class Foo: :meth:`_orm.registry.map_imperatively` """ - _as_declarative(self, cls, cls.__dict__) + _ORMClassConfigurator._as_declarative(self, cls, cls.__dict__) return cls.__mapper__ # type: ignore def map_imperatively( @@ -1818,7 +1919,7 @@ class MyClass: :ref:`orm_declarative_mapping` """ - return _mapper(self, class_, local_table, kw) + return _ORMClassConfigurator._mapper(self, class_, local_table, kw) RegistryType = registry @@ -1828,6 +1929,140 @@ class MyClass: _RegistryType = registry # noqa +class TypeResolve: + """Primary argument to the :meth:`.RegistryEvents.resolve_type_annotation` + event. + + This object contains all the information needed to resolve a Python + type to a SQLAlchemy type. The :attr:`.TypeResolve.resolved_type` is + typically the main type that's resolved. To resolve an arbitrary + Python type against the current type map, the :meth:`.TypeResolve.resolve` + method may be used. + + .. versionadded:: 2.1 + + """ + + __slots__ = ( + "registry", + "cls", + "key", + "raw_type", + "resolved_type", + "raw_pep_593_type", + "raw_pep_695_type", + "pep_593_resolved_argument", + "pep_695_resolved_value", + ) + + cls: Any + "The class being processed during declarative mapping" + + registry: "registry" + "The :class:`registry` being used" + + key: str + "String name of the ORM mapped attribute being processed" + + raw_type: _MatchedOnType + """The type annotation object directly from the attribute's annotations. + + It's recommended to look at :attr:`.TypeResolve.resolved_type` or + one of :attr:`.TypeResolve.pep_593_resolved_argument` or + :attr:`.TypeResolve.pep_695_resolved_value` rather than the raw type, as + the raw type will not be de-optionalized. + + """ + + resolved_type: _MatchedOnType + """The de-optionalized, "resolved" type after accounting for :pep:`695` + and :pep:`593` indirection: + + * If the annotation were a plain Python type or simple alias e.g. + ``Mapped[int]``, the resolved_type will be ``int`` + * If the annotation refers to a :pep:`695` type that references a + plain Python type or simple alias, e.g. ``type MyType = int`` + then ``Mapped[MyType]``, the type will refer to the ``__value__`` + of the :pep:`695` type, e.g. ``int``, the same as + :attr:`.TypeResolve.pep_695_resolved_value`. + * If the annotation refers to a :pep:`593` ``Annotated`` object, or + a :pep:`695` type alias that in turn refers to a :pep:`593` type, + then the type will be the inner type inside of the ``Annotated``, + e.g. ``MyType = Annotated[float, mapped_column(...)]`` with + ``Mapped[MyType]`` becomes ``float``, the same as + :attr:`.TypeResolve.pep_593_resolved_argument`. + + """ + + raw_pep_593_type: Optional[GenericProtocol[Any]] + """The de-optionalized :pep:`593` type, if the raw type referred to one. + + This would refer to an ``Annotated`` object. + + """ + + pep_593_resolved_argument: Optional[_MatchedOnType] + """The type extracted from a :pep:`593` ``Annotated`` construct, if the + type referred to one. + + When present, this type would be the same as the + :attr:`.TypeResolve.resolved_type`. + + """ + + raw_pep_695_type: Optional[TypeAliasType] + "The de-optionalized :pep:`695` type, if the raw type referred to one." + + pep_695_resolved_value: Optional[_MatchedOnType] + """The de-optionalized type referenced by the raw :pep:`695` type, if the + raw type referred to one. + + When present, and a :pep:`593` type is not present, this type would be the + same as the :attr:`.TypeResolve.resolved_type`. + + """ + + def __init__( + self, + registry: RegistryType, + cls: Any, + key: str, + raw_type: _MatchedOnType, + resolved_type: _MatchedOnType, + raw_pep_593_type: Optional[GenericProtocol[Any]], + pep_593_resolved_argument: Optional[_MatchedOnType], + raw_pep_695_type: Optional[TypeAliasType], + pep_695_resolved_value: Optional[_MatchedOnType], + ): + self.registry = registry + self.cls = cls + self.key = key + self.raw_type = raw_type + self.resolved_type = resolved_type + self.raw_pep_593_type = raw_pep_593_type + self.pep_593_resolved_argument = pep_593_resolved_argument + self.raw_pep_695_type = raw_pep_695_type + self.pep_695_resolved_value = pep_695_resolved_value + + def resolve( + self, python_type: _MatchedOnType + ) -> Optional[sqltypes.TypeEngine[Any]]: + """Resolve the given python type using the type_annotation_map of + the :class:`registry`. + + :param python_type: a Python type (e.g. ``int``, ``str``, etc.) Any + type object that's present in + :paramref:`_orm.registry_type_annotation_map` should produce a + non-``None`` result. + :return: a SQLAlchemy :class:`.TypeEngine` instance + (e.g. :class:`.Integer`, + :class:`.String`, etc.), or ``None`` to indicate no type could be + matched. + + """ + return self.registry._resolve_type(python_type) + + def as_declarative(**kw: Any) -> Callable[[Type[_T]], Type[_T]]: """ Class decorator which will adapt a given class into a @@ -1868,12 +2103,178 @@ class MyMappedClass(Base): ... ).as_declarative_base(**kw) +@compat_typing.dataclass_transform( + field_specifiers=( + MappedColumn, + RelationshipProperty, + Composite, + Synonym, + mapped_column, + relationship, + composite, + synonym, + deferred, + ), +) +def mapped_as_dataclass( + registry: RegistryType, + /, + *, + init: Union[_NoArg, bool] = _NoArg.NO_ARG, + repr: Union[_NoArg, bool] = _NoArg.NO_ARG, # noqa: A002 + eq: Union[_NoArg, bool] = _NoArg.NO_ARG, + order: Union[_NoArg, bool] = _NoArg.NO_ARG, + unsafe_hash: Union[_NoArg, bool] = _NoArg.NO_ARG, + match_args: Union[_NoArg, bool] = _NoArg.NO_ARG, + kw_only: Union[_NoArg, bool] = _NoArg.NO_ARG, + dataclass_callable: Union[ + _NoArg, Callable[..., Type[Any]] + ] = _NoArg.NO_ARG, +) -> Callable[[Type[_O]], Type[_O]]: + """Standalone function form of :meth:`_orm.registry.mapped_as_dataclass` + which may have better compatibility with mypy. + + The :class:`_orm.registry` is passed as the first argument to the + decorator. + + e.g.:: + + from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_as_dataclass + from sqlalchemy.orm import mapped_column + from sqlalchemy.orm import registry + + some_registry = registry() + + + @mapped_as_dataclass(some_registry) + class Relationships: + __tablename__ = "relationships" + + entity_id1: Mapped[int] = mapped_column(primary_key=True) + entity_id2: Mapped[int] = mapped_column(primary_key=True) + level: Mapped[int] = mapped_column(Integer) + + .. versionadded:: 2.0.44 + + """ + + def decorate(cls: Type[_O]) -> Type[_O]: + _generate_dc_transforms( + init=init, + repr=repr, + eq=eq, + order=order, + unsafe_hash=unsafe_hash, + match_args=match_args, + kw_only=kw_only, + dataclass_callable=dataclass_callable, + cls_=cls, + ) + _ORMClassConfigurator._as_declarative(registry, cls, cls.__dict__) + return cls + + return decorate + + @inspection._inspects( DeclarativeMeta, DeclarativeBase, DeclarativeAttributeIntercept ) def _inspect_decl_meta(cls: Type[Any]) -> Optional[Mapper[Any]]: mp: Optional[Mapper[Any]] = _inspect_mapped_class(cls) if mp is None: - if _DeferredMapperConfig.has_cls(cls): - _DeferredMapperConfig.raise_unmapped_for_cls(cls) + if _DeferredDeclarativeConfig.has_cls(cls): + _DeferredDeclarativeConfig.raise_unmapped_for_cls(cls) return mp + + +@compat_typing.dataclass_transform( + field_specifiers=( + MappedColumn, + RelationshipProperty, + Composite, + Synonym, + mapped_column, + relationship, + composite, + synonym, + deferred, + ), +) +@overload +def unmapped_dataclass(__cls: Type[_O], /) -> Type[_O]: ... + + +@overload +def unmapped_dataclass( + __cls: Literal[None] = ..., + /, + *, + init: Union[_NoArg, bool] = ..., + repr: Union[_NoArg, bool] = ..., # noqa: A002 + eq: Union[_NoArg, bool] = ..., + order: Union[_NoArg, bool] = ..., + unsafe_hash: Union[_NoArg, bool] = ..., + match_args: Union[_NoArg, bool] = ..., + kw_only: Union[_NoArg, bool] = ..., + dataclass_callable: Union[_NoArg, Callable[..., Type[Any]]] = ..., +) -> Callable[[Type[_O]], Type[_O]]: ... + + +def unmapped_dataclass( + __cls: Optional[Type[_O]] = None, + /, + *, + init: Union[_NoArg, bool] = _NoArg.NO_ARG, + repr: Union[_NoArg, bool] = _NoArg.NO_ARG, # noqa: A002 + eq: Union[_NoArg, bool] = _NoArg.NO_ARG, + order: Union[_NoArg, bool] = _NoArg.NO_ARG, + unsafe_hash: Union[_NoArg, bool] = _NoArg.NO_ARG, + match_args: Union[_NoArg, bool] = _NoArg.NO_ARG, + kw_only: Union[_NoArg, bool] = _NoArg.NO_ARG, + dataclass_callable: Union[ + _NoArg, Callable[..., Type[Any]] + ] = _NoArg.NO_ARG, +) -> Union[Type[_O], Callable[[Type[_O]], Type[_O]]]: + """Decorator which allows the creation of dataclass-compatible mixins + within mapped class hierarchies based on the + :func:`_orm.mapped_as_dataclass` decorator. + + Parameters are the same as those of :func:`_orm.mapped_as_dataclass`. + The decorator turns the given class into a SQLAlchemy-compatible dataclass + in the same way that :func:`_orm.mapped_as_dataclass` does, taking + into account :func:`_orm.mapped_column` and other attributes for dataclass- + specific directives, but not actually mapping the class. + + To create unmapped dataclass mixins when using a class hierarchy defined + by :class:`.DeclarativeBase` and :class:`.MappedAsDataclass`, the + :class:`.MappedAsDataclass` class may be subclassed alone for a similar + effect. + + .. versionadded:: 2.1 + + .. seealso:: + + :ref:`orm_declarative_dc_mixins` - background and example use. + + """ + + def decorate(cls: Type[_O]) -> Type[_O]: + _generate_dc_transforms( + init=init, + repr=repr, + eq=eq, + order=order, + unsafe_hash=unsafe_hash, + match_args=match_args, + kw_only=kw_only, + dataclass_callable=dataclass_callable, + cls_=cls, + ) + _ORMClassConfigurator._as_unmapped_dataclass(cls, cls.__dict__) + return cls + + if __cls: + return decorate(__cls) + else: + return decorate diff --git a/lib/sqlalchemy/orm/decl_base.py b/lib/sqlalchemy/orm/decl_base.py index a2291d2d755..f646fbb01b8 100644 --- a/lib/sqlalchemy/orm/decl_base.py +++ b/lib/sqlalchemy/orm/decl_base.py @@ -1,5 +1,5 @@ # orm/decl_base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -11,11 +11,13 @@ import collections import dataclasses +import itertools import re from typing import Any from typing import Callable from typing import cast from typing import Dict +from typing import get_args from typing import Iterable from typing import List from typing import Mapping @@ -27,7 +29,6 @@ from typing import Tuple from typing import Type from typing import TYPE_CHECKING -from typing import TypedDict from typing import TypeVar from typing import Union import weakref @@ -46,6 +47,7 @@ from .descriptor_props import CompositeProperty from .descriptor_props import SynonymProperty from .interfaces import _AttributeOptions +from .interfaces import _DataclassArguments from .interfaces import _DCAttributeOptions from .interfaces import _IntrospectsAnnotations from .interfaces import _MappedAttribute @@ -62,12 +64,12 @@ from .. import exc from .. import util from ..sql import expression +from ..sql._annotated_cols import TypedColumns from ..sql.base import _NoArg from ..sql.schema import Column from ..sql.schema import Table from ..util import topological from ..util.typing import _AnnotationScanType -from ..util.typing import get_args from ..util.typing import is_fwd_ref from ..util.typing import is_literal @@ -103,6 +105,7 @@ def __call__(self, **kw: Any) -> _O: ... class _DeclMappedClassProtocol(MappedClassProtocol[_O], Protocol): "Internal more detailed version of ``MappedClassProtocol``." + metadata: MetaData __tablename__: str __mapper_args__: _MapperKwArgs @@ -115,23 +118,12 @@ def __declare_first__(self) -> None: ... def __declare_last__(self) -> None: ... -class _DataclassArguments(TypedDict): - init: Union[_NoArg, bool] - repr: Union[_NoArg, bool] - eq: Union[_NoArg, bool] - order: Union[_NoArg, bool] - unsafe_hash: Union[_NoArg, bool] - match_args: Union[_NoArg, bool] - kw_only: Union[_NoArg, bool] - dataclass_callable: Union[_NoArg, Callable[..., Type[Any]]] - - def _declared_mapping_info( cls: Type[Any], -) -> Optional[Union[_DeferredMapperConfig, Mapper[Any]]]: +) -> Optional[Union[_DeferredDeclarativeConfig, Mapper[Any]]]: # deferred mapping - if _DeferredMapperConfig.has_cls(cls): - return _DeferredMapperConfig.config_for_cls(cls) + if _DeferredDeclarativeConfig.has_cls(cls): + return _DeferredDeclarativeConfig.config_for_cls(cls) # regular mapping elif _is_mapped_class(cls): return class_mapper(cls, configure=False) @@ -150,7 +142,7 @@ def _is_supercls_for_inherits(cls: Type[Any]) -> bool: mapper._set_concrete_base() """ - if _DeferredMapperConfig.has_cls(cls): + if _DeferredDeclarativeConfig.has_cls(cls): return not _get_immediate_cls_attr( cls, "_sa_decl_prepare_nocascade", strict=True ) @@ -236,24 +228,6 @@ def _dive_for_cls_manager(cls: Type[_O]) -> Optional[ClassManager[_O]]: return None -def _as_declarative( - registry: _RegistryType, cls: Type[Any], dict_: _ClassDict -) -> Optional[_MapperConfig]: - # declarative scans the class for attributes. no table or mapper - # args passed separately. - return _MapperConfig.setup_mapping(registry, cls, dict_, None, {}) - - -def _mapper( - registry: _RegistryType, - cls: Type[_O], - table: Optional[FromClause], - mapper_kw: _MapperKwArgs, -) -> Mapper[_O]: - _ImperativeMapperConfig(registry, cls, table, mapper_kw) - return cast("MappedClassProtocol[_O]", cls).__mapper__ - - @util.preload_module("sqlalchemy.orm.decl_api") def _is_declarative_props(obj: Any) -> bool: _declared_attr_common = util.preloaded.orm_decl_api._declared_attr_common @@ -276,41 +250,38 @@ def _check_declared_props_nocascade( return False -class _MapperConfig: - __slots__ = ( - "cls", - "classname", - "properties", - "declared_attr_reg", - "__weakref__", - ) +class _ORMClassConfigurator: + """Object that configures a class that's potentially going to be + mapped, and/or turned into an ORM dataclass. + + This is the base class for all the configurator objects. + + """ + + __slots__ = ("cls", "classname", "__weakref__") cls: Type[Any] classname: str - properties: util.OrderedDict[ - str, - Union[ - Sequence[NamedColumn[Any]], NamedColumn[Any], MapperProperty[Any] - ], - ] - declared_attr_reg: Dict[declared_attr[Any], Any] + + def __init__(self, cls_: Type[Any]): + self.cls = util.assert_arg_type(cls_, type, "cls_") + self.classname = cls_.__name__ @classmethod - def setup_mapping( - cls, - registry: _RegistryType, - cls_: Type[_O], - dict_: _ClassDict, - table: Optional[FromClause], - mapper_kw: _MapperKwArgs, + def _as_declarative( + cls, registry: _RegistryType, cls_: Type[Any], dict_: _ClassDict ) -> Optional[_MapperConfig]: - manager = attributes.opt_manager_of_class(cls) + manager = attributes.opt_manager_of_class(cls_) if manager and manager.class_ is cls_: raise exc.InvalidRequestError( - f"Class {cls!r} already has been instrumented declaratively" + f"Class {cls_!r} already has been instrumented declaratively" ) - if cls_.__dict__.get("__abstract__", False): + # allow subclassing an orm class with typed columns without + # generating an orm class + if cls_.__dict__.get("__abstract__", False) or issubclass( + cls_, TypedColumns + ): return None defer_map = _get_immediate_cls_attr( @@ -318,48 +289,68 @@ def setup_mapping( ) or hasattr(cls_, "_sa_decl_prepare") if defer_map: - return _DeferredMapperConfig( - registry, cls_, dict_, table, mapper_kw - ) + return _DeferredDeclarativeConfig(registry, cls_, dict_) else: - return _ClassScanMapperConfig( - registry, cls_, dict_, table, mapper_kw - ) + return _DeclarativeMapperConfig(registry, cls_, dict_) + + @classmethod + def _as_unmapped_dataclass( + cls, cls_: Type[Any], dict_: _ClassDict + ) -> _UnmappedDataclassConfig: + return _UnmappedDataclassConfig(cls_, dict_) + + @classmethod + def _mapper( + cls, + registry: _RegistryType, + cls_: Type[_O], + table: Optional[FromClause], + mapper_kw: _MapperKwArgs, + ) -> Mapper[_O]: + _ImperativeMapperConfig(registry, cls_, table, mapper_kw) + return cast("MappedClassProtocol[_O]", cls_).__mapper__ + + +class _MapperConfig(_ORMClassConfigurator): + """Configurator that configures a class that's potentially going to be + mapped, and optionally turned into a dataclass as well.""" + + __slots__ = ( + "properties", + "declared_attr_reg", + ) + + properties: util.OrderedDict[ + str, + Union[ + Sequence[NamedColumn[Any]], NamedColumn[Any], MapperProperty[Any] + ], + ] + declared_attr_reg: Dict[declared_attr[Any], Any] def __init__( self, registry: _RegistryType, cls_: Type[Any], - mapper_kw: _MapperKwArgs, ): - self.cls = util.assert_arg_type(cls_, type, "cls_") - self.classname = cls_.__name__ + super().__init__(cls_) self.properties = util.OrderedDict() self.declared_attr_reg = {} - if not mapper_kw.get("non_primary", False): - instrumentation.register_class( - self.cls, - finalize=False, - registry=registry, - declarative_scan=self, - init_method=registry.constructor, - ) - else: - manager = attributes.opt_manager_of_class(self.cls) - if not manager or not manager.is_mapped: - raise exc.InvalidRequestError( - "Class %s has no primary mapper configured. Configure " - "a primary mapper first before setting up a non primary " - "Mapper." % self.cls - ) + instrumentation.register_class( + self.cls, + finalize=False, + registry=registry, + declarative_scan=self, + init_method=registry.constructor, + ) def set_cls_attribute(self, attrname: str, value: _T) -> _T: manager = instrumentation.manager_of_class(self.cls) manager.install_member(attrname, value) return value - def map(self, mapper_kw: _MapperKwArgs = ...) -> Mapper[Any]: + def map(self, mapper_kw: _MapperKwArgs) -> Mapper[Any]: raise NotImplementedError() def _early_mapping(self, mapper_kw: _MapperKwArgs) -> None: @@ -367,6 +358,8 @@ def _early_mapping(self, mapper_kw: _MapperKwArgs) -> None: class _ImperativeMapperConfig(_MapperConfig): + """Configurator that configures a class for an imperative mapping.""" + __slots__ = ("local_table", "inherits") def __init__( @@ -376,15 +369,14 @@ def __init__( table: Optional[FromClause], mapper_kw: _MapperKwArgs, ): - super().__init__(registry, cls_, mapper_kw) + super().__init__(registry, cls_) self.local_table = self.set_cls_attribute("__table__", table) with mapperlib._CONFIGURE_MUTEX: - if not mapper_kw.get("non_primary", False): - clsregistry._add_class( - self.classname, self.cls, registry._class_registry - ) + clsregistry._add_class( + self.classname, self.cls, registry._class_registry + ) self._setup_inheritance(mapper_kw) @@ -401,29 +393,26 @@ def map(self, mapper_kw: _MapperKwArgs = util.EMPTY_DICT) -> Mapper[Any]: def _setup_inheritance(self, mapper_kw: _MapperKwArgs) -> None: cls = self.cls - inherits = mapper_kw.get("inherits", None) + inherits = None + inherits_search = [] - if inherits is None: - # since we search for classical mappings now, search for - # multiple mapped bases as well and raise an error. - inherits_search = [] - for base_ in cls.__bases__: - c = _resolve_for_abstract_or_classical(base_) - if c is None: - continue + # since we search for classical mappings now, search for + # multiple mapped bases as well and raise an error. + for base_ in cls.__bases__: + c = _resolve_for_abstract_or_classical(base_) + if c is None: + continue - if _is_supercls_for_inherits(c) and c not in inherits_search: - inherits_search.append(c) + if _is_supercls_for_inherits(c) and c not in inherits_search: + inherits_search.append(c) - if inherits_search: - if len(inherits_search) > 1: - raise exc.InvalidRequestError( - "Class %s has multiple mapped bases: %r" - % (cls, inherits_search) - ) - inherits = inherits_search[0] - elif isinstance(inherits, Mapper): - inherits = inherits.class_ + if inherits_search: + if len(inherits_search) > 1: + raise exc.InvalidRequestError( + "Class %s has multiple mapped bases: %r" + % (cls, inherits_search) + ) + inherits = inherits_search[0] self.inherits = inherits @@ -438,49 +427,20 @@ class _CollectedAnnotation(NamedTuple): originating_class: Type[Any] -class _ClassScanMapperConfig(_MapperConfig): - __slots__ = ( - "registry", - "clsdict_view", - "collected_attributes", - "collected_annotations", - "local_table", - "persist_selectable", - "declared_columns", - "column_ordering", - "column_copies", - "table_args", - "tablename", - "mapper_args", - "mapper_args_fn", - "table_fn", - "inherits", - "single", - "allow_dataclass_fields", - "dataclass_setup_arguments", - "is_dataclass_prior_to_mapping", - "allow_unmapped_annotations", - ) +class _ClassScanAbstractConfig(_ORMClassConfigurator): + """Abstract base for a configurator that configures a class for a + declarative mapping, or an unmapped ORM dataclass. + + Defines scanning of pep-484 annotations as well as ORM dataclass + applicators + + """ + + __slots__ = () - is_deferred = False - registry: _RegistryType clsdict_view: _ClassDict collected_annotations: Dict[str, _CollectedAnnotation] collected_attributes: Dict[str, Any] - local_table: Optional[FromClause] - persist_selectable: Optional[FromClause] - declared_columns: util.OrderedSet[Column[Any]] - column_ordering: Dict[Column[Any], int] - column_copies: Dict[ - Union[MappedColumn[Any], Column[Any]], - Union[MappedColumn[Any], Column[Any]], - ] - tablename: Optional[str] - mapper_args: Mapping[str, Any] - table_args: Optional[_TableArgsType] - mapper_args_fn: Optional[Callable[[], Dict[str, Any]]] - inherits: Optional[Type[Any]] - single: bool is_dataclass_prior_to_mapping: bool allow_unmapped_annotations: bool @@ -502,104 +462,310 @@ class as well as superclasses and extract ORM mapping directives from """ - def __init__( - self, - registry: _RegistryType, - cls_: Type[_O], - dict_: _ClassDict, - table: Optional[FromClause], - mapper_kw: _MapperKwArgs, - ): - # grab class dict before the instrumentation manager has been added. - # reduces cycles - self.clsdict_view = ( - util.immutabledict(dict_) if dict_ else util.EMPTY_DICT - ) - super().__init__(registry, cls_, mapper_kw) - self.registry = registry - self.persist_selectable = None - - self.collected_attributes = {} - self.collected_annotations = {} - self.declared_columns = util.OrderedSet() - self.column_ordering = {} - self.column_copies = {} - self.single = False - self.dataclass_setup_arguments = dca = getattr( - self.cls, "_sa_apply_dc_transforms", None - ) + _include_dunders = { + "__table__", + "__mapper_args__", + "__tablename__", + "__table_args__", + } - self.allow_unmapped_annotations = getattr( - self.cls, "__allow_unmapped__", False - ) or bool(self.dataclass_setup_arguments) + _match_exclude_dunders = re.compile(r"^(?:_sa_|__)") - self.is_dataclass_prior_to_mapping = cld = dataclasses.is_dataclass( - cls_ - ) + def _scan_attributes(self) -> None: + raise NotImplementedError() - sdk = _get_immediate_cls_attr(cls_, "__sa_dataclass_metadata_key__") + def _setup_dataclasses_transforms( + self, *, enable_descriptor_defaults: bool, revert: bool = False + ) -> None: + dataclass_setup_arguments = self.dataclass_setup_arguments + if not dataclass_setup_arguments: + return - # we don't want to consume Field objects from a not-already-dataclass. - # the Field objects won't have their "name" or "type" populated, - # and while it seems like we could just set these on Field as we - # read them, Field is documented as "user read only" and we need to - # stay far away from any off-label use of dataclasses APIs. - if (not cld or dca) and sdk: + # can't use is_dataclass since it uses hasattr + if "__dataclass_fields__" in self.cls.__dict__: raise exc.InvalidRequestError( - "SQLAlchemy mapped dataclasses can't consume mapping " - "information from dataclass.Field() objects if the immediate " - "class is not already a dataclass." + f"Class {self.cls} is already a dataclass; ensure that " + "base classes / decorator styles of establishing dataclasses " + "are not being mixed. " + "This can happen if a class that inherits from " + "'MappedAsDataclass', even indirectly, is been mapped with " + "'@registry.mapped_as_dataclass'" ) - # if already a dataclass, and __sa_dataclass_metadata_key__ present, - # then also look inside of dataclass.Field() objects yielded by - # dataclasses.get_fields(cls) when scanning for attributes - self.allow_dataclass_fields = bool(sdk and cld) - - self._setup_declared_events() - - self._scan_attributes() - - self._setup_dataclasses_transforms() - - with mapperlib._CONFIGURE_MUTEX: - clsregistry._add_class( - self.classname, self.cls, registry._class_registry + # can't create a dataclass if __table__ is already there. This would + # fail an assertion when calling _get_arguments_for_make_dataclass: + # assert False, "Mapped[] received without a mapping declaration" + if "__table__" in self.cls.__dict__: + raise exc.InvalidRequestError( + f"Class {self.cls} already defines a '__table__'. " + "ORM Annotated Dataclasses do not support a pre-existing " + "'__table__' element" ) - self._setup_inheriting_mapper(mapper_kw) - - self._extract_mappable_attributes() - - self._extract_declared_columns() - - self._setup_table(table) - - self._setup_inheriting_columns(mapper_kw) - - self._early_mapping(mapper_kw) - - def _setup_declared_events(self) -> None: - if _get_immediate_cls_attr(self.cls, "__declare_last__"): + raise_for_non_dc_attrs = collections.defaultdict(list) - @event.listens_for(Mapper, "after_configured") - def after_configured() -> None: - cast( - "_DeclMappedClassProtocol[Any]", self.cls - ).__declare_last__() - - if _get_immediate_cls_attr(self.cls, "__declare_first__"): + def _allow_dataclass_field( + key: str, originating_class: Type[Any] + ) -> bool: + if ( + originating_class is not self.cls + and "__dataclass_fields__" not in originating_class.__dict__ + ): + raise_for_non_dc_attrs[originating_class].append(key) - @event.listens_for(Mapper, "before_configured") - def before_configured() -> None: - cast( - "_DeclMappedClassProtocol[Any]", self.cls - ).__declare_first__() + return True - def _cls_attr_override_checker( - self, cls: Type[_O] - ) -> Callable[[str, Any], bool]: - """Produce a function that checks if a class has overridden an + field_list = [ + _AttributeOptions._get_arguments_for_make_dataclass( + self, + key, + anno, + mapped_container, + self.collected_attributes.get(key, _NoArg.NO_ARG), + dataclass_setup_arguments, + enable_descriptor_defaults, + ) + for key, anno, mapped_container in ( + ( + key, + raw_anno, + mapped_container, + ) + for key, ( + raw_anno, + mapped_container, + mapped_anno, + is_dc, + attr_value, + originating_module, + originating_class, + ) in self.collected_annotations.items() + if _allow_dataclass_field(key, originating_class) + and ( + key not in self.collected_attributes + # issue #9226; check for attributes that we've collected + # which are already instrumented, which we would assume + # mean we are in an ORM inheritance mapping and this + # attribute is already mapped on the superclass. Under + # no circumstance should any QueryableAttribute be sent to + # the dataclass() function; anything that's mapped should + # be Field and that's it + or not isinstance( + self.collected_attributes[key], QueryableAttribute + ) + ) + ) + ] + if raise_for_non_dc_attrs: + for ( + originating_class, + non_dc_attrs, + ) in raise_for_non_dc_attrs.items(): + raise exc.InvalidRequestError( + f"When transforming {self.cls} to a dataclass, " + f"attribute(s) " + f"{', '.join(repr(key) for key in non_dc_attrs)} " + f"originates from superclass " + f"{originating_class}, which is not a dataclass. When " + f"declaring SQLAlchemy Declarative " + f"Dataclasses, ensure that all mixin classes and other " + f"superclasses which include attributes are also a " + f"subclass of MappedAsDataclass or make use of the " + f"@unmapped_dataclass decorator.", + code="dcmx", + ) + + if revert: + # the "revert" case is used only by an unmapped mixin class + # that is nonetheless using Mapped construct and needs to + # itself be a dataclass + revert_dict = { + name: self.cls.__dict__[name] + for name in (item[0] for item in field_list) + if name in self.cls.__dict__ + } + else: + revert_dict = None + + # get original annotations using ForwardRef for symbols that + # are unresolvable + orig_annotations = util.get_annotations(self.cls) + + # build a new __annotations__ dict from the fields we have. + # this has to be done carefully since we have to maintain + # the correct order! wow + swap_annotations = {} + defaults = {} + + for item in field_list: + if len(item) == 2: + name, tp = item + elif len(item) == 3: + name, tp, spec = item + defaults[name] = spec + else: + assert False + + # add the annotation to the new dict we are creating. + # note that if name is in orig_annotations, we expect + # tp and orig_annotations[name] to be identical. + swap_annotations[name] = orig_annotations.get(name, tp) + + for k, v in defaults.items(): + setattr(self.cls, k, v) + + self._assert_dc_arguments(dataclass_setup_arguments) + + dataclass_callable = dataclass_setup_arguments["dataclass_callable"] + if dataclass_callable is _NoArg.NO_ARG: + dataclass_callable = dataclasses.dataclass + + # create a merged __annotations__ dictionary, maintaining order + # as best we can: + + # 1. merge all keys in orig_annotations that occur before + # we see any of our mapped fields (this can be attributes like + # __table_args__ etc.) + new_annotations = { + k: orig_annotations[k] + for k in itertools.takewhile( + lambda k: k not in swap_annotations, orig_annotations + ) + } + + # 2. then put in all the dataclass annotations we have + new_annotations |= swap_annotations + + # 3. them merge all of orig_annotations which will add remaining + # keys + new_annotations |= orig_annotations + + # 4. this becomes the new class annotations. + restore_anno = util.restore_annotations(self.cls, new_annotations) + + try: + dataclass_callable( # type: ignore[call-overload] + self.cls, + **{ # type: ignore[call-overload,unused-ignore] + k: v + for k, v in dataclass_setup_arguments.items() + if v is not _NoArg.NO_ARG + and k not in ("dataclass_callable",) + }, + ) + except (TypeError, ValueError) as ex: + raise exc.InvalidRequestError( + f"Python dataclasses error encountered when creating " + f"dataclass for {self.cls.__name__!r}: " + f"{ex!r}. Please refer to Python dataclasses " + "documentation for additional information.", + code="dcte", + ) from ex + finally: + if revert and revert_dict: + # used for mixin dataclasses; we have to restore the + # mapped_column(), relationship() etc. to the class so these + # take place for a mapped class scan + for k, v in revert_dict.items(): + setattr(self.cls, k, v) + + restore_anno() + + def _collect_annotation( + self, + name: str, + raw_annotation: _AnnotationScanType, + originating_class: Type[Any], + expect_mapped: Optional[bool], + attr_value: Any, + ) -> Optional[_CollectedAnnotation]: + if name in self.collected_annotations: + return self.collected_annotations[name] + + if raw_annotation is None: + return None + + is_dataclass = self.is_dataclass_prior_to_mapping + allow_unmapped = self.allow_unmapped_annotations + + if expect_mapped is None: + is_dataclass_field = isinstance(attr_value, dataclasses.Field) + expect_mapped = ( + not is_dataclass_field + and not allow_unmapped + and ( + attr_value is None + or isinstance(attr_value, _MappedAttribute) + ) + ) + + is_dataclass_field = False + extracted = _extract_mapped_subtype( + raw_annotation, + self.cls, + originating_class.__module__, + name, + type(attr_value), + required=False, + is_dataclass_field=is_dataclass_field, + expect_mapped=expect_mapped and not is_dataclass, + ) + if extracted is None: + # ClassVar can come out here + return None + + extracted_mapped_annotation, mapped_container = extracted + + if attr_value is None and not is_literal(extracted_mapped_annotation): + for elem in get_args(extracted_mapped_annotation): + if is_fwd_ref( + elem, check_generic=True, check_for_plain_string=True + ): + elem = de_stringify_annotation( + self.cls, + elem, + originating_class.__module__, + include_generic=True, + ) + # look in Annotated[...] for an ORM construct, + # such as Annotated[int, mapped_column(primary_key=True)] + if isinstance(elem, _IntrospectsAnnotations): + attr_value = elem.found_in_pep593_annotated() + + self.collected_annotations[name] = ca = _CollectedAnnotation( + raw_annotation, + mapped_container, + extracted_mapped_annotation, + is_dataclass, + attr_value, + originating_class.__module__, + originating_class, + ) + return ca + + @classmethod + def _assert_dc_arguments(cls, arguments: _DataclassArguments) -> None: + allowed = { + "init", + "repr", + "order", + "eq", + "unsafe_hash", + "kw_only", + "match_args", + "dataclass_callable", + } + disallowed_args = set(arguments).difference(allowed) + if disallowed_args: + msg = ", ".join(f"{arg!r}" for arg in sorted(disallowed_args)) + raise exc.ArgumentError( + f"Dataclass argument(s) {msg} are not accepted" + ) + + def _cls_attr_override_checker( + self, cls: Type[_O] + ) -> Callable[[str, Any], bool]: + """Produce a function that checks if a class has overridden an attribute, taking SQLAlchemy-enabled dataclass fields into account. """ @@ -672,15 +838,6 @@ def attribute_is_overridden(key: str, obj: Any) -> bool: return attribute_is_overridden - _include_dunders = { - "__table__", - "__mapper_args__", - "__tablename__", - "__table_args__", - } - - _match_exclude_dunders = re.compile(r"^(?:_sa_|__)") - def _cls_attr_resolver( self, cls: Type[Any] ) -> Callable[[], Iterable[Tuple[str, Any, Any, bool]]]: @@ -749,6 +906,142 @@ def local_attributes_for_class() -> ( return local_attributes_for_class + +class _DeclarativeMapperConfig(_MapperConfig, _ClassScanAbstractConfig): + """Configurator that will produce a declarative mapped class""" + + __slots__ = ( + "registry", + "local_table", + "persist_selectable", + "declared_columns", + "column_ordering", + "column_copies", + "table_args", + "tablename", + "mapper_args", + "mapper_args_fn", + "table_fn", + "inherits", + "single", + "clsdict_view", + "collected_attributes", + "collected_annotations", + "allow_dataclass_fields", + "dataclass_setup_arguments", + "is_dataclass_prior_to_mapping", + "allow_unmapped_annotations", + ) + + is_deferred = False + registry: _RegistryType + local_table: Optional[FromClause] + persist_selectable: Optional[FromClause] + declared_columns: util.OrderedSet[Column[Any]] + column_ordering: Dict[Column[Any], int] + column_copies: Dict[ + Union[MappedColumn[Any], Column[Any]], + Union[MappedColumn[Any], Column[Any]], + ] + tablename: Optional[str] + mapper_args: Mapping[str, Any] + table_args: Optional[_TableArgsType] + mapper_args_fn: Optional[Callable[[], Dict[str, Any]]] + inherits: Optional[Type[Any]] + single: bool + + def __init__( + self, + registry: _RegistryType, + cls_: Type[_O], + dict_: _ClassDict, + ): + # grab class dict before the instrumentation manager has been added. + # reduces cycles + self.clsdict_view = ( + util.immutabledict(dict_) if dict_ else util.EMPTY_DICT + ) + super().__init__(registry, cls_) + self.registry = registry + self.persist_selectable = None + + self.collected_attributes = {} + self.collected_annotations = {} + self.declared_columns = util.OrderedSet() + self.column_ordering = {} + self.column_copies = {} + self.single = False + self.dataclass_setup_arguments = dca = getattr( + self.cls, "_sa_apply_dc_transforms", None + ) + + self.allow_unmapped_annotations = getattr( + self.cls, "__allow_unmapped__", False + ) or bool(self.dataclass_setup_arguments) + + self.is_dataclass_prior_to_mapping = cld = dataclasses.is_dataclass( + cls_ + ) + + sdk = _get_immediate_cls_attr(cls_, "__sa_dataclass_metadata_key__") + + # we don't want to consume Field objects from a not-already-dataclass. + # the Field objects won't have their "name" or "type" populated, + # and while it seems like we could just set these on Field as we + # read them, Field is documented as "user read only" and we need to + # stay far away from any off-label use of dataclasses APIs. + if (not cld or dca) and sdk: + raise exc.InvalidRequestError( + "SQLAlchemy mapped dataclasses can't consume mapping " + "information from dataclass.Field() objects if the immediate " + "class is not already a dataclass." + ) + + # if already a dataclass, and __sa_dataclass_metadata_key__ present, + # then also look inside of dataclass.Field() objects yielded by + # dataclasses.get_fields(cls) when scanning for attributes + self.allow_dataclass_fields = bool(sdk and cld) + + self._setup_declared_events() + + self._scan_attributes() + + self._setup_dataclasses_transforms(enable_descriptor_defaults=True) + + with mapperlib._CONFIGURE_MUTEX: + clsregistry._add_class( + self.classname, self.cls, registry._class_registry + ) + + self._setup_inheriting_mapper() + + self._extract_mappable_attributes() + + self._extract_declared_columns() + + self._setup_table() + + self._setup_inheriting_columns() + + self._early_mapping(util.EMPTY_DICT) + + def _setup_declared_events(self) -> None: + if _get_immediate_cls_attr(self.cls, "__declare_last__"): + + @event.listens_for(Mapper, "after_configured") + def after_configured() -> None: + cast( + "_DeclMappedClassProtocol[Any]", self.cls + ).__declare_last__() + + if _get_immediate_cls_attr(self.cls, "__declare_first__"): + + @event.listens_for(Mapper, "before_configured") + def before_configured() -> None: + cast( + "_DeclMappedClassProtocol[Any]", self.cls + ).__declare_first__() + def _scan_attributes(self) -> None: cls = self.cls @@ -989,187 +1282,67 @@ def _table_fn() -> FromClause: assert ( name in collected_attributes or attribute_is_overridden(name, None) - ) - continue - else: - # here, the attribute is some other kind of - # property that we assume is not part of the - # declarative mapping. however, check for some - # more common mistakes - self._warn_for_decl_attributes(base, name, obj) - elif is_dataclass_field and ( - name not in clsdict_view or clsdict_view[name] is not obj - ): - # here, we are definitely looking at the target class - # and not a superclass. this is currently a - # dataclass-only path. if the name is only - # a dataclass field and isn't in local cls.__dict__, - # put the object there. - # assert that the dataclass-enabled resolver agrees - # with what we are seeing - - assert not attribute_is_overridden(name, obj) - - if _is_declarative_props(obj): - obj = obj.fget() - - collected_attributes[name] = obj - self._collect_annotation( - name, annotation, base, False, obj - ) - else: - collected_annotation = self._collect_annotation( - name, annotation, base, None, obj - ) - is_mapped = ( - collected_annotation is not None - and collected_annotation.mapped_container is not None - ) - generated_obj = ( - collected_annotation.attr_value - if collected_annotation is not None - else obj - ) - if obj is None and not fixed_table and is_mapped: - collected_attributes[name] = ( - generated_obj - if generated_obj is not None - else MappedColumn() - ) - elif name in clsdict_view: - collected_attributes[name] = obj - # else if the name is not in the cls.__dict__, - # don't collect it as an attribute. - # we will see the annotation only, which is meaningful - # both for mapping and dataclasses setup - - if inherited_table_args and not tablename: - table_args = None - - self.table_args = table_args - self.tablename = tablename - self.mapper_args_fn = mapper_args_fn - self.table_fn = table_fn - - def _setup_dataclasses_transforms(self) -> None: - dataclass_setup_arguments = self.dataclass_setup_arguments - if not dataclass_setup_arguments: - return - - # can't use is_dataclass since it uses hasattr - if "__dataclass_fields__" in self.cls.__dict__: - raise exc.InvalidRequestError( - f"Class {self.cls} is already a dataclass; ensure that " - "base classes / decorator styles of establishing dataclasses " - "are not being mixed. " - "This can happen if a class that inherits from " - "'MappedAsDataclass', even indirectly, is been mapped with " - "'@registry.mapped_as_dataclass'" - ) - - # can't create a dataclass if __table__ is already there. This would - # fail an assertion when calling _get_arguments_for_make_dataclass: - # assert False, "Mapped[] received without a mapping declaration" - if "__table__" in self.cls.__dict__: - raise exc.InvalidRequestError( - f"Class {self.cls} already defines a '__table__'. " - "ORM Annotated Dataclasses do not support a pre-existing " - "'__table__' element" - ) - - warn_for_non_dc_attrs = collections.defaultdict(list) - - def _allow_dataclass_field( - key: str, originating_class: Type[Any] - ) -> bool: - if ( - originating_class is not self.cls - and "__dataclass_fields__" not in originating_class.__dict__ - ): - warn_for_non_dc_attrs[originating_class].append(key) - - return True - - manager = instrumentation.manager_of_class(self.cls) - assert manager is not None - - field_list = [ - _AttributeOptions._get_arguments_for_make_dataclass( - key, - anno, - mapped_container, - self.collected_attributes.get(key, _NoArg.NO_ARG), - ) - for key, anno, mapped_container in ( - ( - key, - mapped_anno if mapped_anno else raw_anno, - mapped_container, - ) - for key, ( - raw_anno, - mapped_container, - mapped_anno, - is_dc, - attr_value, - originating_module, - originating_class, - ) in self.collected_annotations.items() - if _allow_dataclass_field(key, originating_class) - and ( - key not in self.collected_attributes - # issue #9226; check for attributes that we've collected - # which are already instrumented, which we would assume - # mean we are in an ORM inheritance mapping and this - # attribute is already mapped on the superclass. Under - # no circumstance should any QueryableAttribute be sent to - # the dataclass() function; anything that's mapped should - # be Field and that's it - or not isinstance( - self.collected_attributes[key], QueryableAttribute - ) - ) - ) - ] + ) + continue + else: + # here, the attribute is some other kind of + # property that we assume is not part of the + # declarative mapping. however, check for some + # more common mistakes + self._warn_for_decl_attributes(base, name, obj) + elif is_dataclass_field and ( + name not in clsdict_view or clsdict_view[name] is not obj + ): + # here, we are definitely looking at the target class + # and not a superclass. this is currently a + # dataclass-only path. if the name is only + # a dataclass field and isn't in local cls.__dict__, + # put the object there. + # assert that the dataclass-enabled resolver agrees + # with what we are seeing - if warn_for_non_dc_attrs: - for ( - originating_class, - non_dc_attrs, - ) in warn_for_non_dc_attrs.items(): - util.warn_deprecated( - f"When transforming {self.cls} to a dataclass, " - f"attribute(s) " - f"{', '.join(repr(key) for key in non_dc_attrs)} " - f"originates from superclass " - f"{originating_class}, which is not a dataclass. This " - f"usage is deprecated and will raise an error in " - f"SQLAlchemy 2.1. When declaring SQLAlchemy Declarative " - f"Dataclasses, ensure that all mixin classes and other " - f"superclasses which include attributes are also a " - f"subclass of MappedAsDataclass.", - "2.0", - code="dcmx", - ) + assert not attribute_is_overridden(name, obj) - annotations = {} - defaults = {} - for item in field_list: - if len(item) == 2: - name, tp = item - elif len(item) == 3: - name, tp, spec = item - defaults[name] = spec - else: - assert False - annotations[name] = tp + if _is_declarative_props(obj): + obj = obj.fget() - for k, v in defaults.items(): - setattr(self.cls, k, v) + collected_attributes[name] = obj + self._collect_annotation( + name, annotation, base, False, obj + ) + else: + collected_annotation = self._collect_annotation( + name, annotation, base, None, obj + ) + is_mapped = ( + collected_annotation is not None + and collected_annotation.mapped_container is not None + ) + generated_obj = ( + collected_annotation.attr_value + if collected_annotation is not None + else obj + ) + if obj is None and not fixed_table and is_mapped: + collected_attributes[name] = ( + generated_obj + if generated_obj is not None + else MappedColumn() + ) + elif name in clsdict_view: + collected_attributes[name] = obj + # else if the name is not in the cls.__dict__, + # don't collect it as an attribute. + # we will see the annotation only, which is meaningful + # both for mapping and dataclasses setup - self._apply_dataclasses_to_any_class( - dataclass_setup_arguments, self.cls, annotations - ) + if inherited_table_args and not tablename: + table_args = None + + self.table_args = table_args + self.tablename = tablename + self.mapper_args_fn = mapper_args_fn + self.table_fn = table_fn @classmethod def _update_annotations_for_non_mapped_class( @@ -1197,152 +1370,6 @@ def _update_annotations_for_non_mapped_class( new_anno[name] = annotation return new_anno - @classmethod - def _apply_dataclasses_to_any_class( - cls, - dataclass_setup_arguments: _DataclassArguments, - klass: Type[_O], - use_annotations: Mapping[str, _AnnotationScanType], - ) -> None: - cls._assert_dc_arguments(dataclass_setup_arguments) - - dataclass_callable = dataclass_setup_arguments["dataclass_callable"] - if dataclass_callable is _NoArg.NO_ARG: - dataclass_callable = dataclasses.dataclass - - restored: Optional[Any] - - if use_annotations: - # apply constructed annotations that should look "normal" to a - # dataclasses callable, based on the fields present. This - # means remove the Mapped[] container and ensure all Field - # entries have an annotation - restored = getattr(klass, "__annotations__", None) - klass.__annotations__ = cast("Dict[str, Any]", use_annotations) - else: - restored = None - - try: - dataclass_callable( - klass, - **{ - k: v - for k, v in dataclass_setup_arguments.items() - if v is not _NoArg.NO_ARG and k != "dataclass_callable" - }, - ) - except (TypeError, ValueError) as ex: - raise exc.InvalidRequestError( - f"Python dataclasses error encountered when creating " - f"dataclass for {klass.__name__!r}: " - f"{ex!r}. Please refer to Python dataclasses " - "documentation for additional information.", - code="dcte", - ) from ex - finally: - # restore original annotations outside of the dataclasses - # process; for mixins and __abstract__ superclasses, SQLAlchemy - # Declarative will need to see the Mapped[] container inside the - # annotations in order to map subclasses - if use_annotations: - if restored is None: - del klass.__annotations__ - else: - klass.__annotations__ = restored - - @classmethod - def _assert_dc_arguments(cls, arguments: _DataclassArguments) -> None: - allowed = { - "init", - "repr", - "order", - "eq", - "unsafe_hash", - "kw_only", - "match_args", - "dataclass_callable", - } - disallowed_args = set(arguments).difference(allowed) - if disallowed_args: - msg = ", ".join(f"{arg!r}" for arg in sorted(disallowed_args)) - raise exc.ArgumentError( - f"Dataclass argument(s) {msg} are not accepted" - ) - - def _collect_annotation( - self, - name: str, - raw_annotation: _AnnotationScanType, - originating_class: Type[Any], - expect_mapped: Optional[bool], - attr_value: Any, - ) -> Optional[_CollectedAnnotation]: - if name in self.collected_annotations: - return self.collected_annotations[name] - - if raw_annotation is None: - return None - - is_dataclass = self.is_dataclass_prior_to_mapping - allow_unmapped = self.allow_unmapped_annotations - - if expect_mapped is None: - is_dataclass_field = isinstance(attr_value, dataclasses.Field) - expect_mapped = ( - not is_dataclass_field - and not allow_unmapped - and ( - attr_value is None - or isinstance(attr_value, _MappedAttribute) - ) - ) - else: - is_dataclass_field = False - - is_dataclass_field = False - extracted = _extract_mapped_subtype( - raw_annotation, - self.cls, - originating_class.__module__, - name, - type(attr_value), - required=False, - is_dataclass_field=is_dataclass_field, - expect_mapped=expect_mapped and not is_dataclass, - ) - if extracted is None: - # ClassVar can come out here - return None - - extracted_mapped_annotation, mapped_container = extracted - - if attr_value is None and not is_literal(extracted_mapped_annotation): - for elem in get_args(extracted_mapped_annotation): - if is_fwd_ref( - elem, check_generic=True, check_for_plain_string=True - ): - elem = de_stringify_annotation( - self.cls, - elem, - originating_class.__module__, - include_generic=True, - ) - # look in Annotated[...] for an ORM construct, - # such as Annotated[int, mapped_column(primary_key=True)] - if isinstance(elem, _IntrospectsAnnotations): - attr_value = elem.found_in_pep593_annotated() - - self.collected_annotations[name] = ca = _CollectedAnnotation( - raw_annotation, - mapped_container, - extracted_mapped_annotation, - is_dataclass, - attr_value, - originating_class.__module__, - originating_class, - ) - return ca - def _warn_for_decl_attributes( self, cls: Type[Any], key: str, c: Any ) -> None: @@ -1577,7 +1604,7 @@ def _extract_mappable_attributes(self) -> None: is_dataclass, ) except NameError as ne: - raise exc.ArgumentError( + raise orm_exc.MappedAnnotationError( f"Could not resolve all types within mapped " f'annotation: "{annotation}". Ensure all ' f"types are written correctly and are " @@ -1601,9 +1628,15 @@ def _extract_mappable_attributes(self) -> None: "default_factory", "repr", "default", + "dataclass_metadata", ] else: - argnames = ["init", "default_factory", "repr"] + argnames = [ + "init", + "default_factory", + "repr", + "dataclass_metadata", + ] args = { a @@ -1787,10 +1820,10 @@ def _metadata_for_cls(self, manager: ClassManager[Any]) -> MetaData: else: return manager.registry.metadata - def _setup_inheriting_mapper(self, mapper_kw: _MapperKwArgs) -> None: + def _setup_inheriting_mapper(self) -> None: cls = self.cls - inherits = mapper_kw.get("inherits", None) + inherits = None if inherits is None: # since we search for classical mappings now, search for @@ -1820,7 +1853,7 @@ def _setup_inheriting_mapper(self, mapper_kw: _MapperKwArgs) -> None: if "__table__" not in clsdict_view and self.tablename is None: self.single = True - def _setup_inheriting_columns(self, mapper_kw: _MapperKwArgs) -> None: + def _setup_inheriting_columns(self) -> None: table = self.local_table cls = self.cls table_args = self.table_args @@ -1991,6 +2024,86 @@ def map(self, mapper_kw: _MapperKwArgs = util.EMPTY_DICT) -> Mapper[Any]: ) +class _UnmappedDataclassConfig(_ClassScanAbstractConfig): + """Configurator that will produce an unmapped dataclass.""" + + __slots__ = ( + "clsdict_view", + "collected_attributes", + "collected_annotations", + "allow_dataclass_fields", + "dataclass_setup_arguments", + "is_dataclass_prior_to_mapping", + "allow_unmapped_annotations", + ) + + def __init__( + self, + cls_: Type[_O], + dict_: _ClassDict, + ): + super().__init__(cls_) + self.clsdict_view = ( + util.immutabledict(dict_) if dict_ else util.EMPTY_DICT + ) + self.dataclass_setup_arguments = getattr( + self.cls, "_sa_apply_dc_transforms", None + ) + + self.is_dataclass_prior_to_mapping = dataclasses.is_dataclass(cls_) + self.allow_dataclass_fields = False + self.allow_unmapped_annotations = True + self.collected_attributes = {} + self.collected_annotations = {} + + self._scan_attributes() + + self._setup_dataclasses_transforms( + enable_descriptor_defaults=False, revert=True + ) + + def _scan_attributes(self) -> None: + cls = self.cls + + clsdict_view = self.clsdict_view + collected_attributes = self.collected_attributes + _include_dunders = self._include_dunders + + attribute_is_overridden = self._cls_attr_override_checker(self.cls) + + local_attributes_for_class = self._cls_attr_resolver(cls) + for ( + name, + obj, + annotation, + is_dataclass_field, + ) in local_attributes_for_class(): + if name in _include_dunders: + continue + elif is_dataclass_field and ( + name not in clsdict_view or clsdict_view[name] is not obj + ): + # here, we are definitely looking at the target class + # and not a superclass. this is currently a + # dataclass-only path. if the name is only + # a dataclass field and isn't in local cls.__dict__, + # put the object there. + # assert that the dataclass-enabled resolver agrees + # with what we are seeing + + assert not attribute_is_overridden(name, obj) + + if _is_declarative_props(obj): + obj = obj.fget() + + collected_attributes[name] = obj + self._collect_annotation(name, annotation, cls, False, obj) + else: + self._collect_annotation(name, annotation, cls, None, obj) + if name in clsdict_view: + collected_attributes[name] = obj + + @util.preload_module("sqlalchemy.orm.decl_api") def _as_dc_declaredattr( field_metadata: Mapping[str, Any], sa_dataclass_metadata_key: str @@ -2006,20 +2119,26 @@ def _as_dc_declaredattr( return obj -class _DeferredMapperConfig(_ClassScanMapperConfig): +class _DeferredDeclarativeConfig(_DeclarativeMapperConfig): + """Configurator that extends _DeclarativeMapperConfig to add a + "deferred" step, to allow extensions like AbstractConcreteBase, + DeferredMapping to partially set up a mapping that is "prepared" + when table metadata is ready. + + """ + _cls: weakref.ref[Type[Any]] is_deferred = True _configs: util.OrderedDict[ - weakref.ref[Type[Any]], _DeferredMapperConfig + weakref.ref[Type[Any]], _DeferredDeclarativeConfig ] = util.OrderedDict() def _early_mapping(self, mapper_kw: _MapperKwArgs) -> None: pass - # mypy disallows plain property override of variable - @property # type: ignore + @property def cls(self) -> Type[Any]: return self._cls() # type: ignore @@ -2051,13 +2170,13 @@ def raise_unmapped_for_cls(cls, class_: Type[Any]) -> NoReturn: ) @classmethod - def config_for_cls(cls, class_: Type[Any]) -> _DeferredMapperConfig: + def config_for_cls(cls, class_: Type[Any]) -> _DeferredDeclarativeConfig: return cls._configs[weakref.ref(class_)] @classmethod def classes_for_base( cls, base_cls: Type[Any], sort: bool = True - ) -> List[_DeferredMapperConfig]: + ) -> List[_DeferredDeclarativeConfig]: classes_for_base = [ m for m, cls_ in [(m, m.cls) for m in cls._configs.values()] @@ -2069,7 +2188,9 @@ def classes_for_base( all_m_by_cls = {m.cls: m for m in classes_for_base} - tuples: List[Tuple[_DeferredMapperConfig, _DeferredMapperConfig]] = [] + tuples: List[ + Tuple[_DeferredDeclarativeConfig, _DeferredDeclarativeConfig] + ] = [] for m_cls in all_m_by_cls: tuples.extend( (all_m_by_cls[base_cls], all_m_by_cls[m_cls]) diff --git a/lib/sqlalchemy/orm/dependency.py b/lib/sqlalchemy/orm/dependency.py index 88413485c4c..2c6158dd3fc 100644 --- a/lib/sqlalchemy/orm/dependency.py +++ b/lib/sqlalchemy/orm/dependency.py @@ -1,5 +1,5 @@ # orm/dependency.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -7,9 +7,7 @@ # mypy: ignore-errors -"""Relationship dependencies. - -""" +"""Relationship dependencies.""" from __future__ import annotations @@ -1058,7 +1056,7 @@ def presort_saves(self, uowcommit, states): # so that prop_has_changes() returns True for state in states: if self._pks_changed(uowcommit, state): - history = uowcommit.get_attribute_history( + uowcommit.get_attribute_history( state, self.key, attributes.PASSIVE_OFF ) diff --git a/lib/sqlalchemy/orm/descriptor_props.py b/lib/sqlalchemy/orm/descriptor_props.py index 89124c4e439..ae810de388e 100644 --- a/lib/sqlalchemy/orm/descriptor_props.py +++ b/lib/sqlalchemy/orm/descriptor_props.py @@ -1,5 +1,5 @@ # orm/descriptor_props.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -20,6 +20,7 @@ from typing import Any from typing import Callable from typing import Dict +from typing import get_args from typing import List from typing import NoReturn from typing import Optional @@ -34,6 +35,7 @@ from . import attributes from . import util as orm_util from .base import _DeclarativeMapped +from .base import DONT_SET from .base import LoaderCallableStatus from .base import Mapped from .base import PassiveFlag @@ -43,7 +45,6 @@ from .interfaces import _MapsColumns from .interfaces import MapperProperty from .interfaces import PropComparator -from .util import _none_set from .util import de_stringify_annotation from .. import event from .. import exc as sa_exc @@ -52,10 +53,13 @@ from .. import util from ..sql import expression from ..sql import operators +from ..sql.base import _NoArg from ..sql.elements import BindParameter -from ..util.typing import get_args +from ..util.typing import de_optionalize_union_types +from ..util.typing import includes_none from ..util.typing import is_fwd_ref from ..util.typing import is_pep593 +from ..util.typing import is_union from ..util.typing import TupleAny from ..util.typing import Unpack @@ -67,7 +71,9 @@ from .attributes import InstrumentedAttribute from .attributes import QueryableAttribute from .context import _ORMCompileState - from .decl_base import _ClassScanMapperConfig + from .decl_base import _ClassScanAbstractConfig + from .decl_base import _DeclarativeMapperConfig + from .interfaces import _DataclassArguments from .mapper import Mapper from .properties import ColumnProperty from .properties import MappedColumn @@ -101,6 +107,11 @@ class DescriptorProperty(MapperProperty[_T]): descriptor: DescriptorReference[Any] + def _column_strategy_attrs(self) -> Sequence[QueryableAttribute[Any]]: + raise NotImplementedError( + "This MapperProperty does not implement column loader strategies" + ) + def get_history( self, state: InstanceState[Any], @@ -158,6 +169,7 @@ def fget(obj: Any) -> Any: doc=self.doc, original_property=self, ) + proxy_attr.impl = _ProxyImpl(self.key) mapper.class_manager.instrument_attribute(self.key, proxy_attr) @@ -209,6 +221,9 @@ def __init__( None, Type[_CC], Callable[..., _CC], _CompositeAttrType[Any] ] = None, *attrs: _CompositeAttrType[Any], + return_none_on: Union[ + _NoArg, None, Callable[..., bool] + ] = _NoArg.NO_ARG, attribute_options: Optional[_AttributeOptions] = None, active_history: bool = False, deferred: bool = False, @@ -227,6 +242,7 @@ def __init__( self.composite_class = _class_or_attr # type: ignore self.attrs = attrs + self.return_none_on = return_none_on self.active_history = active_history self.deferred = deferred self.group = group @@ -243,6 +259,21 @@ def __init__( self._create_descriptor() self._init_accessor() + @util.memoized_property + def _construct_composite(self) -> Callable[..., Any]: + return_none_on = self.return_none_on + if callable(return_none_on): + + def construct(*args: Any) -> Any: + if return_none_on(*args): + return None + else: + return self.composite_class(*args) + + return construct + else: + return self.composite_class + def instrument_class(self, mapper: Mapper[Any]) -> None: super().instrument_class(mapper) self._setup_event_handlers() @@ -289,15 +320,8 @@ def fget(instance: Any) -> Any: getattr(instance, key) for key in self._attribute_keys ] - # current expected behavior here is that the composite is - # created on access if the object is persistent or if - # col attributes have non-None. This would be better - # if the composite were created unconditionally, - # but that would be a behavioral change. - if self.key not in dict_ and ( - state.key is not None or not _none_set.issuperset(values) - ): - dict_[self.key] = self.composite_class(*values) + if self.key not in dict_: + dict_[self.key] = self._construct_composite(*values) state.manager.dispatch.refresh( state, self._COMPOSITE_FGET, [self.key] ) @@ -305,6 +329,9 @@ def fget(instance: Any) -> Any: return dict_.get(self.key, None) def fset(instance: Any, value: Any) -> None: + if value is LoaderCallableStatus.DONT_SET: + return + dict_ = attributes.instance_dict(instance) state = attributes.instance_state(instance) attr = state.manager[self.key] @@ -348,7 +375,7 @@ def fdel(instance: Any) -> None: @util.preload_module("sqlalchemy.orm.properties") def declarative_scan( self, - decl_scan: _ClassScanMapperConfig, + decl_scan: _DeclarativeMapperConfig, registry: _RegistryType, cls: Type[Any], originating_module: Optional[str], @@ -387,10 +414,19 @@ def declarative_scan( cls, argument, originating_module, include_generic=True ) + if is_union(argument) and includes_none(argument): + if self.return_none_on is _NoArg.NO_ARG: + self.return_none_on = lambda *args: all( + arg is None for arg in args + ) + argument = de_optionalize_union_types(argument) + self.composite_class = argument if is_dataclass(self.composite_class): - self._setup_for_dataclass(registry, cls, originating_module, key) + self._setup_for_dataclass( + decl_scan, registry, cls, originating_module, key + ) else: for attr in self.attrs: if ( @@ -434,6 +470,7 @@ def _init_accessor(self) -> None: @util.preload_module("sqlalchemy.orm.decl_base") def _setup_for_dataclass( self, + decl_scan: _DeclarativeMapperConfig, registry: _RegistryType, cls: Type[Any], originating_module: Optional[str], @@ -461,6 +498,7 @@ def _setup_for_dataclass( if isinstance(attr, MappedColumn): attr.declarative_scan_for_composite( + decl_scan, registry, cls, originating_module, @@ -502,6 +540,9 @@ def props(self) -> Sequence[MapperProperty[Any]]: props.append(prop) return props + def _column_strategy_attrs(self) -> Sequence[QueryableAttribute[Any]]: + return self._comparable_elements + @util.non_memoized_property @util.preload_module("orm.properties") def columns(self) -> Sequence[Column[Any]]: @@ -589,7 +630,7 @@ def _load_refresh_handler( if k not in dict_: return - dict_[self.key] = self.composite_class( + dict_[self.key] = self._construct_composite( *[state.dict[key] for key in self._attribute_keys] ) @@ -693,12 +734,14 @@ def get_history( if has_history: return attributes.History( - [self.composite_class(*added)], + [self._construct_composite(*added)], (), - [self.composite_class(*deleted)], + [self._construct_composite(*deleted)], ) else: - return attributes.History((), [self.composite_class(*added)], ()) + return attributes.History( + (), [self._construct_composite(*added)], () + ) def _comparator_factory( self, mapper: Mapper[Any] @@ -721,7 +764,7 @@ def create_row_processor( labels: Sequence[str], ) -> Callable[[Row[Unpack[TupleAny]]], Any]: def proc(row: Row[Unpack[TupleAny]]) -> Any: - return self.property.composite_class( + return self.property._construct_composite( *[proc(row) for proc in procs] ) @@ -795,6 +838,9 @@ def _bulk_update_tuples( return list(zip(self._comparable_elements, values)) + def _bulk_dml_setter(self, key: str) -> Optional[Callable[..., Any]]: + return self.prop._populate_composite_bulk_save_mappings_fn() + @util.memoized_property def _comparable_elements(self) -> Sequence[QueryableAttribute[Any]]: if self._adapt_to_entity: @@ -823,6 +869,26 @@ def __le__(self, other: Any) -> ColumnElement[bool]: def __ge__(self, other: Any) -> ColumnElement[bool]: return self._compare(operators.ge, other) + def desc(self) -> operators.OrderingOperators: # type: ignore[override] # noqa: E501 + return expression.OrderByList( + [e.desc() for e in self._comparable_elements] + ) + + def asc(self) -> operators.OrderingOperators: # type: ignore[override] # noqa: E501 + return expression.OrderByList( + [e.asc() for e in self._comparable_elements] + ) + + def nulls_first(self) -> operators.OrderingOperators: # type: ignore[override] # noqa: E501 + return expression.OrderByList( + [e.nulls_first() for e in self._comparable_elements] + ) + + def nulls_last(self) -> operators.OrderingOperators: # type: ignore[override] # noqa: E501 + return expression.OrderByList( + [e.nulls_last() for e in self._comparable_elements] + ) + # what might be interesting would be if we create # an instance of the composite class itself with # the columns as data members, then use "hybrid style" comparison @@ -991,7 +1057,7 @@ def _proxied_object( if isinstance(attr, attributes.QueryableAttribute): return attr.comparator elif isinstance(attr, SQLORMOperations): - # assocaition proxy comes here + # association proxy comes here return attr raise sa_exc.InvalidRequestError( @@ -1001,6 +1067,9 @@ def _proxied_object( ) return attr.property + def _column_strategy_attrs(self) -> Sequence[QueryableAttribute[Any]]: + return (getattr(self.parent.class_, self.name),) + def _comparator_factory(self, mapper: Mapper[Any]) -> SQLORMOperations[_T]: prop = self._proxied_object @@ -1022,6 +1091,41 @@ def get_history( attr: QueryableAttribute[Any] = getattr(self.parent.class_, self.name) return attr.impl.get_history(state, dict_, passive=passive) + def _get_dataclass_setup_options( + self, + decl_scan: _ClassScanAbstractConfig, + key: str, + dataclass_setup_arguments: _DataclassArguments, + enable_descriptor_defaults: bool, + ) -> _AttributeOptions: + dataclasses_default = self._attribute_options.dataclasses_default + if ( + dataclasses_default is not _NoArg.NO_ARG + and not callable(dataclasses_default) + and enable_descriptor_defaults + and not getattr( + decl_scan.cls, "_sa_disable_descriptor_defaults", False + ) + ): + proxied = decl_scan.collected_attributes[self.name] + proxied_default = proxied._attribute_options.dataclasses_default + if proxied_default != dataclasses_default: + raise sa_exc.ArgumentError( + f"Synonym {key!r} default argument " + f"{dataclasses_default!r} must match the dataclasses " + f"default value of proxied object {self.name!r}, " + f"""currently { + repr(proxied_default) + if proxied_default is not _NoArg.NO_ARG + else 'not set'}""" + ) + self._default_scalar_value = dataclasses_default + return self._attribute_options._replace( + dataclasses_default=DONT_SET + ) + + return self._attribute_options + @util.preload_module("sqlalchemy.orm.properties") def set_parent(self, parent: Mapper[Any], init: bool) -> None: properties = util.preloaded.orm_properties diff --git a/lib/sqlalchemy/orm/dynamic.py b/lib/sqlalchemy/orm/dynamic.py index 6961170ff63..b7ed1c9399d 100644 --- a/lib/sqlalchemy/orm/dynamic.py +++ b/lib/sqlalchemy/orm/dynamic.py @@ -1,5 +1,5 @@ # orm/dynamic.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -24,6 +24,7 @@ from typing import Iterator from typing import List from typing import Optional +from typing import overload from typing import Tuple from typing import Type from typing import TYPE_CHECKING @@ -176,7 +177,13 @@ def _iter(self) -> Union[result.ScalarResult[_T], result.Result[_T]]: def __iter__(self) -> Iterator[_T]: ... - def __getitem__(self, index: Any) -> Union[_T, List[_T]]: + @overload + def __getitem__(self, index: int) -> _T: ... + + @overload + def __getitem__(self, index: slice) -> List[_T]: ... + + def __getitem__(self, index: Union[int, slice]) -> Union[_T, List[_T]]: sess = self.session if sess is None: return self.attr._get_collection_history( @@ -184,7 +191,7 @@ def __getitem__(self, index: Any) -> Union[_T, List[_T]]: PassiveFlag.PASSIVE_NO_INITIALIZE, ).indexed(index) else: - return self._generate(sess).__getitem__(index) # type: ignore[no-any-return] # noqa: E501 + return self._generate(sess).__getitem__(index) def count(self) -> int: sess = self.session diff --git a/lib/sqlalchemy/orm/evaluator.py b/lib/sqlalchemy/orm/evaluator.py index 57aae5a3c49..392d726e990 100644 --- a/lib/sqlalchemy/orm/evaluator.py +++ b/lib/sqlalchemy/orm/evaluator.py @@ -1,5 +1,5 @@ # orm/evaluator.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/orm/events.py b/lib/sqlalchemy/orm/events.py index 63e7ff20464..c97c3d94454 100644 --- a/lib/sqlalchemy/orm/events.py +++ b/lib/sqlalchemy/orm/events.py @@ -1,13 +1,11 @@ # orm/events.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -"""ORM event interfaces. - -""" +"""ORM event interfaces.""" from __future__ import annotations from typing import Any @@ -25,6 +23,7 @@ from typing import Union import weakref +from . import decl_api from . import instrumentation from . import interfaces from . import mapperlib @@ -66,6 +65,7 @@ from ..orm.context import QueryContext from ..orm.decl_api import DeclarativeAttributeIntercept from ..orm.decl_api import DeclarativeMeta + from ..orm.decl_api import registry from ..orm.mapper import Mapper from ..orm.state import InstanceState @@ -245,9 +245,6 @@ class which is the target of this listener. object is moved to a new loader context from within one of these events if this flag is not set. - .. versionadded:: 1.3.14 - - """ _target_class_doc = "SomeClass" @@ -462,15 +459,6 @@ def load(self, target: _O, context: QueryContext) -> None: def on_load(instance, context): instance.some_unloaded_attribute - .. versionchanged:: 1.3.14 Added - :paramref:`.InstanceEvents.restore_load_context` - and :paramref:`.SessionEvents.restore_load_context` flags which - apply to "on load" events, which will ensure that the loading - context for an object is restored when the event hook is - complete; a warning is emitted if the load context of the object - changes without this flag being set. - - The :meth:`.InstanceEvents.load` event is also available in a class-method decorator format called :func:`_orm.reconstructor`. @@ -727,7 +715,8 @@ class _InstanceEventsHold(_EventsHold[_ET]): def resolve(self, class_: Type[_O]) -> Optional[ClassManager[_O]]: return instrumentation.opt_manager_of_class(class_) - class HoldInstanceEvents(_EventsHold.HoldEvents[_ET], InstanceEvents): # type: ignore [misc] # noqa: E501 + # this fails on pyright if you use Any. Fails on mypy if you use _ET + class HoldInstanceEvents(_EventsHold.HoldEvents[_ET], InstanceEvents): # type: ignore[valid-type,misc] # noqa: E501 pass dispatch = event.dispatcher(HoldInstanceEvents) @@ -827,7 +816,14 @@ def _accept_with( "event target, use the 'sqlalchemy.orm.Mapper' class.", "2.0", ) - return mapperlib.Mapper + target = mapperlib.Mapper + + if identifier in ("before_configured", "after_configured"): + if target is mapperlib.Mapper: + return target + else: + return None + elif isinstance(target, type): if issubclass(target, mapperlib.Mapper): return target @@ -855,16 +851,6 @@ def _listen( event_key._listen_fn, ) - if ( - identifier in ("before_configured", "after_configured") - and target is not mapperlib.Mapper - ): - util.warn( - "'before_configured' and 'after_configured' ORM events " - "only invoke with the Mapper class " - "as the target." - ) - if not raw or not retval: if not raw: meth = getattr(cls, identifier) @@ -966,44 +952,42 @@ def after_mapper_constructed( """ + @event._omit_standard_example def before_mapper_configured( self, mapper: Mapper[_O], class_: Type[_O] ) -> None: """Called right before a specific mapper is to be configured. - This event is intended to allow a specific mapper to be skipped during - the configure step, by returning the :attr:`.orm.interfaces.EXT_SKIP` - symbol which indicates to the :func:`.configure_mappers` call that this - particular mapper (or hierarchy of mappers, if ``propagate=True`` is - used) should be skipped in the current configuration run. When one or - more mappers are skipped, the "new mappers" flag will remain set, - meaning the :func:`.configure_mappers` function will continue to be - called when mappers are used, to continue to try to configure all - available mappers. - - In comparison to the other configure-level events, - :meth:`.MapperEvents.before_configured`, - :meth:`.MapperEvents.after_configured`, and - :meth:`.MapperEvents.mapper_configured`, the - :meth:`.MapperEvents.before_mapper_configured` event provides for a - meaningful return value when it is registered with the ``retval=True`` - parameter. - - .. versionadded:: 1.3 - - e.g.:: - + The :meth:`.MapperEvents.before_mapper_configured` event is invoked + for each mapper that is encountered when the + :func:`_orm.configure_mappers` function proceeds through the current + list of not-yet-configured mappers. It is similar to the + :meth:`.MapperEvents.mapper_configured` event, except that it's invoked + right before the configuration occurs, rather than afterwards. + + The :meth:`.MapperEvents.before_mapper_configured` event includes + the special capability where it can force the configure step for a + specific mapper to be skipped; to use this feature, establish + the event using the ``retval=True`` parameter and return + the :attr:`.orm.interfaces.EXT_SKIP` symbol to indicate the mapper + should be left unconfigured:: + + from sqlalchemy import event from sqlalchemy.orm import EXT_SKIP + from sqlalchemy.orm import DeclarativeBase - Base = declarative_base() - DontConfigureBase = declarative_base() + class DontConfigureBase(DeclarativeBase): + pass @event.listens_for( DontConfigureBase, "before_mapper_configured", + # support return values for the event retval=True, + # propagate the listener to all subclasses of + # DontConfigureBase propagate=True, ) def dont_configure(mapper, cls): @@ -1015,6 +999,10 @@ def dont_configure(mapper, cls): :meth:`.MapperEvents.after_configured` + :meth:`.RegistryEvents.before_configured` + + :meth:`.RegistryEvents.after_configured` + :meth:`.MapperEvents.mapper_configured` """ @@ -1048,15 +1036,14 @@ def mapper_configured(self, mapper: Mapper[_O], class_: Type[_O]) -> None: event; this event invokes only after all known mappings have been fully configured. - The :meth:`.MapperEvents.mapper_configured` event, unlike + The :meth:`.MapperEvents.mapper_configured` event, unlike the :meth:`.MapperEvents.before_configured` or - :meth:`.MapperEvents.after_configured`, - is called for each mapper/class individually, and the mapper is - passed to the event itself. It also is called exactly once for - a particular mapper. The event is therefore useful for - configurational steps that benefit from being invoked just once - on a specific mapper basis, which don't require that "backref" - configurations are necessarily ready yet. + :meth:`.MapperEvents.after_configured` events, is called for each + mapper/class individually, and the mapper is passed to the event + itself. It also is called exactly once for a particular mapper. The + event is therefore useful for configurational steps that benefit from + being invoked just once on a specific mapper basis, which don't require + that "backref" configurations are necessarily ready yet. :param mapper: the :class:`_orm.Mapper` which is the target of this event. @@ -1068,11 +1055,16 @@ def mapper_configured(self, mapper: Mapper[_O], class_: Type[_O]) -> None: :meth:`.MapperEvents.after_configured` + :meth:`.RegistryEvents.before_configured` + + :meth:`.RegistryEvents.after_configured` + :meth:`.MapperEvents.before_mapper_configured` """ # TODO: need coverage for this event + @event._omit_standard_example def before_configured(self) -> None: """Called before a series of mappers have been configured. @@ -1084,9 +1076,15 @@ def before_configured(self) -> None: new mappers have been made available and new mapper use is detected. + Similar events to this one include + :meth:`.MapperEvents.after_configured`, which is invoked after a series + of mappers has been configured, as well as + :meth:`.MapperEvents.before_mapper_configured` and + :meth:`.MapperEvents.mapper_configured`, which are both invoked on a + per-mapper basis. + This event can **only** be applied to the :class:`_orm.Mapper` class, - and not to individual mappings or mapped classes. It is only invoked - for all mappings as a whole:: + and not to individual mappings or mapped classes:: from sqlalchemy.orm import Mapper @@ -1094,25 +1092,11 @@ def before_configured(self) -> None: @event.listens_for(Mapper, "before_configured") def go(): ... - Contrast this event to :meth:`.MapperEvents.after_configured`, - which is invoked after the series of mappers has been configured, - as well as :meth:`.MapperEvents.before_mapper_configured` - and :meth:`.MapperEvents.mapper_configured`, which are both invoked - on a per-mapper basis. - - Theoretically this event is called once per - application, but is actually called any time new mappers - are to be affected by a :func:`_orm.configure_mappers` - call. If new mappings are constructed after existing ones have - already been used, this event will likely be called again. To ensure - that a particular event is only called once and no further, the - ``once=True`` argument (new in 0.9.4) can be applied:: - - from sqlalchemy.orm import mapper - - - @event.listens_for(mapper, "before_configured", once=True) - def go(): ... + Typically, this event is called once per application, but in practice + may be called more than once, any time new mappers are to be affected + by a :func:`_orm.configure_mappers` call. If new mappings are + constructed after existing ones have already been used, this event will + likely be called again. .. seealso:: @@ -1122,8 +1106,13 @@ def go(): ... :meth:`.MapperEvents.after_configured` + :meth:`.RegistryEvents.before_configured` + + :meth:`.RegistryEvents.after_configured` + """ + @event._omit_standard_example def after_configured(self) -> None: """Called after a series of mappers have been configured. @@ -1135,17 +1124,15 @@ def after_configured(self) -> None: new mappers have been made available and new mapper use is detected. - Contrast this event to the :meth:`.MapperEvents.mapper_configured` - event, which is called on a per-mapper basis while the configuration - operation proceeds; unlike that event, when this event is invoked, - all cross-configurations (e.g. backrefs) will also have been made - available for any mappers that were pending. - Also contrast to :meth:`.MapperEvents.before_configured`, - which is invoked before the series of mappers has been configured. + Similar events to this one include + :meth:`.MapperEvents.before_configured`, which is invoked before a + series of mappers are configured, as well as + :meth:`.MapperEvents.before_mapper_configured` and + :meth:`.MapperEvents.mapper_configured`, which are both invoked on a + per-mapper basis. This event can **only** be applied to the :class:`_orm.Mapper` class, - and not to individual mappings or - mapped classes. It is only invoked for all mappings as a whole:: + and not to individual mappings or mapped classes:: from sqlalchemy.orm import Mapper @@ -1153,19 +1140,11 @@ def after_configured(self) -> None: @event.listens_for(Mapper, "after_configured") def go(): ... - Theoretically this event is called once per - application, but is actually called any time new mappers - have been affected by a :func:`_orm.configure_mappers` - call. If new mappings are constructed after existing ones have - already been used, this event will likely be called again. To ensure - that a particular event is only called once and no further, the - ``once=True`` argument (new in 0.9.4) can be applied:: - - from sqlalchemy.orm import mapper - - - @event.listens_for(mapper, "after_configured", once=True) - def go(): ... + Typically, this event is called once per application, but in practice + may be called more than once, any time new mappers are to be affected + by a :func:`_orm.configure_mappers` call. If new mappings are + constructed after existing ones have already been used, this event will + likely be called again. .. seealso:: @@ -1175,6 +1154,10 @@ def go(): ... :meth:`.MapperEvents.before_configured` + :meth:`.RegistryEvents.before_configured` + + :meth:`.RegistryEvents.after_configured` + """ def before_insert( @@ -1535,7 +1518,8 @@ def resolve( ) -> Optional[Mapper[_T]]: return _mapper_or_none(class_) - class HoldMapperEvents(_EventsHold.HoldEvents[_ET], MapperEvents): # type: ignore [misc] # noqa: E501 + # this fails on pyright if you use Any. Fails on mypy if you use _ET + class HoldMapperEvents(_EventsHold.HoldEvents[_ET], MapperEvents): # type: ignore[valid-type,misc] # noqa: E501 pass dispatch = event.dispatcher(HoldMapperEvents) @@ -1574,8 +1558,6 @@ def my_before_commit(session): objects will be the instance's :class:`.InstanceState` management object, rather than the mapped instance itself. - .. versionadded:: 1.3.14 - :param restore_load_context=False: Applies to the :meth:`.SessionEvents.loaded_as_persistent` event. Restores the loader context of the object when the event hook is complete, so that ongoing @@ -1583,8 +1565,6 @@ def my_before_commit(session): warning is emitted if the object is moved to a new loader context from within this event if this flag is not set. - .. versionadded:: 1.3.14 - """ _target_class_doc = "SomeSessionClassOrObject" @@ -1592,7 +1572,7 @@ def my_before_commit(session): _dispatch_target = Session def _lifecycle_event( # type: ignore [misc] - fn: Callable[[SessionEvents, Session, Any], None] + fn: Callable[[SessionEvents, Session, Any], None], ) -> Callable[[SessionEvents, Session, Any], None]: _sessionevents_lifecycle_event_names.add(fn.__name__) return fn @@ -2705,8 +2685,6 @@ def process_collection(target, value, initiator): else: return value - .. versionadded:: 1.2 - :param target: the object instance receiving the event. If the listener is registered with ``raw=True``, this will be the :class:`.InstanceState` object. @@ -2993,11 +2971,6 @@ def dispose_collection( The old collection received will contain its previous contents. - .. versionchanged:: 1.2 The collection passed to - :meth:`.AttributeEvents.dispose_collection` will now have its - contents before the dispose intact; previously, the collection - would be empty. - .. seealso:: :class:`.AttributeEvents` - background on listener options such @@ -3012,8 +2985,6 @@ def modified(self, target: _O, initiator: Event) -> None: function is used to trigger a modify event on an attribute without any specific value being set. - .. versionadded:: 1.2 - :param target: the object instance receiving the event. If the listener is registered with ``raw=True``, this will be the :class:`.InstanceState` object. @@ -3098,11 +3069,6 @@ def my_event(query): once, and not called for subsequent invocations of a particular query that is being cached. - .. versionadded:: 1.3.11 - added the "bake_ok" flag to the - :meth:`.QueryEvents.before_compile` event and disallowed caching via - the "baked" extension from occurring for event handlers that - return a new :class:`_query.Query` object if this flag is not set. - .. seealso:: :meth:`.QueryEvents.before_compile_update` @@ -3156,8 +3122,6 @@ def no_deleted(query, update_context): dictionary can be modified to alter the VALUES clause of the resulting UPDATE statement. - .. versionadded:: 1.2.17 - .. seealso:: :meth:`.QueryEvents.before_compile` @@ -3197,8 +3161,6 @@ def no_deleted(query, delete_context): the same kind of object as described in :paramref:`.QueryEvents.after_bulk_delete.delete_context`. - .. versionadded:: 1.2.17 - .. seealso:: :meth:`.QueryEvents.before_compile` @@ -3239,3 +3201,186 @@ def wrap(*arg: Any, **kw: Any) -> Any: wrap._bake_ok = bake_ok # type: ignore [attr-defined] event_key.base_listen(**kw) + + +class RegistryEvents(event.Events["registry"]): + """Define events specific to :class:`_orm.registry` lifecycle. + + The :class:`_orm.RegistryEvents` class defines events that are specific + to the lifecycle and operation of the :class:`_orm.registry` object. + + e.g.:: + + from typing import Any + + from sqlalchemy import event + from sqlalchemy.orm import registry + from sqlalchemy.orm import TypeResolve + from sqlalchemy.types import TypeEngine + + reg = registry() + + + @event.listens_for(reg, "resolve_type_annotation") + def resolve_custom_type( + resolve_type: TypeResolve, + ) -> TypeEngine[Any] | None: + if python_type is MyCustomType: + return MyCustomSQLType() + return None + + The events defined by :class:`_orm.RegistryEvents` include + :meth:`_orm.RegistryEvents.resolve_type_annotation`, + :meth:`_orm.RegistryEvents.before_configured`, and + :meth:`_orm.RegistryEvents.after_configured`.`. These events may be + applied to a :class:`_orm.registry` object as shown in the preceding + example, as well as to a declarative base class directly, which will + automatically locate the registry for the event to be applied:: + + from typing import Any + + from sqlalchemy import event + from sqlalchemy.orm import DeclarativeBase + from sqlalchemy.orm import registry as RegistryType + from sqlalchemy.orm import TypeResolve + from sqlalchemy.types import TypeEngine + + + class Base(DeclarativeBase): + pass + + + @event.listens_for(Base, "resolve_type_annotation") + def resolve_custom_type( + resolve_type: TypeResolve, + ) -> TypeEngine[Any] | None: + if resolve_type.resolved_type is MyCustomType: + return MyCustomSQLType() + else: + return None + + + @event.listens_for(Base, "after_configured") + def after_base_configured(registry: RegistryType) -> None: + print(f"Registry {registry} fully configured") + + .. versionadded:: 2.1 + + + """ + + _target_class_doc = "SomeRegistry" + _dispatch_target = decl_api.registry + + @classmethod + def _accept_with( + cls, + target: Any, + identifier: str, + ) -> Any: + if isinstance(target, decl_api.registry): + return target + elif ( + isinstance(target, type) + and "_sa_registry" in target.__dict__ + and isinstance(target.__dict__["_sa_registry"], decl_api.registry) + ): + return target._sa_registry # type: ignore[attr-defined] + else: + return None + + @classmethod + def _listen( + cls, + event_key: _EventKey["registry"], + **kw: Any, + ) -> None: + identifier = event_key.identifier + + # Only resolve_type_annotation needs retval=True + if identifier == "resolve_type_annotation": + kw["retval"] = True + + event_key.base_listen(**kw) + + def resolve_type_annotation( + self, resolve_type: decl_api.TypeResolve + ) -> Optional[Any]: + """Intercept and customize type annotation resolution. + + This event is fired when the :class:`_orm.registry` attempts to + resolve a Python type annotation to a SQLAlchemy type. This is + particularly useful for handling advanced typing scenarios such as + nested :pep:`695` type aliases. + + The :meth:`.RegistryEvents.resolve_type_annotation` event automatically + sets up ``retval=True`` when the event is set up, so that implementing + functions may return a resolved type, or ``None`` to indicate no type + was resolved, and the default resolution for the type should proceed. + + :param resolve_type: A :class:`_orm.TypeResolve` object which contains + all the relevant information about the type, including a link to the + registry and its resolver function. + + :return: A SQLAlchemy type to use for the given Python type. If + ``None`` is returned, the default resolution behavior will proceed + from there. + + .. versionadded:: 2.1 + + .. seealso:: + + :ref:`orm_declarative_resolve_type_event` + + """ + + def before_configured(self, registry: "registry") -> None: + """Called before a series of mappers in this registry are configured. + + This event is invoked each time the :func:`_orm.configure_mappers` + function is invoked and this registry has mappers that are part of + the configuration process. + + Compared to the :meth:`.MapperEvents.before_configured` event hook, + this event is local to the mappers within a specific + :class:`_orm.registry` and not for all :class:`.Mapper` objects + globally. + + :param registry: The :class:`_orm.registry` instance. + + .. versionadded:: 2.1 + + .. seealso:: + + :meth:`.RegistryEvents.after_configured` + + :meth:`.MapperEvents.before_configured` + + :meth:`.MapperEvents.after_configured` + + """ + + def after_configured(self, registry: "registry") -> None: + """Called after a series of mappers in this registry are configured. + + This event is invoked each time the :func:`_orm.configure_mappers` + function completes and this registry had mappers that were part of + the configuration process. + + Compared to the :meth:`.MapperEvents.after_configured` event hook, this + event is local to the mappers within a specific :class:`_orm.registry` + and not for all :class:`.Mapper` objects globally. + + :param registry: The :class:`_orm.registry` instance. + + .. versionadded:: 2.1 + + .. seealso:: + + :meth:`.RegistryEvents.before_configured` + + :meth:`.MapperEvents.before_configured` + + :meth:`.MapperEvents.after_configured` + + """ diff --git a/lib/sqlalchemy/orm/exc.py b/lib/sqlalchemy/orm/exc.py index 0494edf983a..fec401b740f 100644 --- a/lib/sqlalchemy/orm/exc.py +++ b/lib/sqlalchemy/orm/exc.py @@ -1,5 +1,5 @@ # orm/exc.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -65,6 +65,15 @@ class FlushError(sa_exc.SQLAlchemyError): """A invalid condition was detected during flush().""" +class MappedAnnotationError(sa_exc.ArgumentError): + """Raised when ORM annotated declarative cannot interpret the + expression present inside of the :class:`.Mapped` construct. + + .. versionadded:: 2.0.40 + + """ + + class UnmappedError(sa_exc.InvalidRequestError): """Base for exceptions that involve expected mappings not present.""" diff --git a/lib/sqlalchemy/orm/identity.py b/lib/sqlalchemy/orm/identity.py index fe1164d57c0..e4428801ac9 100644 --- a/lib/sqlalchemy/orm/identity.py +++ b/lib/sqlalchemy/orm/identity.py @@ -1,5 +1,5 @@ # orm/identity.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/orm/instrumentation.py b/lib/sqlalchemy/orm/instrumentation.py index 95f25b573bf..6f734e8b862 100644 --- a/lib/sqlalchemy/orm/instrumentation.py +++ b/lib/sqlalchemy/orm/instrumentation.py @@ -1,5 +1,5 @@ # orm/instrumentation.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -21,13 +21,6 @@ module, which provides the means to build and specify alternate instrumentation forms. -.. versionchanged: 0.8 - The instrumentation extension system was moved out of the - ORM and into the external :mod:`sqlalchemy.ext.instrumentation` - package. When that package is imported, it installs - itself within sqlalchemy.orm so that its more comprehensive - resolution mechanics take effect. - """ @@ -41,6 +34,7 @@ from typing import Generic from typing import Iterable from typing import List +from typing import Literal from typing import Optional from typing import Protocol from typing import Set @@ -61,7 +55,6 @@ from .. import util from ..event import EventTarget from ..util import HasMemoized -from ..util.typing import Literal if TYPE_CHECKING: from ._typing import _RegistryType diff --git a/lib/sqlalchemy/orm/interfaces.py b/lib/sqlalchemy/orm/interfaces.py index 26c29429496..b17fb0af127 100644 --- a/lib/sqlalchemy/orm/interfaces.py +++ b/lib/sqlalchemy/orm/interfaces.py @@ -1,5 +1,5 @@ # orm/interfaces.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -29,6 +29,7 @@ from typing import Generic from typing import Iterator from typing import List +from typing import Mapping from typing import NamedTuple from typing import NoReturn from typing import Optional @@ -44,6 +45,7 @@ from . import exc as orm_exc from . import path_registry from .base import _MappedAttribute as _MappedAttribute +from .base import DONT_SET as DONT_SET # noqa: F401 from .base import EXT_CONTINUE as EXT_CONTINUE # noqa: F401 from .base import EXT_SKIP as EXT_SKIP # noqa: F401 from .base import EXT_STOP as EXT_STOP # noqa: F401 @@ -88,7 +90,8 @@ from .context import _ORMCompileState from .context import QueryContext from .decl_api import RegistryType - from .decl_base import _ClassScanMapperConfig + from .decl_base import _ClassScanAbstractConfig + from .decl_base import _DeclarativeMapperConfig from .loading import _PopulatorDict from .mapper import Mapper from .path_registry import _AbstractEntityRegistry @@ -167,7 +170,7 @@ def found_in_pep593_annotated(self) -> Any: def declarative_scan( self, - decl_scan: _ClassScanMapperConfig, + decl_scan: _DeclarativeMapperConfig, registry: RegistryType, cls: Type[Any], originating_module: Optional[str], @@ -177,7 +180,7 @@ def declarative_scan( extracted_mapped_annotation: Optional[_AnnotationScanType], is_dataclass_field: bool, ) -> None: - """Perform class-specific initializaton at early declarative scanning + """Perform class-specific initialization at early declarative scanning time. .. versionadded:: 2.0 @@ -193,6 +196,22 @@ def _raise_for_required(self, key: str, cls: Type[Any]) -> NoReturn: ) +class _DataclassArguments(TypedDict): + """define arguments that can be passed to ORM Annotated Dataclass + class definitions. + + """ + + init: Union[_NoArg, bool] + repr: Union[_NoArg, bool] + eq: Union[_NoArg, bool] + order: Union[_NoArg, bool] + unsafe_hash: Union[_NoArg, bool] + match_args: Union[_NoArg, bool] + kw_only: Union[_NoArg, bool] + dataclass_callable: Union[_NoArg, Callable[..., Type[Any]]] + + class _AttributeOptions(NamedTuple): """define Python-local attribute behavior options common to all :class:`.MapperProperty` objects. @@ -210,8 +229,11 @@ class _AttributeOptions(NamedTuple): dataclasses_compare: Union[_NoArg, bool] dataclasses_kw_only: Union[_NoArg, bool] dataclasses_hash: Union[_NoArg, bool, None] + dataclasses_dataclass_metadata: Union[_NoArg, Mapping[Any, Any], None] - def _as_dataclass_field(self, key: str) -> Any: + def _as_dataclass_field( + self, key: str, dataclass_setup_arguments: _DataclassArguments + ) -> Any: """Return a ``dataclasses.Field`` object given these arguments.""" kw: Dict[str, Any] = {} @@ -229,6 +251,8 @@ def _as_dataclass_field(self, key: str) -> Any: kw["kw_only"] = self.dataclasses_kw_only if self.dataclasses_hash is not _NoArg.NO_ARG: kw["hash"] = self.dataclasses_hash + if self.dataclasses_dataclass_metadata is not _NoArg.NO_ARG: + kw["metadata"] = self.dataclasses_dataclass_metadata if "default" in kw and callable(kw["default"]): # callable defaults are ambiguous. deprecate them in favour of @@ -263,13 +287,16 @@ def _as_dataclass_field(self, key: str) -> Any: @classmethod def _get_arguments_for_make_dataclass( cls, + decl_scan: _ClassScanAbstractConfig, key: str, annotation: _AnnotationScanType, mapped_container: Optional[Any], - elem: _T, + elem: Any, + dataclass_setup_arguments: _DataclassArguments, + enable_descriptor_defaults: bool, ) -> Union[ Tuple[str, _AnnotationScanType], - Tuple[str, _AnnotationScanType, dataclasses.Field[Any]], + Tuple[str, _AnnotationScanType, dataclasses.Field[Any] | None], ]: """given attribute key, annotation, and value from a class, return the argument tuple we would pass to dataclasses.make_dataclass() @@ -277,7 +304,15 @@ def _get_arguments_for_make_dataclass( """ if isinstance(elem, _DCAttributeOptions): - dc_field = elem._attribute_options._as_dataclass_field(key) + attribute_options = elem._get_dataclass_setup_options( + decl_scan, + key, + dataclass_setup_arguments, + enable_descriptor_defaults, + ) + dc_field = attribute_options._as_dataclass_field( + key, dataclass_setup_arguments + ) return (key, annotation, dc_field) elif elem is not _NoArg.NO_ARG: @@ -285,14 +320,14 @@ def _get_arguments_for_make_dataclass( return (key, annotation, elem) elif mapped_container is not None: # it's Mapped[], but there's no "element", which means declarative - # did not actually do anything for this field. this shouldn't - # happen. - # previously, this would occur because _scan_attributes would - # skip a field that's on an already mapped superclass, but it - # would still include it in the annotations, leading - # to issue #8718 - - assert False, "Mapped[] received without a mapping declaration" + # did not actually do anything for this field. + # prior to 2.1, this would never happen and we had a false + # assertion here, because the mapper _scan_attributes always + # generates a MappedColumn when one is not present + # (see issue #8718). However, in 2.1 we handle this case for the + # non-mapped dataclass use case without the need to generate + # MappedColumn that gets thrown away anyway. + return (key, annotation) else: # plain dataclass field, not mapped. Is only possible @@ -309,6 +344,7 @@ def _get_arguments_for_make_dataclass( _NoArg.NO_ARG, _NoArg.NO_ARG, _NoArg.NO_ARG, + _NoArg.NO_ARG, ) _DEFAULT_READONLY_ATTRIBUTE_OPTIONS = _AttributeOptions( @@ -319,6 +355,7 @@ def _get_arguments_for_make_dataclass( _NoArg.NO_ARG, _NoArg.NO_ARG, _NoArg.NO_ARG, + _NoArg.NO_ARG, ) @@ -344,6 +381,63 @@ class _DCAttributeOptions: _has_dataclass_arguments: bool + def _get_dataclass_setup_options( + self, + decl_scan: _ClassScanAbstractConfig, + key: str, + dataclass_setup_arguments: _DataclassArguments, + enable_descriptor_defaults: bool, + ) -> _AttributeOptions: + return self._attribute_options + + +class _DataclassDefaultsDontSet(_DCAttributeOptions): + __slots__ = () + + _default_scalar_value: Any + + _disable_dataclass_default_factory: bool = False + + def _get_dataclass_setup_options( + self, + decl_scan: _ClassScanAbstractConfig, + key: str, + dataclass_setup_arguments: _DataclassArguments, + enable_descriptor_defaults: bool, + ) -> _AttributeOptions: + + disable_descriptor_defaults = ( + not enable_descriptor_defaults + or getattr(decl_scan.cls, "_sa_disable_descriptor_defaults", False) + ) + + if disable_descriptor_defaults: + return self._attribute_options + + dataclasses_default = self._attribute_options.dataclasses_default + dataclasses_default_factory = ( + self._attribute_options.dataclasses_default_factory + ) + + if dataclasses_default is not _NoArg.NO_ARG and not callable( + dataclasses_default + ): + self._default_scalar_value = ( + self._attribute_options.dataclasses_default + ) + return self._attribute_options._replace( + dataclasses_default=DONT_SET, + ) + elif ( + self._disable_dataclass_default_factory + and dataclasses_default_factory is not _NoArg.NO_ARG + ): + return self._attribute_options._replace( + dataclasses_default=DONT_SET, + dataclasses_default_factory=_NoArg.NO_ARG, + ) + return self._attribute_options + class _MapsColumns(_DCAttributeOptions, _MappedAttribute[_T]): """interface for declarative-capable construct that delivers one or more @@ -811,6 +905,11 @@ def _bulk_update_tuples( return [(cast("_DMLColumnArgument", self.__clause_element__()), value)] + def _bulk_dml_setter(self, key: str) -> Optional[Callable[..., Any]]: + """return a callable that will process a bulk INSERT value""" + + return None + def adapt_to_entity( self, adapt_to_entity: AliasedInsp[Any] ) -> PropComparator[_T_co]: @@ -1109,10 +1208,7 @@ def do_init(self) -> None: self.strategy = self._get_strategy(self.strategy_key) def post_instrument_class(self, mapper: Mapper[Any]) -> None: - if ( - not self.parent.non_primary - and not mapper.class_manager._attr_has_impl(self.key) - ): + if not mapper.class_manager._attr_has_impl(self.key): self.strategy.init_class_attribute(mapper) _all_strategies: collections.defaultdict[ diff --git a/lib/sqlalchemy/orm/loading.py b/lib/sqlalchemy/orm/loading.py index deee8bc3ada..ad7fabb0f83 100644 --- a/lib/sqlalchemy/orm/loading.py +++ b/lib/sqlalchemy/orm/loading.py @@ -1,5 +1,5 @@ # orm/loading.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -39,6 +39,7 @@ from .context import _ORMCompileState from .context import FromStatement from .context import QueryContext +from .strategies import _SelectInLoader from .util import _none_set from .util import state_str from .. import exc as sa_exc @@ -50,7 +51,6 @@ from ..sql import select from ..sql import util as sql_util from ..sql.selectable import ForUpdateArg -from ..sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL from ..sql.selectable import SelectState from ..util import EMPTY_DICT from ..util.typing import TupleAny @@ -292,7 +292,7 @@ def require_unique(obj): "against collections" ) - result._unique_filter_state = (None, require_unique) + result._unique_filter_state = (set(), require_unique) if context.yield_per: result.yield_per(context.yield_per) @@ -340,7 +340,7 @@ def merge_frozen_result(session, statement, frozen_result, load=True): ) result = [] - for newrow in frozen_result.rewrite_rows(): + for newrow in frozen_result._rewrite_rows(): for i in mapped_entities: if newrow[i] is not None: newrow[i] = session._merge( @@ -580,9 +580,7 @@ def _load_on_pk_identity( "release." ) - q._where_criteria = ( - sql_util._deep_annotate(_get_clause, {"_orm_adapt": True}), - ) + q._where_criteria = (_get_clause,) params = { _get_params[primary_key].key: id_val @@ -688,7 +686,8 @@ def _load_on_pk_identity( load_options += {"_autoflush": False} execution_options = util.EMPTY_DICT.merge_with( - execution_options, {"_sa_orm_load_options": load_options} + execution_options, + util.immutabledict(_sa_orm_load_options=load_options), ) result = ( session.execute( @@ -1309,15 +1308,18 @@ def do_load(context, path, states, load_only, effective_entity): if context.populate_existing: q2 = q2.execution_options(populate_existing=True) - context.session.execute( - q2, - dict( - primary_keys=[ - state.key[1][0] if zero_idx else state.key[1] - for state, load_attrs in states - ] - ), - ).unique().scalars().all() + while states: + chunk = states[0 : _SelectInLoader._chunksize] + states = states[_SelectInLoader._chunksize :] + context.session.execute( + q2, + dict( + primary_keys=[ + state.key[1][0] if zero_idx else state.key[1] + for state, load_attrs in chunk + ] + ), + ).unique().scalars().all() return do_load @@ -1669,7 +1671,7 @@ def _load_scalar_attributes(mapper, state, attribute_names, passive): result = _load_on_ident( session, - select(mapper).set_label_style(LABEL_STYLE_TABLENAME_PLUS_COL), + select(mapper), identity_key, refresh_state=state, only_load_props=attribute_names, diff --git a/lib/sqlalchemy/orm/mapped_collection.py b/lib/sqlalchemy/orm/mapped_collection.py index ca085c40376..46c19dc32ac 100644 --- a/lib/sqlalchemy/orm/mapped_collection.py +++ b/lib/sqlalchemy/orm/mapped_collection.py @@ -1,5 +1,5 @@ # orm/mapped_collection.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -13,6 +13,7 @@ from typing import Dict from typing import Generic from typing import List +from typing import Literal from typing import Optional from typing import Sequence from typing import Tuple @@ -31,7 +32,6 @@ from ..sql import roles from ..util.langhelpers import Missing from ..util.langhelpers import MissingOr -from ..util.typing import Literal if TYPE_CHECKING: from . import AttributeEventToken @@ -443,8 +443,8 @@ def _raise_for_unpopulated( f"parameter on the mapped collection factory." ) - @collection.appender # type: ignore[misc] - @collection.internally_instrumented # type: ignore[misc] + @collection.appender # type: ignore[untyped-decorator] + @collection.internally_instrumented # type: ignore[untyped-decorator] def set( self, value: _KT, @@ -472,8 +472,8 @@ def set( self.__setitem__(key, value, _sa_initiator) # type: ignore[call-arg] - @collection.remover # type: ignore[misc] - @collection.internally_instrumented # type: ignore[misc] + @collection.remover # type: ignore[untyped-decorator] + @collection.internally_instrumented # type: ignore[untyped-decorator] def remove( self, value: _KT, diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 6fb46a2bd81..22528a42402 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1,5 +1,5 @@ # orm/mapper.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -33,6 +33,7 @@ from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import Mapping from typing import Optional from typing import Sequence @@ -88,7 +89,6 @@ from ..sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL from ..util import HasMemoized from ..util import HasMemoized_ro_memoized_attribute -from ..util.typing import Literal from ..util.typing import TupleAny from ..util.typing import Unpack @@ -112,6 +112,7 @@ from ..engine import RowMapping from ..sql._typing import _ColumnExpressionArgument from ..sql._typing import _EquivalentColumnMap + from ..sql.base import _EntityNamespace from ..sql.base import ReadOnlyColumnCollection from ..sql.elements import ColumnClause from ..sql.elements import ColumnElement @@ -190,23 +191,12 @@ class Mapper( _configure_failed: Any = False _ready_for_configure = False - @util.deprecated_params( - non_primary=( - "1.3", - "The :paramref:`.mapper.non_primary` parameter is deprecated, " - "and will be removed in a future release. The functionality " - "of non primary mappers is now better suited using the " - ":class:`.AliasedClass` construct, which can also be used " - "as the target of a :func:`_orm.relationship` in 1.3.", - ), - ) def __init__( self, class_: Type[_O], local_table: Optional[FromClause] = None, properties: Optional[Mapping[str, MapperProperty[Any]]] = None, primary_key: Optional[Iterable[_ORMColumnExprArgument[Any]]] = None, - non_primary: bool = False, inherits: Optional[Union[Mapper[Any], Type[Any]]] = None, inherit_condition: Optional[_ColumnExpressionArgument[bool]] = None, inherit_foreign_keys: Optional[ @@ -448,18 +438,6 @@ class User(Base): See the change note and example at :ref:`legacy_is_orphan_addition` for more detail on this change. - :param non_primary: Specify that this :class:`_orm.Mapper` - is in addition - to the "primary" mapper, that is, the one used for persistence. - The :class:`_orm.Mapper` created here may be used for ad-hoc - mapping of the class to an alternate selectable, for loading - only. - - .. seealso:: - - :ref:`relationship_aliased_class` - the new pattern that removes - the need for the :paramref:`_orm.Mapper.non_primary` flag. - :param passive_deletes: Indicates DELETE behavior of foreign key columns when a joined-table inheritance entity is being deleted. Defaults to ``False`` for a base mapper; for an inheriting mapper, @@ -528,8 +506,6 @@ class User(Base): the columns specific to this subclass. The SELECT uses IN to fetch multiple subclasses at once. - .. versionadded:: 1.2 - .. seealso:: :ref:`with_polymorphic_mapper_config` @@ -734,7 +710,6 @@ def generate_version(version): ) self._primary_key_argument = util.to_list(primary_key) - self.non_primary = non_primary self.always_refresh = always_refresh @@ -1058,7 +1033,7 @@ def entity(self): """ - primary_key: Tuple[Column[Any], ...] + primary_key: Tuple[ColumnElement[Any], ...] """An iterable containing the collection of :class:`_schema.Column` objects which comprise the 'primary key' of the mapped table, from the @@ -1102,16 +1077,6 @@ def entity(self): """ - non_primary: bool - """Represent ``True`` if this :class:`_orm.Mapper` is a "non-primary" - mapper, e.g. a mapper that is used only to select rows but not for - persistence management. - - This is a *read only* attribute determined during mapper construction. - Behavior is undefined if directly modified. - - """ - polymorphic_on: Optional[KeyedColumnElement[Any]] """The :class:`_schema.Column` or SQL expression specified as the ``polymorphic_on`` argument @@ -1168,30 +1133,7 @@ def entity(self): """ - columns: ReadOnlyColumnCollection[str, Column[Any]] - """A collection of :class:`_schema.Column` or other scalar expression - objects maintained by this :class:`_orm.Mapper`. - - The collection behaves the same as that of the ``c`` attribute on - any :class:`_schema.Table` object, - except that only those columns included in - this mapping are present, and are keyed based on the attribute name - defined in the mapping, not necessarily the ``key`` attribute of the - :class:`_schema.Column` itself. Additionally, scalar expressions mapped - by :func:`.column_property` are also present here. - - This is a *read only* attribute determined during mapper construction. - Behavior is undefined if directly modified. - - """ - - c: ReadOnlyColumnCollection[str, Column[Any]] - """A synonym for :attr:`_orm.Mapper.columns`.""" - - @util.non_memoized_property - @util.deprecated("1.3", "Use .persist_selectable") - def mapped_table(self): - return self.persist_selectable + _columns: sql_base.WriteableColumnCollection[str, Column[Any]] @util.memoized_property def _path_registry(self) -> _CachingEntityRegistry: @@ -1213,14 +1155,6 @@ def _configure_inheritance(self): self.dispatch._update(self.inherits.dispatch) - if self.non_primary != self.inherits.non_primary: - np = not self.non_primary and "primary" or "non-primary" - raise sa_exc.ArgumentError( - "Inheritance of %s mapper for class '%s' is " - "only allowed from a %s mapper" - % (np, self.class_.__name__, np) - ) - if self.single: self.persist_selectable = self.inherits.persist_selectable elif self.local_table is not self.inherits.local_table: @@ -1468,8 +1402,7 @@ def _set_polymorphic_on(self, polymorphic_on): self._configure_polymorphic_setter(True) def _configure_class_instrumentation(self): - """If this mapper is to be a primary mapper (i.e. the - non_primary flag is not set), associate this Mapper with the + """Associate this Mapper with the given class and entity name. Subsequent calls to ``class_mapper()`` for the ``class_`` / ``entity`` @@ -1484,21 +1417,6 @@ def _configure_class_instrumentation(self): # this raises as of 2.0. manager = attributes.opt_manager_of_class(self.class_) - if self.non_primary: - if not manager or not manager.is_mapped: - raise sa_exc.InvalidRequestError( - "Class %s has no primary mapper configured. Configure " - "a primary mapper first before setting up a non primary " - "Mapper." % self.class_ - ) - self.class_manager = manager - - assert manager.registry is not None - self.registry = manager.registry - self._identity_class = manager.mapper._identity_class - manager.registry._add_non_primary_mapper(self) - return - if manager is None or not manager.registry: raise sa_exc.InvalidRequestError( "The _mapper() function and Mapper() constructor may not be " @@ -1718,7 +1636,7 @@ def _configure_pks(self) -> None: } def _configure_properties(self) -> None: - self.columns = self.c = sql_base.ColumnCollection() # type: ignore + self._columns = sql_base.WriteableColumnCollection() # object attribute names mapped to MapperProperty objects self._props = util.OrderedDict() @@ -2101,13 +2019,21 @@ def _configure_property( "_configure_property(%s, %s)", key, prop_arg.__class__.__name__ ) + # early setup mode - don't assign any props, only + # ensure a Column is turned into a ColumnProperty. + # see #12858 + early_setup = not hasattr(self, "_props") + if not isinstance(prop_arg, MapperProperty): prop: MapperProperty[Any] = self._property_from_column( - key, prop_arg + key, prop_arg, early_setup ) else: prop = prop_arg + if early_setup: + return prop + if isinstance(prop, properties.ColumnProperty): col = self.persist_selectable.corresponding_column(prop.columns[0]) @@ -2164,7 +2090,7 @@ def _configure_property( # to be addressable in subqueries col.key = col._tq_key_label = key - self.columns.add(col, key) + self._columns.add(col, key) for col in prop.columns: for proxy_col in col.proxy_set: @@ -2242,8 +2168,7 @@ def _configure_property( self._props[key] = prop - if not self.non_primary: - prop.instrument_class(self) + prop.instrument_class(self) for mapper in self._inheriting_mappers: mapper._adapt_inherited_property(key, prop, init) @@ -2343,7 +2268,6 @@ def _reconcile_prop_with_incoming_columns( # existing properties.ColumnProperty from an inheriting # mapper. make a copy and append our column to it - # breakpoint() new_prop = existing_prop.copy() new_prop.columns.insert(0, incoming_column) @@ -2356,16 +2280,17 @@ def _reconcile_prop_with_incoming_columns( @util.preload_module("sqlalchemy.orm.descriptor_props") def _property_from_column( - self, - key: str, - column: KeyedColumnElement[Any], + self, key: str, column: KeyedColumnElement[Any], early_setup: bool ) -> ColumnProperty[Any]: """generate/update a :class:`.ColumnProperty` given a :class:`_schema.Column` or other SQL expression object.""" descriptor_props = util.preloaded.orm_descriptor_props - prop = self._props.get(key) + if early_setup: + prop = None + else: + prop = self._props.get(key) if isinstance(prop, properties.ColumnProperty): return self._reconcile_prop_with_incoming_columns( @@ -2464,7 +2389,6 @@ def _log_desc(self) -> str: and self.local_table.description or str(self.local_table) ) - + (self.non_primary and "|non-primary" or "") + ")" ) @@ -2478,9 +2402,8 @@ def __repr__(self) -> str: return "" % (id(self), self.class_.__name__) def __str__(self) -> str: - return "Mapper[%s%s(%s)]" % ( + return "Mapper[%s(%s)]" % ( self.class_.__name__, - self.non_primary and " (non-primary)" or "", ( self.local_table.description if self.local_table is not None @@ -2536,6 +2459,29 @@ def get_property_by_column( return self._columntoproperty[column] + @HasMemoized.memoized_attribute + def columns(self) -> ReadOnlyColumnCollection[str, Column[Any]]: + """A collection of :class:`_schema.Column` or other scalar expression + objects maintained by this :class:`_orm.Mapper`. + + The collection behaves the same as that of the ``c`` attribute on any + :class:`_schema.Table` object, except that only those columns included + in this mapping are present, and are keyed based on the attribute name + defined in the mapping, not necessarily the ``key`` attribute of the + :class:`_schema.Column` itself. Additionally, scalar expressions + mapped by :func:`.column_property` are also present here. + + This is a *read only* attribute determined during mapper construction. + Behavior is undefined if directly modified. + + """ + return self._columns.as_readonly() + + @HasMemoized.memoized_attribute + def c(self) -> ReadOnlyColumnCollection[str, Column[Any]]: + """A synonym for :attr:`_orm.Mapper.columns`.""" + return self._columns.as_readonly() + @property def iterate_properties(self): """return an iterator of all MapperProperty objects.""" @@ -2555,7 +2501,7 @@ def _mappers_from_spec( if spec == "*": mappers = list(self.self_and_descendants) elif spec: - mapper_set = set() + mapper_set: Set[Mapper[Any]] = set() for m in util.to_list(spec): m = _class_to_mapper(m) if not m.isa(self): @@ -3101,9 +3047,6 @@ class in which it first appeared. The above process produces an ordering that is deterministic in terms of the order in which attributes were assigned to the class. - .. versionchanged:: 1.3.19 ensured deterministic ordering for - :meth:`_orm.Mapper.all_orm_descriptors`. - When dealing with a :class:`.QueryableAttribute`, the :attr:`.QueryableAttribute.property` attribute refers to the :class:`.MapperProperty` property, which is what you get when @@ -3167,9 +3110,9 @@ def synonyms(self) -> util.ReadOnlyProperties[SynonymProperty[Any]]: return self._filter_properties(descriptor_props.SynonymProperty) - @property - def entity_namespace(self): - return self.class_ + @util.ro_non_memoized_property + def entity_namespace(self) -> _EntityNamespace: + return self.class_ # type: ignore[return-value] @HasMemoized.memoized_attribute def column_attrs(self) -> util.ReadOnlyProperties[ColumnProperty[Any]]: @@ -3442,9 +3385,11 @@ def primary_base_mapper(self) -> Mapper[Any]: return self.class_manager.mapper.base_mapper def _result_has_identity_key(self, result, adapter=None): - pk_cols: Sequence[ColumnClause[Any]] = self.primary_key - if adapter: - pk_cols = [adapter.columns[c] for c in pk_cols] + pk_cols: Sequence[ColumnElement[Any]] + if adapter is not None: + pk_cols = [adapter.columns[c] for c in self.primary_key] + else: + pk_cols = self.primary_key rk = result.keys() for col in pk_cols: if col not in rk: @@ -3469,9 +3414,11 @@ def identity_key_from_row( for the "row" argument """ - pk_cols: Sequence[ColumnClause[Any]] = self.primary_key - if adapter: - pk_cols = [adapter.columns[c] for c in pk_cols] + pk_cols: Sequence[ColumnElement[Any]] + if adapter is not None: + pk_cols = [adapter.columns[c] for c in self.primary_key] + else: + pk_cols = self.primary_key mapping: RowMapping if hasattr(row, "_mapping"): @@ -3887,10 +3834,7 @@ def _subclass_load_via_in(self, entity, polymorphic_from): _reconcile_to_other=False, ) - primary_key = [ - sql_util._deep_annotate(pk, {"_orm_adapt": True}) - for pk in self.primary_key - ] + primary_key = list(self.primary_key) in_expr: ColumnElement[Any] @@ -3907,17 +3851,16 @@ def _subclass_load_via_in(self, entity, polymorphic_from): ) in_expr = entity._adapter.traverse(in_expr) - primary_key = [entity._adapter.traverse(k) for k in primary_key] q = q.where( in_expr.in_(sql.bindparam("primary_keys", expanding=True)) - ).order_by(*primary_key) + ) else: q = sql.select(self).set_label_style( LABEL_STYLE_TABLENAME_PLUS_COL ) q = q.where( in_expr.in_(sql.bindparam("primary_keys", expanding=True)) - ).order_by(*primary_key) + ) return q, enable_opt, disable_opt @@ -4177,6 +4120,12 @@ class is instantiated into an instance, as well as when ORM queries work; this can be used to establish additional options, properties, or related mappings before the operation proceeds. + * :meth:`.RegistryEvents.before_configured` - Like + :meth:`.MapperEvents.before_configured`, but local to a specific + :class:`_orm.registry`. + + .. versionadded:: 2.1 - added :meth:`.RegistryEvents.before_configured` + * :meth:`.MapperEvents.mapper_configured` - called as each individual :class:`_orm.Mapper` is configured within the process; will include all mapper state except for backrefs set up by other mappers that are still @@ -4192,6 +4141,12 @@ class is instantiated into an instance, as well as when ORM queries if they are in other :class:`_orm.registry` collections not part of the current scope of configuration. + * :meth:`.RegistryEvents.after_configured` - Like + :meth:`.MapperEvents.after_configured`, but local to a specific + :class:`_orm.registry`. + + .. versionadded:: 2.1 - added :meth:`.RegistryEvents.after_configured` + """ _configure_registries(_all_registries(), cascade=True) @@ -4220,26 +4175,35 @@ def _configure_registries( return Mapper.dispatch._for_class(Mapper).before_configured() # type: ignore # noqa: E501 + # initialize properties on all mappers # note that _mapper_registry is unordered, which # may randomly conceal/reveal issues related to # the order of mapper compilation - _do_configure_registries(registries, cascade) + registries_configured = list( + _do_configure_registries(registries, cascade) + ) + finally: _already_compiling = False + for reg in registries_configured: + reg.dispatch.after_configured(reg) Mapper.dispatch._for_class(Mapper).after_configured() # type: ignore @util.preload_module("sqlalchemy.orm.decl_api") def _do_configure_registries( registries: Set[_RegistryType], cascade: bool -) -> None: +) -> Iterator[registry]: registry = util.preloaded.orm_decl_api.registry orig = set(registries) for reg in registry._recurse_with_dependencies(registries): + if reg._new_mappers: + reg.dispatch.before_configured(reg) + has_skip = False for mapper in reg._mappers_to_configure(): @@ -4274,6 +4238,9 @@ def _do_configure_registries( if not hasattr(exc, "_configure_failed"): mapper._configure_failed = exc raise + + if reg._new_mappers: + yield reg if not has_skip: reg._new_mappers = False @@ -4306,7 +4273,6 @@ def _dispose_registries(registries: Set[_RegistryType], cascade: bool) -> None: else: reg._dispose_manager_and_mapper(manager) - reg._non_primary_mappers.clear() reg._dependents.clear() for dep in reg._dependencies: dep._dependents.discard(reg) @@ -4318,7 +4284,7 @@ def _dispose_registries(registries: Set[_RegistryType], cascade: bool) -> None: reg._new_mappers = False -def reconstructor(fn): +def reconstructor(fn: _Fn) -> _Fn: """Decorate a method as the 'reconstructor' hook. Designates a single method as the "reconstructor", an ``__init__``-like @@ -4344,7 +4310,7 @@ def reconstructor(fn): :meth:`.InstanceEvents.load` """ - fn.__sa_reconstructor__ = True + fn.__sa_reconstructor__ = True # type: ignore[attr-defined] return fn @@ -4378,7 +4344,7 @@ def validates( :func:`.validates` usage where only one validator should emit per attribute operation. - .. versionchanged:: 2.0.16 This paramter inadvertently defaulted to + .. versionchanged:: 2.0.16 This parameter inadvertently defaulted to ``False`` for releases 2.0.0 through 2.0.15. Its correct default of ``True`` is restored in 2.0.16. diff --git a/lib/sqlalchemy/orm/path_registry.py b/lib/sqlalchemy/orm/path_registry.py index aa1363ad826..855b58b3e49 100644 --- a/lib/sqlalchemy/orm/path_registry.py +++ b/lib/sqlalchemy/orm/path_registry.py @@ -1,12 +1,10 @@ # orm/path_registry.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -"""Path tracking utilities, representing mapper graph traversals. - -""" +"""Path tracking utilities, representing mapper graph traversals.""" from __future__ import annotations @@ -34,6 +32,8 @@ from ..sql.cache_key import HasCacheKey if TYPE_CHECKING: + from typing import TypeGuard + from ._typing import _InternalEntityType from .interfaces import StrategizedProperty from .mapper import Mapper @@ -43,7 +43,6 @@ from ..sql.elements import BindParameter from ..sql.visitors import anon_map from ..util.typing import _LiteralStar - from ..util.typing import TypeGuard def is_root(path: PathRegistry) -> TypeGuard[RootRegistry]: ... @@ -705,13 +704,28 @@ def __init__( # This is basically the only place that the "is_unnatural" flag # actually changes behavior. if parent.path and (self.is_aliased_class or parent.is_unnatural): - # this is an infrequent code path used only for loader strategies - # that also make use of of_type(). - if entity.mapper.isa(parent.natural_path[-1].mapper): # type: ignore # noqa: E501 + # this is an infrequent code path used for loader strategies that + # also make use of of_type() or other intricate polymorphic + # base/subclass combinations + parent_natural_entity = parent.natural_path[-1] + + if entity.mapper.isa( + parent_natural_entity.mapper # type: ignore + ) or parent_natural_entity.mapper.isa( # type: ignore + entity.mapper + ): + # when the entity mapper and parent mapper are in an + # inheritance relationship, use entity.mapper in natural_path. + # First case: entity.mapper inherits from parent mapper (e.g., + # accessing a subclass mapper through parent path). Second case + # (issue #13193): parent mapper inherits from entity.mapper + # (e.g., parent path has Sub(Base) but we're accessing with + # Base where Base.related is declared, so use Base in + # natural_path). self.natural_path = parent.natural_path + (entity.mapper,) else: self.natural_path = parent.natural_path + ( - parent.natural_path[-1].entity, # type: ignore + parent_natural_entity.entity, # type: ignore ) # it seems to make sense that since these paths get mixed up # with statements that are cached or not, we should make diff --git a/lib/sqlalchemy/orm/persistence.py b/lib/sqlalchemy/orm/persistence.py index d2f2b2b8f0a..674ac91dbea 100644 --- a/lib/sqlalchemy/orm/persistence.py +++ b/lib/sqlalchemy/orm/persistence.py @@ -1,5 +1,5 @@ # orm/persistence.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -28,13 +28,11 @@ from . import sync from .base import state_str from .. import exc as sa_exc -from .. import future from .. import sql from .. import util from ..engine import cursor as _cursor from ..sql import operators from ..sql.elements import BooleanClauseList -from ..sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL def _save_obj(base_mapper, states, uowtransaction, single=False): @@ -456,8 +454,13 @@ def _collect_update_commands( pks = mapper._pks_by_table[table] - if use_orm_update_stmt is not None: + if ( + use_orm_update_stmt is not None + and not use_orm_update_stmt._maintain_values_ordering + ): # TODO: ordered values, etc + # ORM bulk_persistence will raise for the maintain_values_ordering + # case right now value_params = use_orm_update_stmt._values else: value_params = {} @@ -1374,7 +1377,13 @@ def update_stmt(): ) rows += c.rowcount - for state, state_dict, mapper_rec, connection, params in records: + for i, ( + state, + state_dict, + mapper_rec, + connection, + params, + ) in enumerate(records): _postfetch_post_update( mapper_rec, uowtransaction, @@ -1382,7 +1391,7 @@ def update_stmt(): state, state_dict, c, - c.context.compiled_parameters[0], + c.context.compiled_parameters[i], ) if check_rowcount: @@ -1548,9 +1557,7 @@ def _finalize_insert_update_commands(base_mapper, uowtransaction, states): if toload_now: state.key = base_mapper._identity_key_from_state(state) - stmt = future.select(mapper).set_label_style( - LABEL_STYLE_TABLENAME_PLUS_COL - ) + stmt = sql.select(mapper) loading._load_on_ident( uowtransaction.session, stmt, diff --git a/lib/sqlalchemy/orm/properties.py b/lib/sqlalchemy/orm/properties.py index 2ffa53fb8ef..bb3d7694243 100644 --- a/lib/sqlalchemy/orm/properties.py +++ b/lib/sqlalchemy/orm/properties.py @@ -1,5 +1,5 @@ # orm/properties.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -17,6 +17,7 @@ from typing import Any from typing import cast from typing import Dict +from typing import get_args from typing import List from typing import Optional from typing import Sequence @@ -28,6 +29,7 @@ from typing import Union from . import attributes +from . import exc as orm_exc from . import strategy_options from .base import _DeclarativeMapped from .base import class_mapper @@ -35,6 +37,7 @@ from .descriptor_props import ConcreteInheritedProperty from .descriptor_props import SynonymProperty from .interfaces import _AttributeOptions +from .interfaces import _DataclassDefaultsDontSet from .interfaces import _DEFAULT_ATTRIBUTE_OPTIONS from .interfaces import _IntrospectsAnnotations from .interfaces import _MapsColumns @@ -54,20 +57,22 @@ from ..sql.schema import SchemaConst from ..sql.type_api import TypeEngine from ..util.typing import de_optionalize_union_types -from ..util.typing import get_args from ..util.typing import includes_none +from ..util.typing import is_a_type from ..util.typing import is_fwd_ref from ..util.typing import is_pep593 from ..util.typing import is_pep695 from ..util.typing import Self if TYPE_CHECKING: + from typing import ForwardRef + from ._typing import _IdentityKeyType from ._typing import _InstanceDict from ._typing import _ORMColumnExprArgument from ._typing import _RegistryType from .base import Mapped - from .decl_base import _ClassScanMapperConfig + from .decl_base import _DeclarativeMapperConfig from .mapper import Mapper from .session import Session from .state import _InstallLoaderCallableProto @@ -77,6 +82,7 @@ from ..sql.elements import NamedColumn from ..sql.operators import OperatorType from ..util.typing import _AnnotationScanType + from ..util.typing import _MatchedOnType from ..util.typing import RODescriptorReference _T = TypeVar("_T", bound=Any) @@ -94,6 +100,7 @@ @log.class_logger class ColumnProperty( + _DataclassDefaultsDontSet, _MapsColumns[_T], StrategizedProperty[_T], _IntrospectsAnnotations, @@ -128,6 +135,7 @@ class ColumnProperty( "comparator_factory", "active_history", "expire_on_flush", + "_default_scalar_value", "_creation_order", "_is_polymorphic_discriminator", "_mapped_by_synonym", @@ -147,6 +155,7 @@ def __init__( raiseload: bool = False, comparator_factory: Optional[Type[PropComparator[_T]]] = None, active_history: bool = False, + default_scalar_value: Any = None, expire_on_flush: bool = True, info: Optional[_InfoType] = None, doc: Optional[str] = None, @@ -171,6 +180,7 @@ def __init__( else self.__class__.Comparator ) self.active_history = active_history + self._default_scalar_value = default_scalar_value self.expire_on_flush = expire_on_flush if info is not None: @@ -198,7 +208,7 @@ def __init__( def declarative_scan( self, - decl_scan: _ClassScanMapperConfig, + decl_scan: _DeclarativeMapperConfig, registry: _RegistryType, cls: Type[Any], originating_module: Optional[str], @@ -232,7 +242,7 @@ def _memoized_attr__renders_in_subqueries(self) -> bool: return self.strategy._have_default_expression # type: ignore return ("deferred", True) not in self.strategy_key or ( - self not in self.parent._readonly_props # type: ignore + self not in self.parent._readonly_props ) @util.preload_module("sqlalchemy.orm.state", "sqlalchemy.orm.strategies") @@ -322,6 +332,7 @@ def copy(self) -> ColumnProperty[_T]: deferred=self.deferred, group=self.group, active_history=self.active_history, + default_scalar_value=self._default_scalar_value, ) def merge( @@ -379,8 +390,6 @@ class Comparator(util.MemoizedSlots, PropComparator[_PT]): """The full sequence of columns referenced by this attribute, adjusted for any aliasing in progress. - .. versionadded:: 1.3.17 - .. seealso:: :ref:`maptojoin` - usage example @@ -451,8 +460,6 @@ def _memoized_attr_expressions(self) -> Sequence[NamedColumn[Any]]: """The full sequence of columns referenced by this attribute, adjusted for any aliasing in progress. - .. versionadded:: 1.3.17 - """ if self.adapter: return [ @@ -507,6 +514,7 @@ class MappedSQLExpression(ColumnProperty[_T], _DeclarativeMapped[_T]): class MappedColumn( + _DataclassDefaultsDontSet, _IntrospectsAnnotations, _MapsColumns[_T], _DeclarativeMapped[_T], @@ -536,6 +544,7 @@ class MappedColumn( "deferred_group", "deferred_raiseload", "active_history", + "_default_scalar_value", "_attribute_options", "_has_dataclass_arguments", "_use_existing_column", @@ -566,12 +575,11 @@ def __init__(self, *arg: Any, **kw: Any): ) ) - insert_default = kw.pop("insert_default", _NoArg.NO_ARG) + insert_default = kw.get("insert_default", _NoArg.NO_ARG) self._has_insert_default = insert_default is not _NoArg.NO_ARG + self._default_scalar_value = _NoArg.NO_ARG - if self._has_insert_default: - kw["default"] = insert_default - elif attr_opts.dataclasses_default is not _NoArg.NO_ARG: + if attr_opts.dataclasses_default is not _NoArg.NO_ARG: kw["default"] = attr_opts.dataclasses_default self.deferred_group = kw.pop("deferred_group", None) @@ -580,7 +588,13 @@ def __init__(self, *arg: Any, **kw: Any): self.active_history = kw.pop("active_history", False) self._sort_order = kw.pop("sort_order", _NoArg.NO_ARG) + + # note that this populates "default" into the Column, so that if + # we are a dataclass and "default" is a dataclass default, it is still + # used as a Core-level default for the Column in addition to its + # dataclass role self.column = cast("Column[_T]", Column(*arg, **kw)) + self.foreign_keys = self.column.foreign_keys self._has_nullable = "nullable" in kw and kw.get("nullable") not in ( None, @@ -602,6 +616,7 @@ def _copy(self, **kw: Any) -> Self: new._has_dataclass_arguments = self._has_dataclass_arguments new._use_existing_column = self._use_existing_column new._sort_order = self._sort_order + new._default_scalar_value = self._default_scalar_value util.set_creation_order(new) return new @@ -617,7 +632,11 @@ def mapper_property_to_assign(self) -> Optional[MapperProperty[_T]]: self.deferred_group or self.deferred_raiseload ) - if effective_deferred or self.active_history: + if ( + effective_deferred + or self.active_history + or self._default_scalar_value is not _NoArg.NO_ARG + ): return ColumnProperty( self.column, deferred=effective_deferred, @@ -625,6 +644,11 @@ def mapper_property_to_assign(self) -> Optional[MapperProperty[_T]]: raiseload=self.deferred_raiseload, attribute_options=self._attribute_options, active_history=self.active_history, + default_scalar_value=( + self._default_scalar_value + if self._default_scalar_value is not _NoArg.NO_ARG + else None + ), ) else: return None @@ -661,20 +685,12 @@ def found_in_pep593_annotated(self) -> Any: # Column will be merged into it in _init_column_for_annotation(). return MappedColumn() - def declarative_scan( + def _adjust_for_existing_column( self, - decl_scan: _ClassScanMapperConfig, - registry: _RegistryType, - cls: Type[Any], - originating_module: Optional[str], + decl_scan: _DeclarativeMapperConfig, key: str, - mapped_container: Optional[Type[Mapped[Any]]], - annotation: Optional[_AnnotationScanType], - extracted_mapped_annotation: Optional[_AnnotationScanType], - is_dataclass_field: bool, - ) -> None: - column = self.column - + given_column: Column[_T], + ) -> Column[_T]: if ( self._use_existing_column and decl_scan.inherits @@ -686,10 +702,31 @@ def declarative_scan( ) supercls_mapper = class_mapper(decl_scan.inherits, False) - colname = column.name if column.name is not None else key - column = self.column = supercls_mapper.local_table.c.get( # type: ignore[assignment] # noqa: E501 - colname, column + colname = ( + given_column.name if given_column.name is not None else key ) + given_column = supercls_mapper.local_table.c.get( # type: ignore[assignment] # noqa: E501 + colname, given_column + ) + return given_column + + def declarative_scan( + self, + decl_scan: _DeclarativeMapperConfig, + registry: _RegistryType, + cls: Type[Any], + originating_module: Optional[str], + key: str, + mapped_container: Optional[Type[Mapped[Any]]], + annotation: Optional[_AnnotationScanType], + extracted_mapped_annotation: Optional[_AnnotationScanType], + is_dataclass_field: bool, + ) -> None: + column = self.column + + column = self.column = self._adjust_for_existing_column( + decl_scan, key, self.column + ) if column.key is None: column.key = key @@ -706,6 +743,8 @@ def declarative_scan( self._init_column_for_annotation( cls, + decl_scan, + key, registry, extracted_mapped_annotation, originating_module, @@ -714,6 +753,7 @@ def declarative_scan( @util.preload_module("sqlalchemy.orm.decl_base") def declarative_scan_for_composite( self, + decl_scan: _DeclarativeMapperConfig, registry: _RegistryType, cls: Type[Any], originating_module: Optional[str], @@ -724,48 +764,63 @@ def declarative_scan_for_composite( decl_base = util.preloaded.orm_decl_base decl_base._undefer_column_name(param_name, self.column) self._init_column_for_annotation( - cls, registry, param_annotation, originating_module + cls, decl_scan, key, registry, param_annotation, originating_module ) def _init_column_for_annotation( self, cls: Type[Any], + decl_scan: _DeclarativeMapperConfig, + key: str, registry: _RegistryType, argument: _AnnotationScanType, originating_module: Optional[str], ) -> None: sqltype = self.column.type + de_stringified_argument: _MatchedOnType + if is_fwd_ref( argument, check_generic=True, check_for_plain_string=True ): assert originating_module is not None - argument = de_stringify_annotation( + de_stringified_argument = de_stringify_annotation( cls, argument, originating_module, include_generic=True ) + else: + if TYPE_CHECKING: + assert not isinstance(argument, (str, ForwardRef)) + de_stringified_argument = argument - nullable = includes_none(argument) + nullable = includes_none(de_stringified_argument) if not self._has_nullable: self.column.nullable = nullable - our_type = de_optionalize_union_types(argument) - find_mapped_in: Tuple[Any, ...] = () - our_type_is_pep593 = False - raw_pep_593_type = None + raw_pep_593_type = resolved_pep_593_type = None + raw_pep_695_type = resolved_pep_695_type = None - if is_pep593(our_type): - our_type_is_pep593 = True + our_type: Any = de_optionalize_union_types(de_stringified_argument) + if is_pep695(our_type): + raw_pep_695_type = our_type + our_type = de_optionalize_union_types(raw_pep_695_type.__value__) + our_args = get_args(raw_pep_695_type) + if our_args: + our_type = our_type[our_args] + + resolved_pep_695_type = our_type + + if is_pep593(our_type): pep_593_components = get_args(our_type) - raw_pep_593_type = pep_593_components[0] + raw_pep_593_type = our_type + resolved_pep_593_type = pep_593_components[0] if nullable: - raw_pep_593_type = de_optionalize_union_types(raw_pep_593_type) + resolved_pep_593_type = de_optionalize_union_types( + resolved_pep_593_type + ) find_mapped_in = pep_593_components[1:] - elif is_pep695(argument) and is_pep593(argument.__value__): - # do not support nested annotation inside unions ets - find_mapped_in = get_args(argument.__value__)[1:] use_args_from: Optional[MappedColumn[Any]] for elem in find_mapped_in: @@ -776,13 +831,23 @@ def _init_column_for_annotation( use_args_from = None if use_args_from is not None: + + self.column = use_args_from._adjust_for_existing_column( + decl_scan, key, self.column + ) + if ( - not self._has_insert_default - and use_args_from.column.default is not None + self._has_insert_default + or self._attribute_options.dataclasses_default + is not _NoArg.NO_ARG ): - self.column.default = None + omit_defaults = True + else: + omit_defaults = False - use_args_from.column._merge(self.column) + use_args_from.column._merge( + self.column, omit_defaults=omit_defaults + ) sqltype = self.column.type if ( @@ -845,33 +910,64 @@ def _init_column_for_annotation( ) if sqltype._isnull and not self.column.foreign_keys: - new_sqltype = None - checks: List[Any] - if our_type_is_pep593: - checks = [our_type, raw_pep_593_type] - else: - checks = [our_type] + new_sqltype = registry._resolve_type_with_events( + cls, + key, + de_stringified_argument, + our_type, + raw_pep_593_type=raw_pep_593_type, + pep_593_resolved_argument=resolved_pep_593_type, + raw_pep_695_type=raw_pep_695_type, + pep_695_resolved_value=resolved_pep_695_type, + ) - for check_type in checks: - new_sqltype = registry._resolve_type(check_type) - if new_sqltype is not None: - break - else: + if new_sqltype is None: + checks = [] + if raw_pep_695_type: + checks.append(raw_pep_695_type) + checks.append(our_type) + if resolved_pep_593_type: + checks.append(resolved_pep_593_type) if isinstance(our_type, TypeEngine) or ( isinstance(our_type, type) and issubclass(our_type, TypeEngine) ): - raise sa_exc.ArgumentError( + raise orm_exc.MappedAnnotationError( f"The type provided inside the {self.column.key!r} " "attribute Mapped annotation is the SQLAlchemy type " f"{our_type}. Expected a Python type instead" ) + elif is_a_type(checks[0]): + if len(checks) == 1: + detail = ( + "the type object is not resolvable by the registry" + ) + elif len(checks) == 2: + detail = ( + f"neither '{checks[0]}' nor '{checks[1]}' " + "are resolvable by the registry" + ) + else: + detail = ( + f"""none of { + ", ".join(f"'{t}'" for t in checks) + } """ + "are resolvable by the registry" + ) + raise orm_exc.MappedAnnotationError( + "Could not locate SQLAlchemy Core type when resolving " + f"for Python type indicated by '{checks[0]}' inside " + "the " + f"Mapped[] annotation for the {self.column.key!r} " + f"attribute; {detail}" + ) else: - raise sa_exc.ArgumentError( - "Could not locate SQLAlchemy Core type for Python " - f"type {our_type} inside the {self.column.key!r} " - "attribute Mapped annotation" + raise orm_exc.MappedAnnotationError( + f"The object provided inside the {self.column.key!r} " + "attribute Mapped annotation is not a Python type, " + f"it's the object {de_stringified_argument!r}. " + "Expected a Python type." ) self.column._set_type(new_sqltype) diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index 00607203c12..38ee0297c37 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -1,5 +1,5 @@ # orm/query.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -30,10 +30,12 @@ from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import Mapping from typing import Optional from typing import overload from typing import Sequence +from typing import SupportsIndex from typing import Tuple from typing import Type from typing import TYPE_CHECKING @@ -91,7 +93,7 @@ from ..sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL from ..sql.selectable import SelectLabelStyle from ..util import deprecated -from ..util.typing import Literal +from ..util import warn_deprecated from ..util.typing import Self from ..util.typing import TupleAny from ..util.typing import TypeVarTuple @@ -873,8 +875,6 @@ def is_single_entity(self) -> bool: in its result list, and False if this query returns a tuple of entities for each result. - .. versionadded:: 1.3.11 - .. seealso:: :meth:`_query.Query.only_return_tuples` @@ -1061,7 +1061,7 @@ def yield_per(self, count: int) -> Self: ":meth:`_orm.Query.get`", alternative="The method is now available as :meth:`_orm.Session.get`", ) - def get(self, ident: _PKIdentityArgument) -> Optional[Any]: + def get(self, ident: _PKIdentityArgument) -> Optional[_T]: """Return an instance based on the given primary key identifier, or ``None`` if not found. @@ -1129,12 +1129,6 @@ def get(self, ident: _PKIdentityArgument) -> Optional[Any]: my_object = query.get({"id": 5, "version_id": 10}) - .. versionadded:: 1.3 the :meth:`_query.Query.get` - method now optionally - accepts a dictionary of attribute names to values in order to - indicate a primary key identifier. - - :return: The object instance, or ``None``. """ # noqa: E501 @@ -1716,8 +1710,6 @@ def transform(q): def get_execution_options(self) -> _ImmutableExecuteOptions: """Get the non-SQL options which will take effect during execution. - .. versionadded:: 1.3 - .. seealso:: :meth:`_query.Query.execution_options` @@ -2008,6 +2000,14 @@ def filter_by(self, **kwargs: Any) -> Self: entity of the query, or the last entity that was the target of a call to :meth:`_query.Query.join`. + .. note:: + + :class:`_query.Query` is a legacy construct as of SQLAlchemy 2.0. + See :meth:`_sql.Select.filter_by` for the comparable method on + 2.0-style :func:`_sql.select` constructs, where the behavior has + been enhanced in version 2.1 to search across all FROM clause + entities. See :ref:`change_8601` for background. + .. seealso:: :meth:`_query.Query.filter` - filter on SQL expressions. @@ -2589,6 +2589,12 @@ def select_from(self, *from_obj: _FromClauseArgument) -> Self: self._set_select_from(from_obj, False) return self + @overload + def __getitem__(self, item: slice) -> List[_T]: ... + + @overload + def __getitem__(self, item: SupportsIndex) -> _T: ... + def __getitem__(self, item: Any) -> Any: return orm_util._getitem( self, @@ -2697,11 +2703,18 @@ def distinct(self, *expr: _ColumnExpressionArgument[Any]) -> Self: the PostgreSQL dialect will render a ``DISTINCT ON ()`` construct. - .. deprecated:: 1.4 Using \*expr in other dialects is deprecated - and will raise :class:`_exc.CompileError` in a future version. + .. deprecated:: 2.1 Passing expressions to + :meth:`_orm.Query.distinct` is deprecated, use + :func:`_postgresql.distinct_on` instead. """ if expr: + warn_deprecated( + "Passing expression to ``distinct`` to generate a DISTINCT " + "ON clause is deprecated. Use instead the " + "``postgresql.distinct_on`` function as an extension.", + "2.1", + ) self._distinct = True self._distinct_on = self._distinct_on + tuple( coercions.expect(roles.ByOfRole, e) for e in expr @@ -2718,6 +2731,10 @@ def ext(self, extension: SyntaxExtension) -> Self: :ref:`examples_syntax_extensions` + :func:`_mysql.limit` - DML LIMIT for MySQL + + :func:`_postgresql.distinct_on` - DISTINCT ON for PostgreSQL + .. versionadded:: 2.1 """ @@ -2752,7 +2769,7 @@ def all(self) -> List[_T]: @_generative @_assertions(_no_clauseelement_condition) - def from_statement(self, statement: ExecutableReturnsRows) -> Self: + def from_statement(self, statement: roles.SelectStatementRole) -> Self: """Execute the given SELECT statement and return results. This method bypasses all internal statement compilation, and the @@ -2769,10 +2786,10 @@ def from_statement(self, statement: ExecutableReturnsRows) -> Self: :meth:`_sql.Select.from_statement` - v2 comparable method. """ - statement = coercions.expect( + _statement = coercions.expect( roles.SelectStatementRole, statement, apply_propagate_attrs=self ) - self._statement = statement + self._statement = _statement return self def first(self) -> Optional[_T]: @@ -2834,11 +2851,10 @@ def one_or_none(self) -> Optional[_T]: def one(self) -> _T: """Return exactly one result or raise an exception. - Raises ``sqlalchemy.orm.exc.NoResultFound`` if the query selects - no rows. Raises ``sqlalchemy.orm.exc.MultipleResultsFound`` - if multiple object identities are returned, or if multiple - rows are returned for a query that returns only scalar values - as opposed to full identity-mapped entities. + Raises :class:`_exc.NoResultFound` if the query selects no rows. + Raises :class:`_exc.MultipleResultsFound` if multiple object identities + are returned, or if multiple rows are returned for a query that returns + only scalar values as opposed to full identity-mapped entities. Calling :meth:`.one` results in an execution of the underlying query. @@ -2858,7 +2874,7 @@ def one(self) -> _T: def scalar(self) -> Any: """Return the first element of the first result or None if no rows present. If multiple rows are returned, - raises MultipleResultsFound. + raises :class:`_exc.MultipleResultsFound`. >>> session.query(Item).scalar() @@ -2892,7 +2908,7 @@ def __iter__(self) -> Iterator[_T]: try: yield from result # type: ignore except GeneratorExit: - # issue #8710 - direct iteration is not re-usable after + # issue #8710 - direct iteration is not reusable after # an iterable block is broken, so close the result result._soft_close() raise @@ -3254,11 +3270,14 @@ def delete( for ext in self._syntax_extensions: delete_._apply_syntax_extension_to_self(ext) - result: CursorResult[Any] = self.session.execute( - delete_, - self._params, - execution_options=self._execution_options.union( - {"synchronize_session": synchronize_session} + result = cast( + "CursorResult[Any]", + self.session.execute( + delete_, + self._params, + execution_options=self._execution_options.union( + {"synchronize_session": synchronize_session} + ), ), ) bulk_del.result = result # type: ignore @@ -3349,11 +3368,14 @@ def update( for ext in self._syntax_extensions: upd._apply_syntax_extension_to_self(ext) - result: CursorResult[Any] = self.session.execute( - upd, - self._params, - execution_options=self._execution_options.union( - {"synchronize_session": synchronize_session} + result = cast( + "CursorResult[Any]", + self.session.execute( + upd, + self._params, + execution_options=self._execution_options.union( + {"synchronize_session": synchronize_session} + ), ), ) bulk_ud.result = result # type: ignore diff --git a/lib/sqlalchemy/orm/relationships.py b/lib/sqlalchemy/orm/relationships.py index 608962b2bd7..894cde4936a 100644 --- a/lib/sqlalchemy/orm/relationships.py +++ b/lib/sqlalchemy/orm/relationships.py @@ -1,5 +1,5 @@ # orm/relationships.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -32,6 +32,7 @@ from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import NamedTuple from typing import NoReturn from typing import Optional @@ -39,6 +40,7 @@ from typing import Set from typing import Tuple from typing import Type +from typing import TYPE_CHECKING from typing import TypeVar from typing import Union import weakref @@ -56,6 +58,7 @@ from .base import state_str from .base import WriteOnlyMapped from .interfaces import _AttributeOptions +from .interfaces import _DataclassDefaultsDontSet from .interfaces import _IntrospectsAnnotations from .interfaces import MANYTOMANY from .interfaces import MANYTOONE @@ -63,8 +66,6 @@ from .interfaces import PropComparator from .interfaces import RelationshipDirection from .interfaces import StrategizedProperty -from .util import _orm_annotate -from .util import _orm_deannotate from .util import CascadeOptions from .. import exc as sa_exc from .. import Exists @@ -81,6 +82,7 @@ from ..sql._typing import _ColumnExpressionArgument from ..sql._typing import _HasClauseElement from ..sql.annotation import _safe_annotate +from ..sql.base import _NoArg from ..sql.elements import ColumnClause from ..sql.elements import ColumnElement from ..sql.util import _deep_annotate @@ -92,7 +94,6 @@ from ..sql.util import selectables_overlap from ..sql.util import visit_binary_product from ..util.typing import de_optionalize_union_types -from ..util.typing import Literal from ..util.typing import resolve_name_to_real_class_name if typing.TYPE_CHECKING: @@ -106,7 +107,7 @@ from .base import Mapped from .clsregistry import _class_resolver from .clsregistry import _ModNS - from .decl_base import _ClassScanMapperConfig + from .decl_base import _DeclarativeMapperConfig from .dependency import _DependencyProcessor from .mapper import Mapper from .query import Query @@ -340,7 +341,10 @@ class _RelationshipArgs(NamedTuple): @log.class_logger class RelationshipProperty( - _IntrospectsAnnotations, StrategizedProperty[_T], log.Identified + _DataclassDefaultsDontSet, + _IntrospectsAnnotations, + StrategizedProperty[_T], + log.Identified, ): """Describes an object property that holds a single item or list of items that correspond to a related database table. @@ -389,11 +393,12 @@ class RelationshipProperty( synchronize_pairs: _ColumnPairs secondary_synchronize_pairs: Optional[_ColumnPairs] - local_remote_pairs: Optional[_ColumnPairs] + local_remote_pairs: _ColumnPairs direction: RelationshipDirection _init_args: _RelationshipArgs + _disable_dataclass_default_factory = True def __init__( self, @@ -454,6 +459,15 @@ def __init__( _StringRelationshipArg("back_populates", back_populates, None), ) + if self._attribute_options.dataclasses_default not in ( + _NoArg.NO_ARG, + None, + ): + raise sa_exc.ArgumentError( + "Only 'None' is accepted as dataclass " + "default for a relationship()" + ) + self.post_update = post_update self.viewonly = viewonly if viewonly: @@ -500,7 +514,7 @@ def __init__( ) self.omit_join = omit_join - self.local_remote_pairs = _local_remote_pairs + self.local_remote_pairs = _local_remote_pairs or () self.load_on_pending = load_on_pending self.comparator_factory = ( comparator_factory or RelationshipProperty.Comparator @@ -519,8 +533,7 @@ def __init__( else: self._overlaps = () - # mypy ignoring the @property setter - self.cascade = cascade # type: ignore + self.cascade = cascade if back_populates: if backref: @@ -768,7 +781,7 @@ def __eq__(self, other: Any) -> ColumnElement[bool]: # type: ignore[override] many-to-one comparisons: * Comparisons against collections are not supported. - Use :meth:`~.Relationship.Comparator.contains`. + Use :meth:`~.RelationshipProperty.Comparator.contains`. * Compared to a scalar one-to-many, will produce a clause that compares the target columns in the parent to the given target. @@ -779,7 +792,7 @@ def __eq__(self, other: Any) -> ColumnElement[bool]: # type: ignore[override] queries that go beyond simple AND conjunctions of comparisons, such as those which use OR. Use explicit joins, outerjoins, or - :meth:`~.Relationship.Comparator.has` for + :meth:`~.RelationshipProperty.Comparator.has` for more comprehensive non-many-to-one scalar membership tests. * Comparisons against ``None`` given in a one-to-many @@ -790,10 +803,8 @@ def __eq__(self, other: Any) -> ColumnElement[bool]: # type: ignore[override] if self.property.direction in [ONETOMANY, MANYTOMANY]: return ~self._criterion_exists() else: - return _orm_annotate( - self.property._optimized_compare( - None, adapt_source=self.adapter - ) + return self.property._optimized_compare( + None, adapt_source=self.adapter ) elif self.property.uselist: raise sa_exc.InvalidRequestError( @@ -801,10 +812,8 @@ def __eq__(self, other: Any) -> ColumnElement[bool]: # type: ignore[override] "use contains() to test for membership." ) else: - return _orm_annotate( - self.property._optimized_compare( - other, adapt_source=self.adapter - ) + return self.property._optimized_compare( + other, adapt_source=self.adapter ) def _criterion_exists( @@ -837,9 +846,11 @@ def _criterion_exists( where_criteria = single_crit & where_criteria else: where_criteria = single_crit + dest_entity = info else: is_aliased_class = False to_selectable = None + dest_entity = self.mapper if self.adapter: source_selectable = self._source_selectable() @@ -868,10 +879,11 @@ def _criterion_exists( # annotate the *local* side of the join condition, in the case # of pj + sj this is the full primaryjoin, in the case of just # pj its the local side of the primaryjoin. + j: ColumnElement[bool] if sj is not None: - j = _orm_annotate(pj) & sj + j = pj & sj else: - j = _orm_annotate(pj, exclude=self.property.remote_side) + j = pj if ( where_criteria is not None @@ -893,6 +905,18 @@ def _criterion_exists( crit = j & sql.True_._ifnone(where_criteria) + # ensure the exists query gets picked up by the ORM + # compiler and that it has what we expect as parententity so that + # _adjust_for_extra_criteria() gets set up + dest = dest._annotate( + { + "parentmapper": dest_entity.mapper, + "entity_namespace": dest_entity, + "parententity": dest_entity, + } + )._set_propagate_attrs( + {"compile_state_plugin": "orm", "plugin_subject": dest_entity} + ) if secondary is not None: ex = ( sql.exists(1) @@ -931,12 +955,12 @@ def any( EXISTS (SELECT 1 FROM related WHERE related.my_id=my_table.id AND related.x=2) - Because :meth:`~.Relationship.Comparator.any` uses + Because :meth:`~.RelationshipProperty.Comparator.any` uses a correlated subquery, its performance is not nearly as good when compared against large target tables as that of using a join. - :meth:`~.Relationship.Comparator.any` is particularly + :meth:`~.RelationshipProperty.Comparator.any` is particularly useful for testing for empty collections:: session.query(MyClass).filter(~MyClass.somereference.any()) @@ -949,10 +973,10 @@ def any( NOT (EXISTS (SELECT 1 FROM related WHERE related.my_id=my_table.id)) - :meth:`~.Relationship.Comparator.any` is only + :meth:`~.RelationshipProperty.Comparator.any` is only valid for collections, i.e. a :func:`_orm.relationship` that has ``uselist=True``. For scalar references, - use :meth:`~.Relationship.Comparator.has`. + use :meth:`~.RelationshipProperty.Comparator.has`. """ if not self.property.uselist: @@ -985,15 +1009,15 @@ def has( EXISTS (SELECT 1 FROM related WHERE related.id==my_table.related_id AND related.x=2) - Because :meth:`~.Relationship.Comparator.has` uses + Because :meth:`~.RelationshipProperty.Comparator.has` uses a correlated subquery, its performance is not nearly as good when compared against large target tables as that of using a join. - :meth:`~.Relationship.Comparator.has` is only + :meth:`~.RelationshipProperty.Comparator.has` is only valid for scalar references, i.e. a :func:`_orm.relationship` that has ``uselist=False``. For collection references, - use :meth:`~.Relationship.Comparator.any`. + use :meth:`~.RelationshipProperty.Comparator.any`. """ if self.property.uselist: @@ -1008,7 +1032,7 @@ def contains( """Return a simple expression that tests a collection for containment of a particular item. - :meth:`~.Relationship.Comparator.contains` is + :meth:`~.RelationshipProperty.Comparator.contains` is only valid for a collection, i.e. a :func:`_orm.relationship` that implements one-to-many or many-to-many with ``uselist=True``. @@ -1027,12 +1051,12 @@ def contains( Where ```` is the value of the foreign key attribute on ``other`` which refers to the primary key of its parent object. From this it follows that - :meth:`~.Relationship.Comparator.contains` is + :meth:`~.RelationshipProperty.Comparator.contains` is very useful when used with simple one-to-many operations. For many-to-many operations, the behavior of - :meth:`~.Relationship.Comparator.contains` + :meth:`~.RelationshipProperty.Comparator.contains` has more caveats. The association table will be rendered in the statement, producing an "implicit" join, that is, includes multiple tables in the FROM @@ -1051,14 +1075,14 @@ def contains( Where ```` would be the primary key of ``other``. From the above, it is clear that - :meth:`~.Relationship.Comparator.contains` + :meth:`~.RelationshipProperty.Comparator.contains` will **not** work with many-to-many collections when used in queries that move beyond simple AND conjunctions, such as multiple - :meth:`~.Relationship.Comparator.contains` + :meth:`~.RelationshipProperty.Comparator.contains` expressions joined by OR. In such cases subqueries or explicit "outer joins" will need to be used instead. - See :meth:`~.Relationship.Comparator.any` for + See :meth:`~.RelationshipProperty.Comparator.any` for a less-performant alternative using EXISTS, or refer to :meth:`_query.Query.outerjoin` as well as :ref:`orm_queryguide_joins` @@ -1158,7 +1182,7 @@ def __ne__(self, other: Any) -> ColumnElement[bool]: # type: ignore[override] * Comparisons against collections are not supported. Use - :meth:`~.Relationship.Comparator.contains` + :meth:`~.RelationshipProperty.Comparator.contains` in conjunction with :func:`_expression.not_`. * Compared to a scalar one-to-many, will produce a clause that compares the target columns in the parent to @@ -1170,7 +1194,7 @@ def __ne__(self, other: Any) -> ColumnElement[bool]: # type: ignore[override] queries that go beyond simple AND conjunctions of comparisons, such as those which use OR. Use explicit joins, outerjoins, or - :meth:`~.Relationship.Comparator.has` in + :meth:`~.RelationshipProperty.Comparator.has` in conjunction with :func:`_expression.not_` for more comprehensive non-many-to-one scalar membership tests. @@ -1180,10 +1204,8 @@ def __ne__(self, other: Any) -> ColumnElement[bool]: # type: ignore[override] """ if other is None or isinstance(other, expression.Null): if self.property.direction == MANYTOONE: - return _orm_annotate( - ~self.property._optimized_compare( - None, adapt_source=self.adapter - ) + return ~self.property._optimized_compare( + None, adapt_source=self.adapter ) else: @@ -1195,7 +1217,10 @@ def __ne__(self, other: Any) -> ColumnElement[bool]: # type: ignore[override] "contains() to test for membership." ) else: - return _orm_annotate(self.__negated_contains_or_equals(other)) + return self.__negated_contains_or_equals(other) + + if TYPE_CHECKING: + property: RelationshipProperty[_PT] # noqa: A001 def _memoized_attr_property(self) -> RelationshipProperty[_PT]: self.prop.parent._check_configure() @@ -1418,8 +1443,11 @@ def _lazy_none_clause( criterion = adapt_source(criterion) return criterion + def _format_as_string(self, class_: type, key: str) -> str: + return f"{class_.__name__}.{key}" + def __str__(self) -> str: - return str(self.parent.class_.__name__) + "." + self.key + return self._format_as_string(self.parent.class_, self.key) def merge( self, @@ -1690,7 +1718,6 @@ def mapper(self) -> Mapper[_T]: return self.entity.mapper def do_init(self) -> None: - self._check_conflicts() self._process_dependent_arguments() self._setup_entity() self._setup_registry_dependencies() @@ -1741,10 +1768,8 @@ def _process_dependent_arguments(self) -> None: rel_arg = getattr(init_args, attr) val = rel_arg.resolved if val is not None: - rel_arg.resolved = _orm_deannotate( - coercions.expect( - roles.ColumnArgumentRole, val, argname=attr - ) + rel_arg.resolved = coercions.expect( + roles.ColumnArgumentRole, val, argname=attr ) secondary = init_args.secondary.resolved @@ -1788,7 +1813,7 @@ def _process_dependent_arguments(self) -> None: def declarative_scan( self, - decl_scan: _ClassScanMapperConfig, + decl_scan: _DeclarativeMapperConfig, registry: _RegistryType, cls: Type[Any], originating_module: Optional[str], @@ -1798,8 +1823,6 @@ def declarative_scan( extracted_mapped_annotation: Optional[_AnnotationScanType], is_dataclass_field: bool, ) -> None: - argument = extracted_mapped_annotation - if extracted_mapped_annotation is None: if self.argument is None: self._raise_for_required(key, cls) @@ -1886,6 +1909,18 @@ def declarative_scan( if self.argument is None: self.argument = cast("_RelationshipArgumentType[_T]", argument) + if ( + self._attribute_options.dataclasses_default_factory + is not _NoArg.NO_ARG + and self._attribute_options.dataclasses_default_factory + is not self.collection_class + ): + raise sa_exc.ArgumentError( + f"For relationship {self._format_as_string(cls, key)} using " + "dataclass options, default_factory must be exactly " + f"{self.collection_class}" + ) + @util.preload_module("sqlalchemy.orm.mapper") def _setup_entity(self, __argument: Any = None, /) -> None: if "entity" in self.__dict__: @@ -1988,25 +2023,6 @@ def _clsregistry_resolvers( return _resolver(self.parent.class_, self) - def _check_conflicts(self) -> None: - """Test that this relationship is legal, warn about - inheritance conflicts.""" - if self.parent.non_primary and not class_mapper( - self.parent.class_, configure=False - ).has_property(self.key): - raise sa_exc.ArgumentError( - "Attempting to assign a new " - "relationship '%s' to a non-primary mapper on " - "class '%s'. New relationships can only be added " - "to the primary mapper, i.e. the very first mapper " - "created for class '%s' " - % ( - self.key, - self.parent.class_.__name__, - self.parent.class_.__name__, - ) - ) - @property def cascade(self) -> CascadeOptions: """Return the current cascade setting for this @@ -2110,9 +2126,6 @@ def _generate_backref(self) -> None: """Interpret the 'backref' instruction to create a :func:`_orm.relationship` complementary to this one.""" - if self.parent.non_primary: - return - resolve_back_populates = self._init_args.back_populates.resolved if self.backref is not None and not resolve_back_populates: @@ -2210,6 +2223,18 @@ def _post_init(self) -> None: dependency._DependencyProcessor.from_relationship )(self) + if ( + self.uselist + and self._attribute_options.dataclasses_default + is not _NoArg.NO_ARG + ): + raise sa_exc.ArgumentError( + f"On relationship {self}, the dataclass default for " + "relationship may only be set for " + "a relationship that references a scalar value, i.e. " + "many-to-one or explicitly uselist=False" + ) + @util.memoized_property def _use_get(self) -> bool: """memoize the 'use_get' attribute of this RelationshipLoader's @@ -2377,7 +2402,6 @@ def __init__( self._determine_joins() assert self.primaryjoin is not None - self._sanitize_joins() self._annotate_fks() self._annotate_remote() self._annotate_local() @@ -2428,24 +2452,6 @@ def _log_joins(self) -> None: ) log.info("%s relationship direction %s", self.prop, self.direction) - def _sanitize_joins(self) -> None: - """remove the parententity annotation from our join conditions which - can leak in here based on some declarative patterns and maybe others. - - "parentmapper" is relied upon both by the ORM evaluator as well as - the use case in _join_fixture_inh_selfref_w_entity - that relies upon it being present, see :ticket:`3364`. - - """ - - self.primaryjoin = _deep_deannotate( - self.primaryjoin, values=("parententity", "proxy_key") - ) - if self.secondaryjoin is not None: - self.secondaryjoin = _deep_deannotate( - self.secondaryjoin, values=("parententity", "proxy_key") - ) - def _determine_joins(self) -> None: """Determine the 'primaryjoin' and 'secondaryjoin' attributes, if not passed to the constructor already. @@ -2965,9 +2971,6 @@ def _check_foreign_cols( ) -> None: """Check the foreign key columns collected and emit error messages.""" - - can_sync = False - foreign_cols = self._gather_columns_with_annotation( join_condition, "foreign" ) diff --git a/lib/sqlalchemy/orm/scoping.py b/lib/sqlalchemy/orm/scoping.py index 61cd0bd75d6..0634086ea24 100644 --- a/lib/sqlalchemy/orm/scoping.py +++ b/lib/sqlalchemy/orm/scoping.py @@ -1,5 +1,5 @@ # orm/scoping.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -32,6 +32,7 @@ from ..util import ThreadLocalRegistry from ..util import warn from ..util import warn_deprecated +from ..util.typing import Never from ..util.typing import TupleAny from ..util.typing import TypeVarTuple from ..util.typing import Unpack @@ -52,13 +53,13 @@ from .session import sessionmaker from .session import SessionTransaction from ..engine import Connection - from ..engine import CursorResult from ..engine import Engine from ..engine import Result from ..engine import Row from ..engine import RowMapping from ..engine.interfaces import _CoreAnyExecuteParams from ..engine.interfaces import _CoreSingleExecuteParams + from ..engine.interfaces import _ExecuteOptions from ..engine.interfaces import CoreExecuteOptionsParameter from ..engine.result import ScalarResult from ..sql._typing import _ColumnsClauseArgument @@ -72,7 +73,6 @@ from ..sql._typing import _T7 from ..sql._typing import _TypedColumnClauseArgument as _TCCA from ..sql.base import Executable - from ..sql.dml import UpdateBase from ..sql.elements import ClauseElement from ..sql.roles import TypedColumnsClauseRole from ..sql.selectable import ForUpdateParameter @@ -103,7 +103,7 @@ def __get__(self, instance: Any, owner: Type[_T]) -> Query[_T]: ... Session, ":class:`_orm.Session`", ":class:`_orm.scoping.scoped_session`", - classmethods=["close_all", "object_session", "identity_key"], + classmethods=["object_session", "identity_key"], methods=[ "__contains__", "__iter__", @@ -148,6 +148,7 @@ def __get__(self, instance: Any, owner: Type[_T]) -> Query[_T]: ... "autoflush", "no_autoflush", "info", + "execution_options", ], ) class scoped_session(Generic[_S]): @@ -557,7 +558,7 @@ def reset(self) -> None: :meth:`_orm.Session.close` and :meth:`_orm.Session.reset`. :meth:`_orm.Session.close` - a similar method will additionally - prevent re-use of the Session when the parameter + prevent reuse of the Session when the parameter :paramref:`_orm.Session.close_resets_only` is set to ``False``. """ # noqa: E501 @@ -694,7 +695,7 @@ def delete_all(self, instances: Iterable[object]) -> None: :meth:`.Session.delete` - main documentation on delete - .. versionadded: 2.1 + .. versionadded:: 2.1 """ # noqa: E501 @@ -713,18 +714,6 @@ def execute( _add_event: Optional[Any] = None, ) -> Result[Unpack[_Ts]]: ... - @overload - def execute( - self, - statement: UpdateBase, - params: Optional[_CoreAnyExecuteParams] = None, - *, - execution_options: OrmExecuteOptionsParameter = util.EMPTY_DICT, - bind_arguments: Optional[_BindArguments] = None, - _parent_execute_state: Optional[Any] = None, - _add_event: Optional[Any] = None, - ) -> CursorResult[Unpack[TupleAny]]: ... - @overload def execute( self, @@ -788,6 +777,13 @@ def execute( by :meth:`_engine.Connection.execution_options`, and may also provide additional options understood only in an ORM context. + The execution_options are passed along to methods like + :meth:`.Connection.execute` on :class:`.Connection` giving the + highest priority to execution_options that are passed to this + method explicitly, then the options that are present on the + statement object if any, and finally those options present + session-wide. + .. seealso:: :ref:`orm_queryguide_execution_options` - ORM-specific execution @@ -1078,7 +1074,7 @@ def get( Contents of this dictionary are passed to the :meth:`.Session.get_bind` method. - .. versionadded: 2.0.0rc1 + .. versionadded:: 2.0.0rc1 :return: The object instance, or ``None``. @@ -1116,8 +1112,7 @@ def get_one( Proxied for the :class:`_orm.Session` class on behalf of the :class:`_orm.scoping.scoped_session` class. - Raises ``sqlalchemy.orm.exc.NoResultFound`` if the query - selects no rows. + Raises :class:`_exc.NoResultFound` if the query selects no rows. For a detailed documentation of the arguments see the method :meth:`.Session.get`. @@ -1617,7 +1612,7 @@ def merge_all( :meth:`.Session.merge` - main documentation on merge - .. versionadded: 2.1 + .. versionadded:: 2.1 """ # noqa: E501 @@ -1858,6 +1853,17 @@ def rollback(self) -> None: return self._proxied.rollback() + @overload + def scalar( + self, + statement: TypedReturnsRows[Never], + params: Optional[_CoreSingleExecuteParams] = None, + *, + execution_options: OrmExecuteOptionsParameter = util.EMPTY_DICT, + bind_arguments: Optional[_BindArguments] = None, + **kw: Any, + ) -> Optional[Any]: ... + @overload def scalar( self, @@ -2160,20 +2166,18 @@ def info(self) -> Any: return self._proxied.info - @classmethod - def close_all(cls) -> None: - r"""Close *all* sessions in memory. - - .. container:: class_bases - - Proxied for the :class:`_orm.Session` class on - behalf of the :class:`_orm.scoping.scoped_session` class. - - .. deprecated:: 1.3 The :meth:`.Session.close_all` method is deprecated and will be removed in a future release. Please refer to :func:`.session.close_all_sessions`. + @property + def execution_options(self) -> _ExecuteOptions: + r"""Proxy for the :attr:`_orm.Session.execution_options` attribute + on behalf of the :class:`_orm.scoping.scoped_session` class. """ # noqa: E501 - return Session.close_all() + return self._proxied.execution_options + + @execution_options.setter + def execution_options(self, attr: _ExecuteOptions) -> None: + self._proxied.execution_options = attr @classmethod def object_session(cls, instance: object) -> Optional[Session]: diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index e5dd55d12f7..0aa6458f97d 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -1,5 +1,5 @@ # orm/session.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -22,6 +22,7 @@ from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import NoReturn from typing import Optional from typing import overload @@ -88,10 +89,9 @@ from ..sql.base import CompileState from ..sql.schema import Table from ..sql.selectable import ForUpdateArg -from ..sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL from ..util import deprecated_params from ..util import IdentitySet -from ..util.typing import Literal +from ..util.typing import Never from ..util.typing import TupleAny from ..util.typing import TypeVarTuple from ..util.typing import Unpack @@ -107,7 +107,6 @@ from .mapper import Mapper from .path_registry import PathRegistry from .query import RowReturningQuery - from ..engine import CursorResult from ..engine import Result from ..engine import Row from ..engine import RowMapping @@ -132,7 +131,6 @@ from ..sql._typing import _TypedColumnClauseArgument as _TCCA from ..sql.base import Executable from ..sql.base import ExecutableOption - from ..sql.dml import UpdateBase from ..sql.elements import ClauseElement from ..sql.roles import TypedColumnsClauseRole from ..sql.selectable import ForUpdateParameter @@ -207,18 +205,6 @@ def _state_session(state: InstanceState[Any]) -> Optional[Session]: class _SessionClassMethods: """Class-level methods for :class:`.Session`, :class:`.sessionmaker`.""" - @classmethod - @util.deprecated( - "1.3", - "The :meth:`.Session.close_all` method is deprecated and will be " - "removed in a future release. Please refer to " - ":func:`.session.close_all_sessions`.", - ) - def close_all(cls) -> None: - """Close *all* sessions in memory.""" - - close_all_sessions() - @classmethod @util.preload_module("sqlalchemy.orm.util") def identity_key( @@ -308,8 +294,16 @@ class ORMExecuteState(util.MemoizedSlots): """ parameters: Optional[_CoreAnyExecuteParams] - """Dictionary of parameters that was passed to - :meth:`_orm.Session.execute`.""" + """Optional mapping or list of mappings of parameters that was passed to + :meth:`_orm.Session.execute`. + + May be mutated or re-assigned in place, which will take effect as the + effective parameters passed to the method. + + .. versionchanged:: 2.1 :attr:`.ORMExecuteState.parameters` may now be + mutated or replaced. + + """ execution_options: _ExecuteOptions """The complete dictionary of current execution options. @@ -842,7 +836,7 @@ class SessionTransactionOrigin(Enum): """transaction were started by calling :meth:`_orm.Session.begin`""" BEGIN_NESTED = 2 - """tranaction were started by :meth:`_orm.Session.begin_nested`""" + """transaction were started by :meth:`_orm.Session.begin_nested`""" SUBTRANSACTION = 3 """transaction is an internal "subtransaction" """ @@ -1498,6 +1492,7 @@ class Session(_SessionClassMethods, EventTarget): enable_baked_queries: bool twophase: bool join_transaction_mode: JoinTransactionMode + execution_options: _ExecuteOptions = util.EMPTY_DICT _query_cls: Type[Query[Any]] _close_state: _SessionCloseState @@ -1517,6 +1512,7 @@ def __init__( autocommit: Literal[False] = False, join_transaction_mode: JoinTransactionMode = "conditional_savepoint", close_resets_only: Union[bool, _NoArg] = _NoArg.NO_ARG, + execution_options: OrmExecuteOptionsParameter = util.EMPTY_DICT, ): r"""Construct a new :class:`_orm.Session`. @@ -1612,6 +1608,15 @@ def __init__( flag therefore only affects applications that are making explicit use of this extension within their own code. + :param execution_options: optional dictionary of execution options + that will be applied to all calls to :meth:`_orm.Session.execute`, + :meth:`_orm.Session.scalars`, and similar. Execution options + present in statements as well as options passed to methods like + :meth:`_orm.Session.execute` explicitly take precedence over + the session-wide options. + + .. versionadded:: 2.1 + :param expire_on_commit: Defaults to ``True``. When ``True``, all instances will be fully expired after each :meth:`~.commit`, so that all attribute/object access subsequent to a completed @@ -1731,7 +1736,7 @@ def __init__( :param close_resets_only: Defaults to ``True``. Determines if the session should reset itself after calling ``.close()`` - or should pass in a no longer usable state, disabling re-use. + or should pass in a no longer usable state, disabling reuse. .. versionadded:: 2.0.22 added flag ``close_resets_only``. A future SQLAlchemy version may change the default value of @@ -1773,6 +1778,10 @@ def __init__( self.autoflush = autoflush self.expire_on_commit = expire_on_commit self.enable_baked_queries = enable_baked_queries + if execution_options: + self.execution_options = self.execution_options.union( + execution_options + ) # the idea is that at some point NO_ARG will warn that in the future # the default will switch to close_resets_only=False. @@ -2171,7 +2180,28 @@ def _execute_internal( compile_state_cls = None bind_arguments.setdefault("clause", statement) - execution_options = util.coerce_to_immutabledict(execution_options) + combined_execution_options: util.immutabledict[str, Any] = ( + util.coerce_to_immutabledict(execution_options) + ) + if self.execution_options: + # merge given execution options with session-wide execution + # options. if the statement also has execution_options, + # maintain priority of session.execution_options -> + # statement.execution_options -> method passed execution_options + # by omitting from the base execution options those keys that + # will come from the statement + if statement._execution_options: + combined_execution_options = util.immutabledict( + { + k: v + for k, v in self.execution_options.items() + if k not in statement._execution_options + } + ).union(combined_execution_options) + else: + combined_execution_options = self.execution_options.union( + combined_execution_options + ) if _parent_execute_state: events_todo = _parent_execute_state._remaining_events() @@ -2190,12 +2220,13 @@ def _execute_internal( # as "pre fetch" for DML, etc. ( statement, - execution_options, + combined_execution_options, + params, ) = compile_state_cls.orm_pre_session_exec( self, statement, params, - execution_options, + combined_execution_options, bind_arguments, True, ) @@ -2204,7 +2235,7 @@ def _execute_internal( self, statement, params, - execution_options, + combined_execution_options, bind_arguments, compile_state_cls, events_todo, @@ -2221,7 +2252,8 @@ def _execute_internal( return fn_result statement = orm_exec_state.statement - execution_options = orm_exec_state.local_execution_options + combined_execution_options = orm_exec_state.local_execution_options + params = orm_exec_state.parameters if compile_state_cls is not None: # now run orm_pre_session_exec() "for real". if there were @@ -2231,15 +2263,19 @@ def _execute_internal( # autoflush will also be invoked in this step if enabled. ( statement, - execution_options, + combined_execution_options, + params, ) = compile_state_cls.orm_pre_session_exec( self, statement, params, - execution_options, + combined_execution_options, bind_arguments, False, ) + else: + # Issue #9809: unconditionally autoflush for Core statements + self._autoflush() bind = self.get_bind(**bind_arguments) @@ -2249,7 +2285,9 @@ def _execute_internal( if TYPE_CHECKING: params = cast(_CoreSingleExecuteParams, params) return conn.scalar( - statement, params or {}, execution_options=execution_options + statement, + params or {}, + execution_options=combined_execution_options, ) if compile_state_cls: @@ -2258,14 +2296,14 @@ def _execute_internal( self, statement, params or {}, - execution_options, + combined_execution_options, bind_arguments, conn, ) ) else: result = conn.execute( - statement, params, execution_options=execution_options + statement, params, execution_options=combined_execution_options ) if _scalar_result: @@ -2285,18 +2323,6 @@ def execute( _add_event: Optional[Any] = None, ) -> Result[Unpack[_Ts]]: ... - @overload - def execute( - self, - statement: UpdateBase, - params: Optional[_CoreAnyExecuteParams] = None, - *, - execution_options: OrmExecuteOptionsParameter = util.EMPTY_DICT, - bind_arguments: Optional[_BindArguments] = None, - _parent_execute_state: Optional[Any] = None, - _add_event: Optional[Any] = None, - ) -> CursorResult[Unpack[TupleAny]]: ... - @overload def execute( self, @@ -2355,6 +2381,13 @@ def execute( by :meth:`_engine.Connection.execution_options`, and may also provide additional options understood only in an ORM context. + The execution_options are passed along to methods like + :meth:`.Connection.execute` on :class:`.Connection` giving the + highest priority to execution_options that are passed to this + method explicitly, then the options that are present on the + statement object if any, and finally those options present + session-wide. + .. seealso:: :ref:`orm_queryguide_execution_options` - ORM-specific execution @@ -2378,6 +2411,19 @@ def execute( _add_event=_add_event, ) + # special case to handle mypy issue: + # https://github.com/python/mypy/issues/20651 + @overload + def scalar( + self, + statement: TypedReturnsRows[Never], + params: Optional[_CoreSingleExecuteParams] = None, + *, + execution_options: OrmExecuteOptionsParameter = util.EMPTY_DICT, + bind_arguments: Optional[_BindArguments] = None, + **kw: Any, + ) -> Optional[Any]: ... + @overload def scalar( self, @@ -2547,7 +2593,7 @@ def reset(self) -> None: :meth:`_orm.Session.close` and :meth:`_orm.Session.reset`. :meth:`_orm.Session.close` - a similar method will additionally - prevent re-use of the Session when the parameter + prevent reuse of the Session when the parameter :paramref:`_orm.Session.close_resets_only` is set to ``False``. """ self._close_impl(invalidate=False, is_reset=True) @@ -3060,7 +3106,7 @@ def no_autoflush(self) -> Iterator[Session]: "This warning originated from the Session 'autoflush' process, " "which was invoked automatically in response to a user-initiated " "operation. Consider using ``no_autoflush`` context manager if this " - "warning happended while initializing objects.", + "warning happened while initializing objects.", sa_exc.SAWarning, ) def _autoflush(self) -> None: @@ -3560,7 +3606,7 @@ def delete_all(self, instances: Iterable[object]) -> None: :meth:`.Session.delete` - main documentation on delete - .. versionadded: 2.1 + .. versionadded:: 2.1 """ @@ -3715,7 +3761,7 @@ def get( Contents of this dictionary are passed to the :meth:`.Session.get_bind` method. - .. versionadded: 2.0.0rc1 + .. versionadded:: 2.0.0rc1 :return: The object instance, or ``None``. @@ -3747,8 +3793,7 @@ def get_one( """Return exactly one instance based on the given primary key identifier, or raise an exception if not found. - Raises ``sqlalchemy.orm.exc.NoResultFound`` if the query - selects no rows. + Raises :class:`_exc.NoResultFound` if the query selects no rows. For a detailed documentation of the arguments see the method :meth:`.Session.get`. @@ -3859,10 +3904,12 @@ def _get_impl( ) ) from err + for_update_arg = ForUpdateArg._from_argument(with_for_update) + if ( not populate_existing and not mapper.always_refresh - and with_for_update is None + and for_update_arg is None ): instance = self._identity_lookup( mapper, @@ -3882,24 +3929,18 @@ def _get_impl( # TODO: this was being tested before, but this is not possible assert instance is not LoaderCallableStatus.PASSIVE_CLASS_MISMATCH - # set_label_style() not strictly necessary, however this will ensure - # that tablename_colname style is used which at the moment is - # asserted in a lot of unit tests :) - load_options = context.QueryContext.default_load_options if populate_existing: load_options += {"_populate_existing": populate_existing} - statement = sql.select(mapper).set_label_style( - LABEL_STYLE_TABLENAME_PLUS_COL - ) - if with_for_update is not None: - statement._for_update_arg = ForUpdateArg._from_argument( - with_for_update - ) + statement = sql.select(mapper) + if for_update_arg is not None: + statement._for_update_arg = for_update_arg if options: statement = statement.options(*options) + if self.execution_options: + execution_options = self.execution_options.union(execution_options) return db_load_fn( self, statement, @@ -4004,7 +4045,7 @@ def merge_all( :meth:`.Session.merge` - main documentation on merge - .. versionadded: 2.1 + .. versionadded:: 2.1 """ @@ -4074,14 +4115,7 @@ def _merge( else: key_is_persistent = True - if key in self.identity_map: - try: - merged = self.identity_map[key] - except KeyError: - # object was GC'ed right as we checked for it - merged = None - else: - merged = None + merged = self.identity_map.get(key) if merged is None: if key_is_persistent and key in _resolve_conflict_map: @@ -5240,8 +5274,6 @@ def close_all_sessions() -> None: This function is not for general use but may be useful for test suites within the teardown scheme. - .. versionadded:: 1.3 - """ for sess in _sessions.values(): diff --git a/lib/sqlalchemy/orm/state.py b/lib/sqlalchemy/orm/state.py index b5ba1615ca9..3f1d7fa740c 100644 --- a/lib/sqlalchemy/orm/state.py +++ b/lib/sqlalchemy/orm/state.py @@ -1,5 +1,5 @@ # orm/state.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -19,6 +19,7 @@ from typing import Dict from typing import Generic from typing import Iterable +from typing import Literal from typing import Optional from typing import Protocol from typing import Set @@ -45,7 +46,6 @@ from .. import exc as sa_exc from .. import inspection from .. import util -from ..util.typing import Literal from ..util.typing import TupleAny from ..util.typing import Unpack @@ -269,8 +269,6 @@ def deleted(self) -> bool: :class:`.Session`, use the :attr:`.InstanceState.was_deleted` accessor. - .. versionadded: 1.1 - .. seealso:: :ref:`session_object_states` @@ -337,8 +335,6 @@ def _track_last_known_value(self, key: str) -> None: """Track the last known value of a particular key after expiration operations. - .. versionadded:: 1.3 - """ lkv = self._last_known_values @@ -516,6 +512,31 @@ def _dispose(self) -> None: # used by the test suite, apparently self._detach() + def _force_dereference(self) -> None: + """Force this InstanceState to act as though its weakref has + been GC'ed. + + this is used for test code that has to test reactions to objects + being GC'ed. We can't reliably force GCs to happen under all + CI circumstances. + + """ + + # if _strong_obj is set, then our object would not be getting + # GC'ed (at least within the scope of what we use this for in tests). + # so make sure this is not set + assert self._strong_obj is None + + obj = self.obj() + if obj is None: + # object was GC'ed and we're done! woop + return + + del obj + + self._cleanup(self.obj) + self.obj = lambda: None # type: ignore + def _cleanup(self, ref: weakref.ref[_O]) -> None: """Weakref callback cleanup. diff --git a/lib/sqlalchemy/orm/state_changes.py b/lib/sqlalchemy/orm/state_changes.py index 10e417e85d1..e8fb02f9bf5 100644 --- a/lib/sqlalchemy/orm/state_changes.py +++ b/lib/sqlalchemy/orm/state_changes.py @@ -1,13 +1,11 @@ # orm/state_changes.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -"""State tracking utilities used by :class:`_orm.Session`. - -""" +"""State tracking utilities used by :class:`_orm.Session`.""" from __future__ import annotations @@ -17,6 +15,7 @@ from typing import Callable from typing import cast from typing import Iterator +from typing import Literal from typing import NoReturn from typing import Optional from typing import Tuple @@ -25,7 +24,6 @@ from .. import exc as sa_exc from .. import util -from ..util.typing import Literal _F = TypeVar("_F", bound=Callable[..., Any]) @@ -82,7 +80,7 @@ def declare_states( indicate state should not change at the end of the method. """ - assert prerequisite_states, "no prequisite states sent" + assert prerequisite_states, "no prerequisite states sent" has_prerequisite_states = ( prerequisite_states is not _StateChangeStates.ANY ) @@ -127,7 +125,7 @@ def _go(fn: _F, self: Any, *arg: Any, **kw: Any) -> Any: ) else: raise sa_exc.IllegalStateChangeError( - f"Cant run operation '{fn.__name__}()' here; " + f"Can't run operation '{fn.__name__}()' here; " f"will move to state {moves_to!r} where we are " f"expecting {next_state!r}", code="isce", diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index 8b89eb45238..d7672b3e4a0 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -1,5 +1,5 @@ # orm/strategies.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -8,7 +8,7 @@ """sqlalchemy.orm.interfaces.LoaderStrategy - implementations, and related MapperOptions.""" +implementations, and related MapperOptions.""" from __future__ import annotations @@ -16,6 +16,7 @@ import itertools from typing import Any from typing import Dict +from typing import Literal from typing import Optional from typing import Tuple from typing import TYPE_CHECKING @@ -59,7 +60,6 @@ from ..sql import visitors from ..sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL from ..sql.selectable import Select -from ..util.typing import Literal if TYPE_CHECKING: from .mapper import Mapper @@ -77,6 +77,7 @@ def _register_attribute( proxy_property=None, active_history=False, impl_class=None, + default_scalar_value=None, **kw, ): listen_hooks = [] @@ -138,6 +139,7 @@ def _register_attribute( typecallable=typecallable, callable_=callable_, active_history=active_history, + default_scalar_value=default_scalar_value, impl_class=impl_class, send_modified_events=not useobject or not prop.viewonly, doc=prop.doc, @@ -257,6 +259,7 @@ def init_class_attribute(self, mapper): useobject=False, compare_function=coltype.compare_values, active_history=active_history, + default_scalar_value=self.parent_property._default_scalar_value, ) def create_row_processor( @@ -335,6 +338,17 @@ def setup_query( memoized_populators[self.parent_property] = fetch + # if the column being loaded is the polymorphic discriminator, + # and we have a with_expression() providing the actual column, + # update the query_entity to use the actual column instead of + # the default expression + if ( + query_entity._polymorphic_discriminator is self.columns[0] + and loadopt + and loadopt._extra_criteria + ): + query_entity._polymorphic_discriminator = columns[0] + def create_row_processor( self, context, @@ -370,6 +384,7 @@ def init_class_attribute(self, mapper): useobject=False, compare_function=self.columns[0].type.compare_values, accepts_scalar_loader=False, + default_scalar_value=self.parent_property._default_scalar_value, ) @@ -455,6 +470,7 @@ def init_class_attribute(self, mapper): compare_function=self.columns[0].type.compare_values, callable_=self._load_for_state, load_on_unexpire=False, + default_scalar_value=self.parent_property._default_scalar_value, ) def setup_query( @@ -735,10 +751,7 @@ def __init__( ) = join_condition.create_lazy_clause(reverse_direction=True) if self.parent_property.order_by: - self._order_by = [ - sql_util._deep_annotate(elem, {"_orm_adapt": True}) - for elem in util.to_list(self.parent_property.order_by) - ] + self._order_by = util.to_list(self.parent_property.order_by) else: self._order_by = None @@ -807,9 +820,7 @@ def init_class_attribute(self, mapper): ) def _memoized_attr__simple_lazy_clause(self): - lazywhere = sql_util._deep_annotate( - self._lazywhere, {"_orm_adapt": True} - ) + lazywhere = self._lazywhere criterion, bind_to_col = (lazywhere, self._bind_to_col) @@ -1028,7 +1039,6 @@ def _emit_lazyload( stmt = Select._create_raw_select( _raw_columns=[clauseelement], _propagate_attrs=clauseelement._propagate_attrs, - _label_style=LABEL_STYLE_TABLENAME_PLUS_COL, _compile_options=_ORMCompileState.default_compile_options, ) load_options = QueryContext.default_load_options @@ -1117,10 +1127,7 @@ def _lazyload_reverse(compile_context): if execution_options: execution_options = util.EMPTY_DICT.merge_with( - execution_options, - { - "_sa_orm_load_options": load_options, - }, + execution_options, {"_sa_orm_load_options": load_options} ) else: execution_options = { @@ -1442,7 +1449,6 @@ def _load_for_path( alternate_effective_path = path._truncate_recursive() extra_options = (new_opt,) else: - new_opt = None alternate_effective_path = path extra_options = () @@ -1743,7 +1749,7 @@ def _setup_options( loadopt, ): # note that because the subqueryload object - # does not re-use the cached query, instead always making + # does not reuse the cached query, instead always making # use of the current invoked query, while we have two queries # here (orig and context.query), they are both non-cached # queries and we can transfer the options as is without @@ -2028,7 +2034,12 @@ def create_row_processor( if len(path) == 1: if not orm_util._entity_isa(query_entity.entity_zero, self.parent): return - elif not orm_util._entity_isa(path[-1], self.parent): + elif not orm_util._entity_isa( + path[-1], self.parent + ) and not self.parent.isa(path[-1].mapper): + # second check accommodates a polymorphic entity where + # the path has been normalized to the base mapper but + # self.parent is a subclass mapper. Fixes #13209. return subq = self._setup_query_from_rowproc( @@ -2172,8 +2183,6 @@ def setup_query( path = path[self.parent_property] - with_polymorphic = None - user_defined_adapter = ( self._init_user_defined_eager_proc( loadopt, compile_state, compile_state.attributes @@ -2451,10 +2460,7 @@ def _create_eager_join( # whether or not the Query will wrap the selectable in a subquery, # and then attach eager load joins to that (i.e., in the case of # LIMIT/OFFSET etc.) - should_nest_selectable = ( - compile_state.multi_row_eager_loaders - and compile_state._should_nest_selectable - ) + should_nest_selectable = compile_state._should_nest_selectable query_entity_key = None @@ -2971,6 +2977,21 @@ class _SelectInLoader(_PostLoader, util.MemoizedSlots): _chunksize = 500 + @classmethod + def _set_chunksize(cls, loadopt) -> int: + if loadopt is None or hasattr(loadopt, "local_opts") is None: + return cls._chunksize + + user_input = loadopt.local_opts.get("chunksize", None) + if user_input is None: + return cls._chunksize + elif not isinstance(user_input, int) or user_input < 1: + raise sa_exc.ArgumentError( + f"'chunksize={user_input}' is not an appropriate input, " + f"please use a positive non-zero integer." + ) + return user_input + def __init__(self, parent, strategy_key): super().__init__(parent, strategy_key) self.join_depth = self.parent_property.join_depth @@ -3109,7 +3130,14 @@ def create_row_processor( if len(path) == 1: if not orm_util._entity_isa(query_entity.entity_zero, self.parent): return - elif not orm_util._entity_isa(path[-1], self.parent): + elif not orm_util._entity_isa( + path[-1], self.parent + ) and not self.parent.isa(path[-1].mapper): + # second check accommodates a polymorphic entity where + # the path has been normalized to the base mapper but + # self.parent is a subclass mapper, e.g. + # joinedload(A.b.of_type(poly)).selectinload(poly.Sub.rel) + # Fixes #13209. return selectin_path = effective_path @@ -3218,7 +3246,6 @@ def _load_for_path( entity_sql = effective_entity.__clause_element__() q = Select._create_raw_select( _raw_columns=[bundle_sql, entity_sql], - _label_style=LABEL_STYLE_TABLENAME_PLUS_COL, _compile_options=_ORMCompileState.default_compile_options, _propagate_attrs={ "compile_state_plugin": "orm", @@ -3335,6 +3362,8 @@ def _setup_outermost_orderby(compile_context): _setup_outermost_orderby, self.parent_property ) + chunksize = self._set_chunksize(loadopt) + if query_info.load_only_child: self._load_via_child( our_states, @@ -3343,10 +3372,16 @@ def _setup_outermost_orderby(compile_context): q, context, execution_options, + chunksize, ) else: self._load_via_parent( - our_states, query_info, q, context, execution_options + our_states, + query_info, + q, + context, + execution_options, + chunksize, ) def _load_via_child( @@ -3357,14 +3392,15 @@ def _load_via_child( q, context, execution_options, + chunksize, ): uselist = self.uselist # this sort is really for the benefit of the unit tests our_keys = sorted(our_states) while our_keys: - chunk = our_keys[0 : self._chunksize] - our_keys = our_keys[self._chunksize :] + chunk = our_keys[0:chunksize] + our_keys = our_keys[chunksize:] data = { k: v for k, v in context.session.execute( @@ -3405,14 +3441,14 @@ def _load_via_child( state.get_impl(self.key).set_committed_value(state, dict_, None) def _load_via_parent( - self, our_states, query_info, q, context, execution_options + self, our_states, query_info, q, context, execution_options, chunksize ): uselist = self.uselist _empty_result = () if uselist else None while our_states: - chunk = our_states[0 : self._chunksize] - our_states = our_states[self._chunksize :] + chunk = our_states[0:chunksize] + our_states = our_states[chunksize:] primary_keys = [ key[0] if query_info.zero_idx else key diff --git a/lib/sqlalchemy/orm/strategy_options.py b/lib/sqlalchemy/orm/strategy_options.py index 5d212371983..c9bd03b67a3 100644 --- a/lib/sqlalchemy/orm/strategy_options.py +++ b/lib/sqlalchemy/orm/strategy_options.py @@ -1,14 +1,12 @@ # orm/strategy_options.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php # mypy: allow-untyped-defs, allow-untyped-calls -""" - -""" +""" """ from __future__ import annotations @@ -19,6 +17,7 @@ from typing import Dict from typing import Final from typing import Iterable +from typing import Literal from typing import Optional from typing import overload from typing import Sequence @@ -54,7 +53,6 @@ from ..sql import traversals from ..sql import visitors from ..sql.base import _generative -from ..util.typing import Literal from ..util.typing import Self _RELATIONSHIP_TOKEN: Final[Literal["relationship"]] = "relationship" @@ -224,7 +222,7 @@ def load_only(self, *attrs: _AttrType, raiseload: bool = False) -> Self: """ cloned = self._set_column_strategy( - attrs, + _expand_column_strategy_attrs(attrs), {"deferred": False, "instrument": True}, ) @@ -361,6 +359,7 @@ def selectinload( self, attr: _AttrType, recursion_depth: Optional[int] = None, + chunksize: Optional[int] = None, ) -> Self: """Indicate that the given attribute should be loaded using SELECT IN eager loading. @@ -399,6 +398,11 @@ def selectinload( .. versionadded:: 2.0 added :paramref:`_orm.selectinload.recursion_depth` + :param chunksize: optional int; when set to a positive non-zero + integer, the keys from the IN statement will be chunked relative + to the passed parameter + + .. versionadded:: 2.1.0b3 .. seealso:: @@ -410,7 +414,7 @@ def selectinload( return self._set_relationship_strategy( attr, {"lazy": "selectin"}, - opts={"recursion_depth": recursion_depth}, + opts={"recursion_depth": recursion_depth, "chunksize": chunksize}, ) def lazyload(self, attr: _AttrType) -> Self: @@ -637,7 +641,9 @@ def defer(self, key: _AttrType, raiseload: bool = False) -> Self: strategy = {"deferred": True, "instrument": True} if raiseload: strategy["raiseload"] = True - return self._set_column_strategy((key,), strategy) + return self._set_column_strategy( + _expand_column_strategy_attrs((key,)), strategy + ) def undefer(self, key: _AttrType) -> Self: r"""Indicate that the given column-oriented attribute should be @@ -676,7 +682,8 @@ def undefer(self, key: _AttrType) -> Self: """ # noqa: E501 return self._set_column_strategy( - (key,), {"deferred": False, "instrument": True} + _expand_column_strategy_attrs((key,)), + {"deferred": False, "instrument": True}, ) def undefer_group(self, name: str) -> Self: @@ -730,8 +737,6 @@ def with_expression( with_expression(SomeClass.x_y_expr, SomeClass.x + SomeClass.y) ) - .. versionadded:: 1.2 - :param key: Attribute to be populated :param expr: SQL expression to be applied to the attribute. @@ -759,8 +764,6 @@ def selectin_polymorphic(self, classes: Iterable[Type[Any]]) -> Self: key values, and is the per-query analogue to the ``"selectin"`` setting on the :paramref:`.mapper.polymorphic_load` parameter. - .. versionadded:: 1.2 - .. seealso:: :ref:`polymorphic_selectin` @@ -1102,7 +1105,6 @@ def _reconcile_query_entities_with_us(self, mapper_entities, raiseerr): """ path = self.path - ezero = None for ent in mapper_entities: ezero = ent.entity_zero if ezero and orm_util._entity_corresponds_to( @@ -1206,8 +1208,6 @@ def options(self, *opts: _AbstractLoad) -> Self: :class:`_orm.Load` objects) which should be applied to the path specified by this :class:`_orm.Load` object. - .. versionadded:: 1.3.6 - .. seealso:: :func:`.defaultload` @@ -1251,7 +1251,7 @@ def _clone_for_bind_strategy( ) elif path_is_property(self.path): - # re-use the lookup which will raise a nicely formatted + # reuse the lookup which will raise a nicely formatted # LoaderStrategyException if strategy: self.path.prop._strategy_lookup(self.path.prop, strategy[0]) @@ -1585,7 +1585,7 @@ def __eq__(self, other): def is_opts_only(self) -> bool: return bool(self.local_opts and self.strategy is None) - def _clone(self, **kw: Any) -> _LoadElement: + def _clone(self, **kw: Any) -> Self: cls = self.__class__ s = cls.__new__(cls) @@ -1806,7 +1806,7 @@ def _prepend_path_from(self, parent: Load) -> _LoadElement: return self._prepend_path(parent.path) - def _prepend_path(self, path: PathRegistry) -> _LoadElement: + def _prepend_path(self, path: PathRegistry) -> Self: cloned = self._clone() assert cloned.strategy == self.strategy @@ -1975,6 +1975,24 @@ def _init_path( return path + def _prepend_path(self, path: PathRegistry) -> Self: + """Override to also prepend the path for _path_with_polymorphic_path. + + When using .options() to chain loader options with of_type(), this + ensures that the polymorphic path information is correctly updated + to include the parent path. Fixes issue #13202. + """ + cloned = super()._prepend_path(path) + + # Also prepend the parent path to _path_with_polymorphic_path if + # present + if self._path_with_polymorphic_path is not None: + cloned._path_with_polymorphic_path = PathRegistry.coerce( + path[0:-1] + self._path_with_polymorphic_path[:] + ) + + return cloned + def _generate_extra_criteria(self, context): """Apply the current bound parameters in a QueryContext to the immediate "extra_criteria" stored with this Load object. @@ -2394,6 +2412,23 @@ def loader_unbound_fn(fn: _FN) -> _FN: return fn +def _expand_column_strategy_attrs( + attrs: Tuple[_AttrType, ...], +) -> Tuple[_AttrType, ...]: + return cast( + "Tuple[_AttrType, ...]", + tuple( + a + for attr in attrs + for a in ( + cast("QueryableAttribute[Any]", attr)._column_strategy_attrs() + if hasattr(attr, "_column_strategy_attrs") + else (attr,) + ) + ), + ) + + # standalone functions follow. docstrings are filled in # by the ``@loader_unbound_fn`` decorator. @@ -2407,6 +2442,7 @@ def contains_eager(*keys: _AttrType, **kw: Any) -> _AbstractLoad: def load_only(*attrs: _AttrType, raiseload: bool = False) -> _AbstractLoad: # TODO: attrs against different classes. we likely have to # add some extra state to Load of some kind + attrs = _expand_column_strategy_attrs(attrs) _, lead_element, _ = _parse_attr_argument(attrs[0]) return Load(lead_element).load_only(*attrs, raiseload=raiseload) @@ -2423,10 +2459,15 @@ def subqueryload(*keys: _AttrType) -> _AbstractLoad: @loader_unbound_fn def selectinload( - *keys: _AttrType, recursion_depth: Optional[int] = None + *keys: _AttrType, + recursion_depth: Optional[int] = None, + chunksize: Optional[int] = None, ) -> _AbstractLoad: return _generate_from_keys( - Load.selectinload, keys, False, {"recursion_depth": recursion_depth} + Load.selectinload, + keys, + False, + {"recursion_depth": recursion_depth, "chunksize": chunksize}, ) @@ -2460,35 +2501,18 @@ def defaultload(*keys: _AttrType) -> _AbstractLoad: @loader_unbound_fn -def defer( - key: _AttrType, *addl_attrs: _AttrType, raiseload: bool = False -) -> _AbstractLoad: - if addl_attrs: - util.warn_deprecated( - "The *addl_attrs on orm.defer is deprecated. Please use " - "method chaining in conjunction with defaultload() to " - "indicate a path.", - version="1.3", - ) - +def defer(key: _AttrType, *, raiseload: bool = False) -> _AbstractLoad: if raiseload: kw = {"raiseload": raiseload} else: kw = {} - return _generate_from_keys(Load.defer, (key,) + addl_attrs, False, kw) + return _generate_from_keys(Load.defer, (key,), False, kw) @loader_unbound_fn -def undefer(key: _AttrType, *addl_attrs: _AttrType) -> _AbstractLoad: - if addl_attrs: - util.warn_deprecated( - "The *addl_attrs on orm.undefer is deprecated. Please use " - "method chaining in conjunction with defaultload() to " - "indicate a path.", - version="1.3", - ) - return _generate_from_keys(Load.undefer, (key,) + addl_attrs, False, {}) +def undefer(key: _AttrType) -> _AbstractLoad: + return _generate_from_keys(Load.undefer, (key,), False, {}) @loader_unbound_fn diff --git a/lib/sqlalchemy/orm/sync.py b/lib/sqlalchemy/orm/sync.py index 06a1948674b..678d4e53bc4 100644 --- a/lib/sqlalchemy/orm/sync.py +++ b/lib/sqlalchemy/orm/sync.py @@ -1,5 +1,5 @@ # orm/sync.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/orm/unitofwork.py b/lib/sqlalchemy/orm/unitofwork.py index d057f1746ae..3dfbc58ebd1 100644 --- a/lib/sqlalchemy/orm/unitofwork.py +++ b/lib/sqlalchemy/orm/unitofwork.py @@ -1,5 +1,5 @@ # orm/unitofwork.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/orm/util.py b/lib/sqlalchemy/orm/util.py index 81233f6554d..cdf47741691 100644 --- a/lib/sqlalchemy/orm/util.py +++ b/lib/sqlalchemy/orm/util.py @@ -1,5 +1,5 @@ # orm/util.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -20,9 +20,11 @@ from typing import Dict from typing import FrozenSet from typing import Generic +from typing import get_origin from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import Match from typing import Optional from typing import Protocol @@ -35,7 +37,7 @@ import weakref from . import attributes # noqa -from . import exc +from . import exc as orm_exc from ._typing import _O from ._typing import insp_is_aliased_class from ._typing import insp_is_mapper @@ -79,7 +81,7 @@ from ..sql import visitors from ..sql._typing import is_selectable from ..sql.annotation import SupportsCloneAnnotations -from ..sql.base import ColumnCollection +from ..sql.base import WriteableColumnCollection from ..sql.cache_key import HasCacheKey from ..sql.cache_key import MemoizedHasCacheKey from ..sql.elements import ColumnElement @@ -89,9 +91,8 @@ from ..util.typing import de_stringify_annotation as _de_stringify_annotation from ..util.typing import eval_name_only as _eval_name_only from ..util.typing import fixup_container_fwd_refs -from ..util.typing import get_origin +from ..util.typing import GenericProtocol from ..util.typing import is_origin_of_cls -from ..util.typing import Literal from ..util.typing import TupleAny from ..util.typing import Unpack @@ -122,6 +123,7 @@ from ..sql.selectable import Selectable from ..sql.visitors import anon_map from ..util.typing import _AnnotationScanType + from ..util.typing import _MatchedOnType _T = TypeVar("_T", bound=Any) @@ -162,7 +164,7 @@ def __call__( *, str_cleanup_fn: Optional[Callable[[str, str], str]] = None, include_generic: bool = False, - ) -> Type[Any]: ... + ) -> _MatchedOnType: ... de_stringify_annotation = cast( @@ -423,9 +425,6 @@ def identity_key( :param ident: primary key, may be a scalar or tuple argument. :param identity_token: optional identity token - .. versionadded:: 1.2 added identity_token - - * ``identity_key(instance=instance)`` This form will produce the identity key for a given instance. The @@ -462,8 +461,6 @@ def identity_key( (must be given as a keyword arg) :param identity_token: optional identity token - .. versionadded:: 1.2 added identity_token - """ # noqa: E501 if class_ is not None: mapper = class_mapper(class_) @@ -1023,7 +1020,10 @@ def _alias_factory( if name: return element.alias(name=name, flat=flat) else: - return coercions.expect( + # see selectable.py->Alias._factory() for similar + # mypy issue. Cannot get the overload to see this + # in mypy (works fine in pyright) + return coercions.expect( # type: ignore[no-any-return] roles.AnonymizedFromClauseRole, element, flat=flat ) else: @@ -1191,14 +1191,27 @@ def _adapt_element( if key: d["proxy_key"] = key - # IMO mypy should see this one also as returning the same type - # we put into it, but it's not - return ( - self._adapter.traverse(expr) - ._annotate(d) - ._set_propagate_attrs( - {"compile_state_plugin": "orm", "plugin_subject": self} - ) + # userspace adapt of an attribute from AliasedClass; validate that + # it actually was present + adapted = self._adapter.adapt_check_present(expr) + if adapted is None: + adapted = expr + if self._adapter.adapt_on_names: + util.warn_limited( + "Did not locate an expression in selectable for " + "attribute %r; ensure name is correct in expression", + (key,), + ) + else: + util.warn_limited( + "Did not locate an expression in selectable for " + "attribute %r; to match by name, use the " + "adapt_on_names parameter", + (key,), + ) + + return adapted._annotate(d)._set_propagate_attrs( + {"compile_state_plugin": "orm", "plugin_subject": self} ) if TYPE_CHECKING: @@ -1253,7 +1266,7 @@ def _memoized_attr__all_column_expressions(self): (key, self._adapt_element(col)) for key, col in cols_plus_keys ] - return ColumnCollection(cols_plus_keys) + return WriteableColumnCollection(cols_plus_keys) def _memo(self, key, callable_, *args, **kw): if key in self._memoized_values: @@ -1508,7 +1521,7 @@ def _inspect_mc( if class_manager is None or not class_manager.is_mapped: return None mapper = class_manager.mapper - except exc.NO_STATE: + except orm_exc.NO_STATE: return None else: return mapper @@ -1548,6 +1561,7 @@ class Bundle( :ref:`bundles` + :class:`.DictBundle` """ @@ -1565,13 +1579,13 @@ class Bundle( _propagate_attrs: _PropagateAttrsType = util.immutabledict() - proxy_set = util.EMPTY_SET # type: ignore + proxy_set = util.EMPTY_SET exprs: List[_ColumnsClauseElement] def __init__( self, name: str, *exprs: _ColumnExpressionArgument[Any], **kw: Any - ): + ) -> None: r"""Construct a new :class:`.Bundle`. e.g.:: @@ -1597,7 +1611,7 @@ def __init__( ] self.exprs = coerced_exprs - self.c = self.columns = ColumnCollection( + self.c = self.columns = WriteableColumnCollection( (getattr(col, "key", col._label), col) for col in [e._annotations.get("bundle", e) for e in coerced_exprs] ).as_readonly() @@ -1737,6 +1751,12 @@ def proc(row): for row in session.execute(select(bn)).where(bn.c.data1 == "d1"): print(row.mybundle["data1"], row.mybundle["data2"]) + The above example is available natively using :class:`.DictBundle` + + .. seealso:: + + :class:`.DictBundle` + """ # noqa: E501 keyed_tuple = result_tuple(labels, [() for l in labels]) @@ -1746,28 +1766,45 @@ def proc(row: Row[Unpack[TupleAny]]) -> Any: return proc -def _orm_annotate(element: _SA, exclude: Optional[Any] = None) -> _SA: - """Deep copy the given ClauseElement, annotating each element with the - "_orm_adapt" flag. +class DictBundle(Bundle[_T]): + """Like :class:`.Bundle` but returns ``dict`` instances instead of + named tuple like objects:: - Elements within the exclude collection will be cloned but not annotated. + bn = DictBundle("mybundle", MyClass.data1, MyClass.data2) + for row in session.execute(select(bn)).where(bn.c.data1 == "d1"): + print(row.mybundle["data1"], row.mybundle["data2"]) - """ - return sql_util._deep_annotate(element, {"_orm_adapt": True}, exclude) + Differently from :class:`.Bundle`, multiple columns with the same name are + not supported. + .. versionadded:: 2.1 -def _orm_deannotate(element: _SA) -> _SA: - """Remove annotations that link a column to a particular mapping. + .. seealso:: - Note this doesn't affect "remote" and "foreign" annotations - passed by the :func:`_orm.foreign` and :func:`_orm.remote` - annotators. + :ref:`bundles` + :class:`.Bundle` """ - return sql_util._deep_deannotate( - element, values=("_orm_adapt", "parententity") - ) + def __init__( + self, name: str, *exprs: _ColumnExpressionArgument[Any], **kw: Any + ) -> None: + super().__init__(name, *exprs, **kw) + if len(set(self.c.keys())) != len(self.c): + raise sa_exc.ArgumentError( + "DictBundle does not support duplicate column names" + ) + + def create_row_processor( + self, + query: Select[Unpack[TupleAny]], + procs: Sequence[Callable[[Row[Unpack[TupleAny]]], Any]], + labels: Sequence[str], + ) -> Callable[[Row[Unpack[TupleAny]]], dict[str, Any]]: + def proc(row: Row[Unpack[TupleAny]]) -> dict[str, Any]: + return dict(zip(labels, (proc(row) for proc in procs))) + + return proc def _orm_full_deannotate(element: _SA) -> _SA: @@ -1998,8 +2035,6 @@ def with_parent( Entity in which to consider as the left side. This defaults to the "zero" entity of the :class:`_query.Query` itself. - .. versionadded:: 1.2 - """ # noqa: E501 prop_t: RelationshipProperty[Any] @@ -2306,7 +2341,7 @@ def _extract_mapped_subtype( if raw_annotation is None: if required: - raise sa_exc.ArgumentError( + raise orm_exc.MappedAnnotationError( f"Python typing annotation is required for attribute " f'"{cls.__name__}.{key}" when primary argument(s) for ' f'"{attr_cls.__name__}" construct are None or not present' @@ -2326,14 +2361,14 @@ def _extract_mapped_subtype( str_cleanup_fn=_cleanup_mapped_str_annotation, ) except _CleanupError as ce: - raise sa_exc.ArgumentError( + raise orm_exc.MappedAnnotationError( f"Could not interpret annotation {raw_annotation}. " "Check that it uses names that are correctly imported at the " "module level. See chained stack trace for more hints." ) from ce except NameError as ne: if raiseerr and "Mapped[" in raw_annotation: # type: ignore - raise sa_exc.ArgumentError( + raise orm_exc.MappedAnnotationError( f"Could not interpret annotation {raw_annotation}. " "Check that it uses names that are correctly imported at the " "module level. See chained stack trace for more hints." @@ -2362,7 +2397,7 @@ def _extract_mapped_subtype( ): return None - raise sa_exc.ArgumentError( + raise orm_exc.MappedAnnotationError( f'Type annotation for "{cls.__name__}.{key}" ' "can't be correctly interpreted for " "Annotated Declarative Table form. ORM annotations " @@ -2382,15 +2417,16 @@ def _extract_mapped_subtype( else: return annotated, None - if len(annotated.__args__) != 1: - raise sa_exc.ArgumentError( + generic_annotated = cast(GenericProtocol[Any], annotated) + if len(generic_annotated.__args__) != 1: + raise orm_exc.MappedAnnotationError( "Expected sub-type for Mapped[] annotation" ) return ( # fix dict/list/set args to be ForwardRef, see #11814 - fixup_container_fwd_refs(annotated.__args__[0]), - annotated.__origin__, + fixup_container_fwd_refs(generic_annotated.__args__[0]), + generic_annotated.__origin__, ) diff --git a/lib/sqlalchemy/orm/writeonly.py b/lib/sqlalchemy/orm/writeonly.py index 809fdd2b0e1..3757110a9b1 100644 --- a/lib/sqlalchemy/orm/writeonly.py +++ b/lib/sqlalchemy/orm/writeonly.py @@ -1,5 +1,5 @@ # orm/writeonly.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -25,6 +25,7 @@ from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import NoReturn from typing import Optional from typing import overload @@ -39,6 +40,7 @@ from . import interfaces from . import relationships from . import strategies +from .base import ATTR_EMPTY from .base import NEVER_SET from .base import object_mapper from .base import PassiveFlag @@ -54,7 +56,6 @@ from ..sql.dml import Delete from ..sql.dml import Insert from ..sql.dml import Update -from ..util.typing import Literal if TYPE_CHECKING: from . import QueryableAttribute @@ -236,15 +237,11 @@ def get_collection( return _DynamicCollectionAdapter(data) # type: ignore[return-value] @util.memoized_property - def _append_token( # type:ignore[override] - self, - ) -> attributes.AttributeEventToken: + def _append_token(self) -> attributes.AttributeEventToken: return attributes.AttributeEventToken(self, attributes.OP_APPEND) @util.memoized_property - def _remove_token( # type:ignore[override] - self, - ) -> attributes.AttributeEventToken: + def _remove_token(self) -> attributes.AttributeEventToken: return attributes.AttributeEventToken(self, attributes.OP_REMOVE) def fire_append_event( @@ -389,6 +386,17 @@ def get_all_pending( c = self._get_collection_history(state, passive) return [(attributes.instance_state(x), x) for x in c.all_items] + def _default_value( + self, state: InstanceState[Any], dict_: _InstanceDict + ) -> Any: + value = None + for fn in self.dispatch.init_scalar: + ret = fn(state, value, dict_) + if ret is not ATTR_EMPTY: + value = ret + + return value + def _get_collection_history( self, state: InstanceState[Any], passive: PassiveFlag ) -> WriteOnlyHistory[Any]: @@ -415,7 +423,7 @@ def append( initiator: Optional[AttributeEventToken], passive: PassiveFlag = PassiveFlag.PASSIVE_NO_FETCH, ) -> None: - if initiator is not self: + if initiator is not self: # type: ignore[comparison-overlap] self.fire_append_event(state, dict_, value, initiator) def remove( @@ -426,7 +434,7 @@ def remove( initiator: Optional[AttributeEventToken], passive: PassiveFlag = PassiveFlag.PASSIVE_NO_FETCH, ) -> None: - if initiator is not self: + if initiator is not self: # type: ignore[comparison-overlap] self.fire_remove_event(state, dict_, value, initiator) def pop( @@ -523,7 +531,13 @@ def __init__( # note also, we are using the official ORM-annotated selectable # from __clause_element__(), see #7868 - self._from_obj = (prop.mapper.__clause_element__(), prop.secondary) + + # _no_filter_by annotation is to prevent this table from being + # considered by filter_by() as part of #8601 + self._from_obj = ( + prop.mapper.__clause_element__(), + prop.secondary._annotate({"_no_filter_by": True}), + ) else: self._from_obj = () diff --git a/lib/sqlalchemy/pool/__init__.py b/lib/sqlalchemy/pool/__init__.py index 8220ffad497..6b9b90e325c 100644 --- a/lib/sqlalchemy/pool/__init__.py +++ b/lib/sqlalchemy/pool/__init__.py @@ -1,5 +1,5 @@ # pool/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/pool/base.py b/lib/sqlalchemy/pool/base.py index 511eca92346..5ef01568a3d 100644 --- a/lib/sqlalchemy/pool/base.py +++ b/lib/sqlalchemy/pool/base.py @@ -1,14 +1,12 @@ # pool/base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -"""Base constructs for connection pools. - -""" +"""Base constructs for connection pools.""" from __future__ import annotations @@ -24,6 +22,7 @@ from typing import Deque from typing import Dict from typing import List +from typing import Literal from typing import Optional from typing import Protocol from typing import Tuple @@ -35,7 +34,7 @@ from .. import exc from .. import log from .. import util -from ..util.typing import Literal +from ..util.typing import Self if TYPE_CHECKING: from ..engine.interfaces import DBAPIConnection @@ -271,8 +270,6 @@ def __init__( invalidated. Requires that a dialect is passed as well to interpret the disconnection error. - .. versionadded:: 1.2 - """ if logging_name: self.logging_name = self._orig_logging_name = logging_name @@ -601,7 +598,7 @@ class ConnectionPoolEntry(ManagesConnection): connection on behalf of a :class:`_pool.Pool` instance. The :class:`.ConnectionPoolEntry` object represents the long term - maintainance of a particular connection for a pool, including expiring or + maintenance of a particular connection for a pool, including expiring or invalidating that connection to have it replaced with a new one, which will continue to be maintained by that same :class:`.ConnectionPoolEntry` instance. Compared to :class:`.PoolProxiedConnection`, which is the @@ -1075,10 +1072,12 @@ class PoolProxiedConnection(ManagesConnection): def commit(self) -> None: ... - def cursor(self) -> DBAPICursor: ... + def cursor(self, *args: Any, **kwargs: Any) -> DBAPICursor: ... def rollback(self) -> None: ... + def __getattr__(self, key: str) -> Any: ... + @property def is_valid(self) -> bool: """Return True if this :class:`.PoolProxiedConnection` still refers @@ -1124,6 +1123,13 @@ def close(self) -> None: """ raise NotImplementedError() + def __enter__(self) -> Self: + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self.close() + return None + class _AdhocProxiedConnection(PoolProxiedConnection): """provides the :class:`.PoolProxiedConnection` interface for cases where diff --git a/lib/sqlalchemy/pool/events.py b/lib/sqlalchemy/pool/events.py index 4ceb260f79b..2fd02431621 100644 --- a/lib/sqlalchemy/pool/events.py +++ b/lib/sqlalchemy/pool/events.py @@ -1,5 +1,5 @@ # pool/events.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -31,27 +31,30 @@ class PoolEvents(event.Events[Pool]): as the names of members that are passed to listener functions. - e.g.:: + When using an :class:`.Engine` object created via :func:`_sa.create_engine` + (or indirectly via :func:`.create_async_engine`), :class:`.PoolEvents` + listeners are expected to be registered in terms of the :class:`.Engine`, + which will direct the listeners to the :class:`.Pool` contained within:: + from sqlalchemy import create_engine from sqlalchemy import event + engine = create_engine("postgresql+psycopg2://scott:tiger@localhost/test") + + @event.listens_for(engine, "checkout") def my_on_checkout(dbapi_conn, connection_rec, connection_proxy): "handle an on checkout event" + :class:`.PoolEvents` may also be registered with the :class:`_pool.Pool` + class, with the :class:`.Engine` class, as well as with instances of + :class:`_pool.Pool`. - event.listen(Pool, "checkout", my_on_checkout) - - In addition to accepting the :class:`_pool.Pool` class and - :class:`_pool.Pool` instances, :class:`_events.PoolEvents` also accepts - :class:`_engine.Engine` objects and the :class:`_engine.Engine` class as - targets, which will be resolved to the ``.pool`` attribute of the - given engine or the :class:`_pool.Pool` class:: - - engine = create_engine("postgresql+psycopg2://scott:tiger@localhost/test") + .. tip:: - # will associate with engine.pool - event.listen(engine, "checkout", my_on_checkout) + Registering :class:`.PoolEvents` with the :class:`.Engine`, if present, + is recommended since the :meth:`.Engine.dispose` method will carry + along event listeners from the old pool to the new pool. """ # noqa: E501 diff --git a/lib/sqlalchemy/pool/impl.py b/lib/sqlalchemy/pool/impl.py index 44529fb1693..2807acfd7e2 100644 --- a/lib/sqlalchemy/pool/impl.py +++ b/lib/sqlalchemy/pool/impl.py @@ -1,14 +1,12 @@ # pool/impl.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -"""Pool implementation classes. - -""" +"""Pool implementation classes.""" from __future__ import annotations import threading @@ -17,6 +15,7 @@ from typing import Any from typing import cast from typing import List +from typing import Literal from typing import Optional from typing import Set from typing import Type @@ -36,7 +35,6 @@ from .. import util from ..util import chop_traceback from ..util import queue as sqla_queue -from ..util.typing import Literal if typing.TYPE_CHECKING: from ..engine.interfaces import DBAPIConnection @@ -62,7 +60,7 @@ class QueuePool(Pool): """ - _is_asyncio = False # type: ignore[assignment] + _is_asyncio = False _queue_class: Type[sqla_queue.QueueCommon[ConnectionPoolEntry]] = ( sqla_queue.Queue @@ -119,8 +117,6 @@ def __init__( timeouts, ensure that a recycle or pre-ping strategy is in use to gracefully handle stale connections. - .. versionadded:: 1.3 - .. seealso:: :ref:`pool_use_lifo` @@ -271,7 +267,7 @@ class AsyncAdaptedQueuePool(QueuePool): """ - _is_asyncio = True # type: ignore[assignment] + _is_asyncio = True _queue_class: Type[sqla_queue.QueueCommon[ConnectionPoolEntry]] = ( sqla_queue.AsyncAdaptedQueue ) @@ -354,7 +350,7 @@ class SingletonThreadPool(Pool): """ - _is_asyncio = False # type: ignore[assignment] + _is_asyncio = False def __init__( self, @@ -382,6 +378,15 @@ def recreate(self) -> SingletonThreadPool: dialect=self._dialect, ) + def _transfer_from( + self, other_singleton_pool: SingletonThreadPool + ) -> None: + # used by the test suite to make a new engine / pool without + # losing the state of an existing SQLite :memory: connection + assert not hasattr(other_singleton_pool._fairy, "current") + self._conn = other_singleton_pool._conn + self._all_conns = other_singleton_pool._all_conns + def dispose(self) -> None: """Dispose of this pool.""" diff --git a/lib/sqlalchemy/schema.py b/lib/sqlalchemy/schema.py index 32adc9bb218..98c1dc484a0 100644 --- a/lib/sqlalchemy/schema.py +++ b/lib/sqlalchemy/schema.py @@ -1,26 +1,28 @@ # schema.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php -"""Compatibility namespace for sqlalchemy.sql.schema and related. - -""" +"""Compatibility namespace for sqlalchemy.sql.schema and related.""" from __future__ import annotations +from .sql._annotated_cols import Named as Named +from .sql._annotated_cols import TypedColumns as TypedColumns from .sql.base import SchemaVisitor as SchemaVisitor from .sql.ddl import _CreateDropBase as _CreateDropBase -from .sql.ddl import _DropView as _DropView from .sql.ddl import AddConstraint as AddConstraint from .sql.ddl import BaseDDLElement as BaseDDLElement +from .sql.ddl import CheckFirst as CheckFirst from .sql.ddl import CreateColumn as CreateColumn from .sql.ddl import CreateIndex as CreateIndex from .sql.ddl import CreateSchema as CreateSchema from .sql.ddl import CreateSequence as CreateSequence from .sql.ddl import CreateTable as CreateTable +from .sql.ddl import CreateTableAs as CreateTableAs +from .sql.ddl import CreateView as CreateView from .sql.ddl import DDL as DDL from .sql.ddl import DDLElement as DDLElement from .sql.ddl import DropColumnComment as DropColumnComment @@ -31,6 +33,7 @@ from .sql.ddl import DropSequence as DropSequence from .sql.ddl import DropTable as DropTable from .sql.ddl import DropTableComment as DropTableComment +from .sql.ddl import DropView as DropView from .sql.ddl import ExecutableDDLElement as ExecutableDDLElement from .sql.ddl import InvokeDDLBase as InvokeDDLBase from .sql.ddl import SetColumnComment as SetColumnComment @@ -65,6 +68,7 @@ from .sql.schema import PrimaryKeyConstraint as PrimaryKeyConstraint from .sql.schema import SchemaConst as SchemaConst from .sql.schema import SchemaItem as SchemaItem +from .sql.schema import SchemaVisitable as SchemaVisitable from .sql.schema import Sequence as Sequence from .sql.schema import Table as Table from .sql.schema import UniqueConstraint as UniqueConstraint diff --git a/lib/sqlalchemy/sql/__init__.py b/lib/sqlalchemy/sql/__init__.py index 4ac8f343d5c..ade97f1be79 100644 --- a/lib/sqlalchemy/sql/__init__.py +++ b/lib/sqlalchemy/sql/__init__.py @@ -1,5 +1,5 @@ # sql/__init__.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -11,15 +11,18 @@ from ._typing import NotNullable as NotNullable from ._typing import Nullable as Nullable from .base import Executable as Executable +from .base import ExecutableStatement as ExecutableStatement from .base import SyntaxExtension as SyntaxExtension from .compiler import COLLECT_CARTESIAN_PRODUCTS as COLLECT_CARTESIAN_PRODUCTS from .compiler import FROM_LINTING as FROM_LINTING from .compiler import NO_LINTING as NO_LINTING from .compiler import WARN_LINTING as WARN_LINTING from .ddl import BaseDDLElement as BaseDDLElement +from .ddl import CheckFirst as CheckFirst from .ddl import DDL as DDL from .ddl import DDLElement as DDLElement from .ddl import ExecutableDDLElement as ExecutableDDLElement +from .expression import aggregate_order_by as aggregate_order_by from .expression import Alias as Alias from .expression import alias as alias from .expression import all_ as all_ @@ -47,6 +50,7 @@ from .expression import extract as extract from .expression import false as false from .expression import False_ as False_ +from .expression import from_dml_column as from_dml_column from .expression import FromClause as FromClause from .expression import func as func from .expression import funcfilter as funcfilter @@ -93,10 +97,15 @@ from .expression import TableClause as TableClause from .expression import TableSample as TableSample from .expression import tablesample as tablesample +from .expression import TableValuedAlias as TableValuedAlias +from .expression import TableValuedColumn as TableValuedColumn from .expression import text as text +from .expression import TextClause as TextClause from .expression import true as true from .expression import True_ as True_ from .expression import try_cast as try_cast +from .expression import TString as TString +from .expression import tstring as tstring from .expression import tuple_ as tuple_ from .expression import type_coerce as type_coerce from .expression import union as union @@ -106,6 +115,7 @@ from .expression import Values as Values from .expression import values as values from .expression import within_group as within_group +from .expression import WriteableColumnCollection as WriteableColumnCollection from .visitors import ClauseVisitor as ClauseVisitor diff --git a/lib/sqlalchemy/sql/_annotated_cols.py b/lib/sqlalchemy/sql/_annotated_cols.py new file mode 100644 index 00000000000..02c9456092b --- /dev/null +++ b/lib/sqlalchemy/sql/_annotated_cols.py @@ -0,0 +1,397 @@ +# sql/_annotated_cols.py +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors +# +# +# This module is part of SQLAlchemy and is released under +# the MIT License: https://www.opensource.org/licenses/mit-license.php +from __future__ import annotations + +from typing import Any +from typing import Generic +from typing import Literal +from typing import NoReturn +from typing import overload +from typing import Protocol +from typing import TYPE_CHECKING + +from . import sqltypes +from ._typing import _T +from ._typing import _Ts +from .base import _NoArg +from .base import ReadOnlyColumnCollection +from .. import util +from ..exc import ArgumentError +from ..exc import InvalidRequestError +from ..util import typing as sa_typing +from ..util.langhelpers import dunders_re +from ..util.typing import Never +from ..util.typing import Self +from ..util.typing import TypeVar +from ..util.typing import Unpack + +if TYPE_CHECKING: + from .elements import ColumnClause # noqa (for zimports) + from .elements import KeyedColumnElement # noqa (for zimports) + from .schema import Column + from .type_api import TypeEngine + from ..util.typing import _AnnotationScanType + + +class Named(Generic[_T]): + """A named descriptor that is interpreted by SQLAlchemy in various ways. + + .. seealso:: + + :class:`_schema.TypedColumns` Define table columns using this + descriptor. + + .. versionadded:: 2.1.0b2 + """ + + __slots__ = () + + key: str + if TYPE_CHECKING: + + # NOTE: this overload prevents users from using the a TypedColumns + # class like if it were an orm mapped class + @overload + def __get__(self, instance: None, owner: Any) -> Never: ... + + @overload + def __get__( + self, instance: TypedColumns, owner: Any + ) -> Column[_T]: ... + @overload + def __get__(self, instance: Any, owner: Any) -> Self: ... + + def __get__(self, instance: object | None, owner: Any) -> Any: ... + + +# NOTE: TypedColumns subclasses are ignored by the ORM mapping process +class TypedColumns(ReadOnlyColumnCollection[str, "Column[Any]"]): + """Class that generally represent the typed columns of a :class:`.Table`, + but can be used with most :class:`_sql.FromClause` subclasses with the + :meth:`_sql.FromClause.with_cols` method. + + This is a "typing only" class that is never instantiated at runtime: the + type checker will think that this class is exposed as the ``table.c`` + attribute, but in reality a normal :class:`_schema.ColumnCollection` is + used at runtime. + + Subclasses should just list the columns as class attributes, without + specifying method or other non column members. + + To resolve the columns, a simplified version of the ORM logic is used, + in particular, columns can be declared by: + + * directly instantiating them, to declare constraint, custom SQL types and + additional column options; + * using only a :class:`.Named` or :class:`_schema.Column` type annotation, + where nullability and SQL type will be inferred by the python type + provided. + Type inference is available for a common subset of python types. + * a mix of both, where the instance can be used to declare + constraints and other column options while the annotation will be used + to set the SQL type and nullability if not provided by the instance. + + In all cases the name is inferred from the attribute name, unless + explicitly provided. + + .. note:: + + The generated table will create a copy of any column instance assigned + as attributes of this class, so columns should be accessed only via + the ``table.c`` collection, not using this class directly. + + Example of the inference behavior:: + + from sqlalchemy import Column, Integer, Named, String, TypedColumns + + + class tbl_cols(TypedColumns): + # the name will be set to ``id``, type is inferred as Column[int] + id = Column(Integer, primary_key=True) + + # not null String column is generated + name: Named[str] + + # nullable Double column is generated + weight: Named[float | None] + + # nullable Integer column, with sql name 'user_age' + age: Named[int | None] = Column("user_age") + + # not null column with type String(42) + middle_name: Named[str] = Column(String(42)) + + Mixins and subclasses are also supported:: + + class with_id(TypedColumns): + id = Column(Integer, primary_key=True) + + + class named_cols(TypedColumns): + name: Named[str] + description: Named[str | None] + + + class product_cols(named_cols, with_id): + ean: Named[str] = Column(unique=True) + + + product = Table("product", metadata, product_cols) + + + class office_cols(named_cols, with_id): + address: Named[str] + + + office = Table("office", metadata, office_cols) + + The positional types returned when selecting the table can + be optionally declared by specifying a :attr:`.HasRowPos.__row_pos__` + annotation:: + + from sqlalchemy import select + + + class some_cols(TypedColumns): + id = Column(Integer, primary_key=True) + name: Named[str] + weight: Named[float | None] + + __row_pos__: tuple[int, str, float | None] + + + some_table = Table("st", metadata, some_cols) + + # both will be typed as Select[int, str, float | None] + stmt1 = some_table.select() + stmt2 = select(some_table) + + .. seealso:: + + :class:`.Table` for usage details on how to use this class to + create a table instance. + + :meth:`_sql.FromClause.with_cols` to apply a :class:`.TypedColumns` + to a from clause. + + .. versionadded:: 2.1.0b2 + """ # noqa + + __slots__ = () + + if not TYPE_CHECKING: + + def __new__(cls, *args: Any, **kwargs: Any) -> NoReturn: + raise InvalidRequestError( + "Cannot instantiate a TypedColumns object." + ) + + def __init_subclass__(cls) -> None: + methods = { + name + for name, value in cls.__dict__.items() + if not dunders_re.match(name) and callable(value) + } + if methods: + raise InvalidRequestError( + "TypedColumns subclasses may not define methods. " + f"Found {sorted(methods)}" + ) + + +_KeyColCC_co = TypeVar( + "_KeyColCC_co", + bound=ReadOnlyColumnCollection[str, "KeyedColumnElement[Any]"], + covariant=True, + default=ReadOnlyColumnCollection[str, "KeyedColumnElement[Any]"], +) +_ColClauseCC_co = TypeVar( + "_ColClauseCC_co", + bound=ReadOnlyColumnCollection[str, "ColumnClause[Any]"], + covariant=True, + default=ReadOnlyColumnCollection[str, "ColumnClause[Any]"], +) +_ColCC_co = TypeVar( + "_ColCC_co", + bound=ReadOnlyColumnCollection[str, "Column[Any]"], + covariant=True, + default=ReadOnlyColumnCollection[str, "Column[Any]"], +) + +_TC = TypeVar("_TC", bound=TypedColumns) +_TC_co = TypeVar("_TC_co", bound=TypedColumns, covariant=True) + + +class HasRowPos(Protocol[Unpack[_Ts]]): + """Protocol for a :class:`_schema.TypedColumns` used to indicate the + positional types will be returned when selecting the table. + + .. versionadded:: 2.1.0b2 + """ + + __row_pos__: tuple[Unpack[_Ts]] + """A tuple that represents the types that will be returned when + selecting from the table. + """ + + +@util.preload_module("sqlalchemy.sql.schema") +def _extract_columns_from_class( + table_columns_cls: type[TypedColumns], +) -> list[Column[Any]]: + columns: dict[str, Column[Any]] = {} + + Column = util.preloaded.sql_schema.Column + NULL_UNSPECIFIED = util.preloaded.sql_schema.NULL_UNSPECIFIED + + for base in table_columns_cls.__mro__[::-1]: + if base in TypedColumns.__mro__: + continue + + # _ClassScanAbstractConfig._cls_attr_resolver + cls_annotations = util.get_annotations(base) + cls_vars = vars(base) + items = [ + (n, cls_vars.get(n), cls_annotations.get(n)) + for n in util.merge_lists_w_ordering( + list(cls_vars), list(cls_annotations) + ) + if not dunders_re.match(n) + ] + # -- + for name, obj, annotation in items: + if obj is None: + assert annotation is not None + # no attribute, just annotation + extracted_type = _collect_annotation( + table_columns_cls, name, base.__module__, annotation + ) + if extracted_type is _NoArg.NO_ARG: + raise ArgumentError( + "No type information could be extracted from " + f"annotation {annotation} for attribute " + f"'{base.__name__}.{name}'" + ) + sqltype = _get_sqltype(extracted_type) + if sqltype is None: + raise ArgumentError( + f"Could not find a SQL type for type {extracted_type} " + f"obtained from annotation {annotation} in " + f"attribute '{base.__name__}.{name}'" + ) + columns[name] = Column( + name, + sqltype, + nullable=sa_typing.includes_none(extracted_type), + ) + elif isinstance(obj, Column): + # has attribute attribute + # _DeclarativeMapperConfig._produce_column_copies + # as with orm this case is not supported + for fk in obj.foreign_keys: + if ( + fk._table_column is not None + and fk._table_column.table is None + ): + raise InvalidRequestError( + f"Column '{base.__name__}.{name}' with foreign " + "key to non-table-bound columns is not supported " + "when using a TypedColumns. If possible use the " + "qualified string name the column" + ) + + col = obj._copy() + # MapptedColumn.declarative_scan + if col.key == col.name and col.key != name: + col.key = name + if col.name is None: + col.name = name + + sqltype = col.type + anno_sqltype = None + nullable: Literal[_NoArg.NO_ARG] | bool = _NoArg.NO_ARG + if annotation is not None: + # there is an annotation, extract the type + extracted_type = _collect_annotation( + table_columns_cls, name, base.__module__, annotation + ) + if extracted_type is not _NoArg.NO_ARG: + anno_sqltype = _get_sqltype(extracted_type) + nullable = sa_typing.includes_none(extracted_type) + + if sqltype._isnull: + if anno_sqltype is None and not col.foreign_keys: + raise ArgumentError( + "Python typing annotation is required for " + f"attribute '{base.__name__}.{name}' when " + "primary argument(s) for Column construct are " + "None or not present" + ) + elif anno_sqltype is not None: + col._set_type(anno_sqltype) + + if ( + nullable is not _NoArg.NO_ARG + and col._user_defined_nullable is NULL_UNSPECIFIED + and not col.primary_key + ): + col.nullable = nullable + columns[name] = col + else: + raise ArgumentError( + f"Unexpected value for attribute '{base.__name__}.{name}'" + f". Expected a Column, not: {type(obj)}" + ) + + # Return columns as a list + return list(columns.values()) + + +@util.preload_module("sqlalchemy.sql.schema") +def _collect_annotation( + cls: type[Any], name: str, module: str, raw_annotation: _AnnotationScanType +) -> _AnnotationScanType | Literal[_NoArg.NO_ARG]: + Column = util.preloaded.sql_schema.Column + + _locals = {"Column": Column, "Named": Named} + # _ClassScanAbstractConfig._collect_annotation & _extract_mapped_subtype + try: + annotation = sa_typing.de_stringify_annotation( + cls, raw_annotation, module, _locals + ) + except Exception as e: + raise ArgumentError( + f"Could not interpret annotation {raw_annotation} for " + f"attribute '{cls.__name__}.{name}'" + ) from e + + if ( + not sa_typing.is_generic(annotation) + and isinstance(annotation, type) + and issubclass(annotation, (Column, Named)) + ): + # no generic information, ignore + return _NoArg.NO_ARG + elif not sa_typing.is_origin_of_cls(annotation, (Column, Named)): + raise ArgumentError( + f"Annotation {raw_annotation} for attribute " + f"'{cls.__name__}.{name}' is not of type Named/Column[...]" + ) + else: + assert len(annotation.__args__) == 1 # Column[int, int] raises + return annotation.__args__[0] # type: ignore[no-any-return] + + +def _get_sqltype(annotation: _AnnotationScanType) -> TypeEngine[Any] | None: + our_type = sa_typing.de_optionalize_union_types(annotation) + # simplified version of registry._resolve_type given no customizable + # type map + sql_type = sqltypes._type_map_get(our_type) # type: ignore[arg-type] + if sql_type is not None and not sql_type._isnull: + return sqltypes.to_instance(sql_type) + else: + return None diff --git a/lib/sqlalchemy/sql/_dml_constructors.py b/lib/sqlalchemy/sql/_dml_constructors.py index 0a6f60115f1..374055cfc2d 100644 --- a/lib/sqlalchemy/sql/_dml_constructors.py +++ b/lib/sqlalchemy/sql/_dml_constructors.py @@ -1,5 +1,5 @@ # sql/_dml_constructors.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under diff --git a/lib/sqlalchemy/sql/_elements_constructors.py b/lib/sqlalchemy/sql/_elements_constructors.py index b628fcc9b52..4816d172b94 100644 --- a/lib/sqlalchemy/sql/_elements_constructors.py +++ b/lib/sqlalchemy/sql/_elements_constructors.py @@ -1,5 +1,5 @@ # sql/_elements_constructors.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -10,6 +10,7 @@ import typing from typing import Any from typing import Callable +from typing import Literal from typing import Mapping from typing import Optional from typing import overload @@ -20,9 +21,11 @@ from typing import Union from . import coercions +from . import operators from . import roles from .base import _NoArg from .coercions import _document_text_coercion +from .elements import AggregateOrderBy from .elements import BindParameter from .elements import BooleanClauseList from .elements import Case @@ -31,31 +34,37 @@ from .elements import CollectionAggregate from .elements import ColumnClause from .elements import ColumnElement +from .elements import DMLTargetCopy from .elements import Extract from .elements import False_ from .elements import FunctionFilter from .elements import Label from .elements import Null +from .elements import OrderByList from .elements import Over from .elements import TextClause from .elements import True_ from .elements import TryCast +from .elements import TString from .elements import Tuple from .elements import TypeCoerce from .elements import UnaryExpression from .elements import WithinGroup from .functions import FunctionElement -from ..util.typing import Literal if typing.TYPE_CHECKING: from ._typing import _ByArgument from ._typing import _ColumnExpressionArgument from ._typing import _ColumnExpressionOrLiteralArgument from ._typing import _ColumnExpressionOrStrLabelArgument + from ._typing import _OnlyColumnArgument from ._typing import _TypeEngineArgument + from .elements import _FrameIntTuple from .elements import BinaryExpression + from .elements import FrameClause from .selectable import FromClause from .type_api import TypeEngine + from ..util.compat import Template _T = TypeVar("_T") @@ -94,11 +103,8 @@ def all_(expr: _ColumnExpressionArgument[_T]) -> CollectionAggregate[bool]: # would render 'NULL = ALL(somearray)' all_(mytable.c.somearray) == None - .. versionchanged:: 1.4.26 repaired the use of any_() / all_() - comparing to NULL on the right side to be flipped to the left. - The column-level :meth:`_sql.ColumnElement.all_` method (not to be - confused with :class:`_types.ARRAY` level + confused with the deprecated :class:`_types.ARRAY` level :meth:`_types.ARRAY.Comparator.all`) is shorthand for ``all_(col)``:: @@ -111,7 +117,10 @@ def all_(expr: _ColumnExpressionArgument[_T]) -> CollectionAggregate[bool]: :func:`_expression.any_` """ - return CollectionAggregate._create_all(expr) + if isinstance(expr, operators.ColumnOperators): + return expr.all_() + else: + return CollectionAggregate._create_all(expr) def and_( # type: ignore[empty-body] @@ -277,11 +286,8 @@ def any_(expr: _ColumnExpressionArgument[_T]) -> CollectionAggregate[bool]: # would render 'NULL = ANY(somearray)' any_(mytable.c.somearray) == None - .. versionchanged:: 1.4.26 repaired the use of any_() / all_() - comparing to NULL on the right side to be flipped to the left. - The column-level :meth:`_sql.ColumnElement.any_` method (not to be - confused with :class:`_types.ARRAY` level + confused with the deprecated :class:`_types.ARRAY` level :meth:`_types.ARRAY.Comparator.any`) is shorthand for ``any_(col)``:: @@ -294,12 +300,27 @@ def any_(expr: _ColumnExpressionArgument[_T]) -> CollectionAggregate[bool]: :func:`_expression.all_` """ - return CollectionAggregate._create_any(expr) + if isinstance(expr, operators.ColumnOperators): + return expr.any_() + else: + return CollectionAggregate._create_any(expr) + + +@overload +def asc( + column: Union[str, "ColumnElement[_T]"], +) -> UnaryExpression[_T]: ... + + +@overload +def asc( + column: _ColumnExpressionOrStrLabelArgument[_T], +) -> Union[OrderByList, UnaryExpression[_T]]: ... def asc( column: _ColumnExpressionOrStrLabelArgument[_T], -) -> UnaryExpression[_T]: +) -> Union[OrderByList, UnaryExpression[_T]]: """Produce an ascending ``ORDER BY`` clause element. e.g.:: @@ -337,7 +358,11 @@ def asc( :meth:`_expression.Select.order_by` """ - return UnaryExpression._create_asc(column) + + if isinstance(column, operators.OrderingOperators): + return column.asc() # type: ignore[unused-ignore] + else: + return UnaryExpression._create_asc(column) def collate( @@ -358,11 +383,13 @@ def collate( The collation expression is also quoted if it is a case sensitive identifier, e.g. contains uppercase characters. - .. versionchanged:: 1.2 quoting is automatically applied to COLLATE - expressions if they are case sensitive. - """ - return CollationClause._create_collation_expression(expression, collation) + if isinstance(expression, operators.ColumnOperators): + return expression.collate(collation) # type: ignore + else: + return CollationClause._create_collation_expression( + expression, collation + ) def between( @@ -462,6 +489,41 @@ def not_(clause: _ColumnExpressionArgument[_T]) -> ColumnElement[_T]: return coercions.expect(roles.ExpressionElementRole, clause).__invert__() +def from_dml_column(column: _OnlyColumnArgument[_T]) -> DMLTargetCopy[_T]: + r"""A placeholder that may be used in compiled INSERT or UPDATE expressions + to refer to the SQL expression or value being applied to another column. + + Given a table such as:: + + t = Table( + "t", + MetaData(), + Column("x", Integer), + Column("y", Integer), + ) + + The :func:`_sql.from_dml_column` construct allows automatic copying + of an expression assigned to a different column to be reused:: + + >>> stmt = t.insert().values(x=func.foobar(3), y=from_dml_column(t.c.x) + 5) + >>> print(stmt) + INSERT INTO t (x, y) VALUES (foobar(:foobar_1), (foobar(:foobar_1) + :param_1)) + + The :func:`_sql.from_dml_column` construct is intended to be useful primarily + with event-based hooks such as those used by ORM hybrids. + + .. seealso:: + + :ref:`hybrid_bulk_update` + + .. versionadded:: 2.1 + + + """ # noqa: E501 + + return DMLTargetCopy(column) + + def bindparam( key: Optional[str], value: Any = _NoArg.NO_ARG, @@ -687,11 +749,6 @@ def bindparam( .. note:: The "expanding" feature does not support "executemany"- style parameter sets. - .. versionadded:: 1.2 - - .. versionchanged:: 1.3 the "expanding" bound parameter feature now - supports empty lists. - :param literal_execute: if True, the bound parameter will be rendered in the compile phase with a special "POSTCOMPILE" token, and the SQLAlchemy compiler will @@ -1054,9 +1111,21 @@ def column( return ColumnClause(text, type_, is_literal, _selectable) +@overload +def desc( + column: Union[str, "ColumnElement[_T]"], +) -> UnaryExpression[_T]: ... + + +@overload +def desc( + column: _ColumnExpressionOrStrLabelArgument[_T], +) -> Union[OrderByList, UnaryExpression[_T]]: ... + + def desc( column: _ColumnExpressionOrStrLabelArgument[_T], -) -> UnaryExpression[_T]: +) -> Union[OrderByList, UnaryExpression[_T]]: """Produce a descending ``ORDER BY`` clause element. e.g.:: @@ -1094,7 +1163,10 @@ def desc( :meth:`_expression.Select.order_by` """ - return UnaryExpression._create_desc(column) + if isinstance(column, operators.OrderingOperators): + return column.desc() # type: ignore[unused-ignore] + else: + return UnaryExpression._create_desc(column) def distinct(expr: _ColumnExpressionArgument[_T]) -> UnaryExpression[_T]: @@ -1143,7 +1215,10 @@ def distinct(expr: _ColumnExpressionArgument[_T]) -> UnaryExpression[_T]: :data:`.func` """ # noqa: E501 - return UnaryExpression._create_distinct(expr) + if isinstance(expr, operators.ColumnOperators): + return expr.distinct() + else: + return UnaryExpression._create_distinct(expr) def bitwise_not(expr: _ColumnExpressionArgument[_T]) -> UnaryExpression[_T]: @@ -1159,8 +1234,10 @@ def bitwise_not(expr: _ColumnExpressionArgument[_T]) -> UnaryExpression[_T]: """ - - return UnaryExpression._create_bitwise_not(expr) + if isinstance(expr, operators.ColumnOperators): + return expr.bitwise_not() + else: + return UnaryExpression._create_bitwise_not(expr) def extract(field: str, expr: _ColumnExpressionArgument[Any]) -> Extract: @@ -1307,7 +1384,21 @@ def null() -> Null: return Null._instance() -def nulls_first(column: _ColumnExpressionArgument[_T]) -> UnaryExpression[_T]: +@overload +def nulls_first( + column: "ColumnElement[_T]", +) -> UnaryExpression[_T]: ... + + +@overload +def nulls_first( + column: _ColumnExpressionArgument[_T], +) -> Union[OrderByList, UnaryExpression[_T]]: ... + + +def nulls_first( + column: _ColumnExpressionArgument[_T], +) -> Union[OrderByList, UnaryExpression[_T]]: """Produce the ``NULLS FIRST`` modifier for an ``ORDER BY`` expression. :func:`.nulls_first` is intended to modify the expression produced @@ -1350,10 +1441,27 @@ def nulls_first(column: _ColumnExpressionArgument[_T]) -> UnaryExpression[_T]: :meth:`_expression.Select.order_by` """ # noqa: E501 - return UnaryExpression._create_nulls_first(column) + if isinstance(column, operators.OrderingOperators): + return column.nulls_first() + else: + return UnaryExpression._create_nulls_first(column) + + +@overload +def nulls_last( + column: "ColumnElement[_T]", +) -> UnaryExpression[_T]: ... -def nulls_last(column: _ColumnExpressionArgument[_T]) -> UnaryExpression[_T]: +@overload +def nulls_last( + column: _ColumnExpressionArgument[_T], +) -> Union[OrderByList, UnaryExpression[_T]]: ... + + +def nulls_last( + column: _ColumnExpressionArgument[_T], +) -> Union[OrderByList, UnaryExpression[_T]]: """Produce the ``NULLS LAST`` modifier for an ``ORDER BY`` expression. :func:`.nulls_last` is intended to modify the expression produced @@ -1394,7 +1502,10 @@ def nulls_last(column: _ColumnExpressionArgument[_T]) -> UnaryExpression[_T]: :meth:`_expression.Select.order_by` """ # noqa: E501 - return UnaryExpression._create_nulls_last(column) + if isinstance(column, operators.OrderingOperators): + return column.nulls_last() + else: + return UnaryExpression._create_nulls_last(column) def or_( # type: ignore[empty-body] @@ -1504,10 +1615,12 @@ def or_(*clauses): # noqa: F811 def over( element: FunctionElement[_T], - partition_by: Optional[_ByArgument] = None, - order_by: Optional[_ByArgument] = None, - range_: Optional[typing_Tuple[Optional[int], Optional[int]]] = None, - rows: Optional[typing_Tuple[Optional[int], Optional[int]]] = None, + partition_by: _ByArgument | None = None, + order_by: _ByArgument | None = None, + range_: _FrameIntTuple | FrameClause | None = None, + rows: _FrameIntTuple | FrameClause | None = None, + groups: _FrameIntTuple | FrameClause | None = None, + exclude: str | None = None, ) -> Over[_T]: r"""Produce an :class:`.Over` object against a function. @@ -1525,8 +1638,9 @@ def over( ROW_NUMBER() OVER(ORDER BY some_column) - Ranges are also possible using the :paramref:`.expression.over.range_` - and :paramref:`.expression.over.rows` parameters. These + Ranges are also possible using the :paramref:`.expression.over.range_`, + :paramref:`.expression.over.rows`, and :paramref:`.expression.over.groups` + parameters. These mutually-exclusive parameters each accept a 2-tuple, which contains a combination of integers and None:: @@ -1559,6 +1673,30 @@ def over( func.row_number().over(order_by="x", range_=(1, 3)) + * GROUPS BETWEEN 1 FOLLOWING AND 3 FOLLOWING:: + + func.row_number().over(order_by="x", groups=(1, 3)) + + Depending on the type of the order column, the 'RANGE' value may not be + an integer. In this case use a :class:`_expression.FrameClause` directly + to specify the frame boundaries. E.g.:: + + from datetime import timedelta + from sqlalchemy import FrameClause, FrameClauseType + + func.sum(my_table.c.amount).over( + order_by=my_table.c.date, + range_=FrameClause( + start=timedelta(days=7), + end=None, + start_frame_type=FrameClauseType.PRECEDING, + end_frame_type=FrameClauseType.UNBOUNDED, + ), + ) + + .. versionchanged:: 2.1 Added support for range types that are not + integer-based, via the :class:`_expression.FrameClause` construct. + :param element: a :class:`.FunctionElement`, :class:`.WithinGroup`, or other compatible construct. :param partition_by: a column element or string, or a list @@ -1568,12 +1706,33 @@ def over( of such, that will be used as the ORDER BY clause of the OVER construct. :param range\_: optional range clause for the window. This is a - tuple value which can contain integer values or ``None``, + two-tuple value which can contain integer values or ``None``, and will render a RANGE BETWEEN PRECEDING / FOLLOWING clause. + Can also be a :class:`_expression.FrameClause` instance to + specify non-integer values. + + .. versionchanged:: 2.1 Added support for range types that are not + integer-based, via the :class:`_expression.FrameClause` construct. - :param rows: optional rows clause for the window. This is a tuple + :param rows: optional rows clause for the window. This is a two-tuple value which can contain integer values or None, and will render - a ROWS BETWEEN PRECEDING / FOLLOWING clause. + a ROWS BETWEEN PRECEDING / FOLLOWING clause. Can also be a + :class:`_expression.FrameClause` instance. + :param groups: optional groups clause for the window. This is a + two-tuple value which can contain integer values or ``None``, + and will render a GROUPS BETWEEN PRECEDING / FOLLOWING clause. + Can also be a :class:`_expression.FrameClause` instance. + + .. versionadded:: 2.0.40 + + :param exclude: optional string for the frame exclusion clause. This is a + string value which can be one of ``CURRENT ROW``, ``GROUP``, ``TIES``, or + ``NO OTHERS`` and will render an EXCLUDE clause within the window frame + specification. Requires that one of :paramref:`_sql.over.rows`, + :paramref:`_sql.over.range_`, or :paramref:`_sql.over.groups` is also + specified. + + .. versionadded:: 2.1 This function is also available from the :data:`~.expression.func` construct itself via the :meth:`.FunctionElement.over` method. @@ -1587,7 +1746,15 @@ def over( :func:`_expression.within_group` """ # noqa: E501 - return Over(element, partition_by, order_by, range_, rows) + return Over( + element, + partition_by, + order_by, + range_, + rows, + groups, + exclude, + ) @_document_text_coercion("text", ":func:`.text`", ":paramref:`.text.text`") @@ -1670,6 +1837,86 @@ def text(text: str) -> TextClause: return TextClause(text) +def tstring(template: Template) -> TString: + r"""Construct a new :class:`_expression.TString` clause, + representing a SQL template string using Python 3.14+ t-strings. + + .. versionadded:: 2.1 + + E.g.:: + + from sqlalchemy import tstring + + a = 5 + b = 10 + stmt = tstring(t"select {a}, {b}") + result = connection.execute(stmt) + + The :func:`_expression.tstring` function accepts a Python 3.14+ + template string (t-string) and processes it to create a SQL statement. + Unlike :func:`_expression.text`, which requires manual bind parameter + specification, :func:`_expression.tstring` automatically handles + interpolation of Python values and SQLAlchemy expressions. + + **Interpolation Behavior**: + + - **SQL content** expressed in the plain string portions of the template + are rendered directly as SQL + - **SQLAlchemy expressions** (columns, functions, etc.) are embedded + as clause elements + - **Plain Python values** are automatically wrapped in + :func:`_expression.literal` + + For example:: + + from sqlalchemy import tstring, select, literal, JSON, table, column + + # Python values become bound parameters + user_id = 42 + stmt = tstring(t"SELECT * FROM users WHERE id = {user_id}") + # renders: SELECT * FROM users WHERE id = :param_1 + + # SQLAlchemy expressions are embedded + stmt = tstring(t"SELECT {column('q')} FROM {table('t')}") + # renders: SELECT q FROM t + + # Apply explicit SQL types to bound values using literal() + some_json = {"foo": "bar"} + stmt = tstring(t"SELECT {literal(some_json, JSON)}") + + **Column Specification**: + + Like :func:`_expression.text`, the :func:`_expression.tstring` construct + supports the :meth:`_expression.TString.columns` method to specify + return columns and their types:: + + from sqlalchemy import tstring, column, Integer, String + + stmt = tstring(t"SELECT id, name FROM users").columns( + column("id", Integer), column("name", String) + ) + + for id, name in connection.execute(stmt): + print(id, name) + + :param template: + a Python 3.14+ template string (t-string) containing SQL fragments + and Python expressions to be interpolated. + + .. seealso:: + + :ref:`tutorial_select_arbitrary_text` - in the :ref:`unified_tutorial` + + :class:`_expression.TString` + + :func:`_expression.text` + + `PEP 750 `_ - Template Strings + + """ + return TString(template) + + def true() -> True_: """Return a constant :class:`.True_` construct. @@ -1711,7 +1958,7 @@ def true() -> True_: def tuple_( - *clauses: _ColumnExpressionArgument[Any], + *clauses: _ColumnExpressionOrLiteralArgument[Any], types: Optional[Sequence[_TypeEngineArgument[Any]]] = None, ) -> Tuple: """Return a :class:`.Tuple`. @@ -1723,8 +1970,6 @@ def tuple_( tuple_(table.c.col1, table.c.col2).in_([(1, 2), (5, 12), (10, 19)]) - .. versionchanged:: 1.3.6 Added support for SQLite IN tuples. - .. warning:: The composite IN construct is not supported by all backends, and is @@ -1828,20 +2073,24 @@ def within_group( Used against so-called "ordered set aggregate" and "hypothetical set aggregate" functions, including :class:`.percentile_cont`, - :class:`.rank`, :class:`.dense_rank`, etc. + :class:`.rank`, :class:`.dense_rank`, etc. This feature is typically + used by Oracle Database, Microsoft SQL Server. + + For generalized ORDER BY of aggregate functions on all included + backends, including PostgreSQL, MySQL/MariaDB, SQLite as well as Oracle + and SQL Server, the :func:`_sql.aggregate_order_by` provides a more + general approach that compiles to "WITHIN GROUP" only on those backends + which require it. :func:`_expression.within_group` is usually called using the :meth:`.FunctionElement.within_group` method, e.g.:: - from sqlalchemy import within_group - stmt = select( - department.c.id, func.percentile_cont(0.5).within_group(department.c.salary.desc()), ) The above statement would produce SQL similar to - ``SELECT department.id, percentile_cont(0.5) + ``SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY department.salary DESC)``. :param element: a :class:`.FunctionElement` construct, typically @@ -1854,9 +2103,62 @@ def within_group( :ref:`tutorial_functions_within_group` - in the :ref:`unified_tutorial` + :func:`_sql.aggregate_order_by` - helper for PostgreSQL, MySQL, + SQLite aggregate functions + :data:`.expression.func` :func:`_expression.over` """ return WithinGroup(element, *order_by) + + +def aggregate_order_by( + element: FunctionElement[_T], *order_by: _ColumnExpressionArgument[Any] +) -> AggregateOrderBy[_T]: + r"""Produce a :class:`.AggregateOrderBy` object against a function. + + Used for aggregating functions such as :class:`_functions.array_agg`, + ``group_concat``, ``json_agg`` on backends that support ordering via an + embedded ``ORDER BY`` parameter, e.g. PostgreSQL, MySQL/MariaDB, SQLite. + When used on backends like Oracle and SQL Server, SQL compilation uses that + of :class:`.WithinGroup`. On PostgreSQL, compilation is fixed at embedded + ``ORDER BY``; for set aggregation functions where PostgreSQL requires the + use of ``WITHIN GROUP``, :func:`_expression.within_group` should be used + explicitly. + + :func:`_expression.aggregate_order_by` is usually called using + the :meth:`.FunctionElement.aggregate_order_by` method, e.g.:: + + stmt = select( + func.array_agg(department.c.code).aggregate_order_by( + department.c.code.desc() + ), + ) + + which would produce an expression resembling: + + .. sourcecode:: sql + + SELECT array_agg(department.code ORDER BY department.code DESC) + AS array_agg_1 FROM department + + The ORDER BY argument may also be multiple terms. + + When using the backend-agnostic :class:`_functions.aggregate_strings` + string aggregation function, use the + :paramref:`_functions.aggregate_strings.order_by` parameter to indicate a + dialect-agnostic ORDER BY expression. + + .. versionadded:: 2.1 Generalized the PostgreSQL-specific + :func:`_postgresql.aggregate_order_by` function to a method on + :class:`.Function` that is backend agnostic. + + .. seealso:: + + :class:`_functions.aggregate_strings` - backend-agnostic string + concatenation function which also supports ORDER BY + + """ # noqa: E501 + return AggregateOrderBy(element, *order_by) diff --git a/lib/sqlalchemy/sql/_orm_types.py b/lib/sqlalchemy/sql/_orm_types.py index c37d805ef3f..06ea37900f2 100644 --- a/lib/sqlalchemy/sql/_orm_types.py +++ b/lib/sqlalchemy/sql/_orm_types.py @@ -1,5 +1,5 @@ # sql/_orm_types.py -# Copyright (C) 2022-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2022-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -14,7 +14,7 @@ from __future__ import annotations -from ..util.typing import Literal +from typing import Literal SynchronizeSessionArgument = Literal[False, "auto", "evaluate", "fetch"] DMLStrategyArgument = Literal["bulk", "raw", "orm", "auto"] diff --git a/lib/sqlalchemy/sql/_selectable_constructors.py b/lib/sqlalchemy/sql/_selectable_constructors.py index 08149771b16..d03b925ed76 100644 --- a/lib/sqlalchemy/sql/_selectable_constructors.py +++ b/lib/sqlalchemy/sql/_selectable_constructors.py @@ -1,5 +1,5 @@ # sql/_selectable_constructors.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -15,6 +15,8 @@ from . import coercions from . import roles +from ._annotated_cols import _KeyColCC_co +from ._annotated_cols import HasRowPos from ._typing import _ColumnsClauseArgument from ._typing import _no_kw from .elements import ColumnClause @@ -36,6 +38,7 @@ if TYPE_CHECKING: from ._typing import _FromClauseArgument from ._typing import _OnClauseArgument + from ._typing import _OnlyColumnArgument from ._typing import _SelectStatementForCompoundArgument from ._typing import _T0 from ._typing import _T1 @@ -48,6 +51,7 @@ from ._typing import _T8 from ._typing import _T9 from ._typing import _Ts + from ._typing import _Ts2 from ._typing import _TypedColumnClauseArgument as _TCCA from .functions import Function from .selectable import CTE @@ -57,8 +61,10 @@ def alias( - selectable: FromClause, name: Optional[str] = None, flat: bool = False -) -> NamedFromClause: + selectable: FromClause[_KeyColCC_co], + name: Optional[str] = None, + flat: bool = False, +) -> NamedFromClause[_KeyColCC_co]: """Return a named alias of the given :class:`.FromClause`. For :class:`.Table` and :class:`.Join` objects, the return type is the @@ -495,6 +501,68 @@ def select( # END OVERLOADED FUNCTIONS select +@overload +def select( + __table: FromClause[HasRowPos[Unpack[_Ts]]], # type: ignore[type-var] +) -> Select[Unpack[_Ts]]: ... + + +# NOTE: this seems to currently be interpreted by mypy as not allowed. +# https://peps.python.org/pep-0646/#multiple-type-variable-tuples-not-allowed +# https://github.com/python/mypy/issues/20188 +@overload +def select( + __table: FromClause[HasRowPos[Unpack[_Ts]]], # type: ignore[type-var] + __table2: FromClause[HasRowPos[Unpack[_Ts2]]], # type: ignore[type-var] +) -> Select[Unpack[_Ts], Unpack[_Ts2]]: ... # type: ignore[misc] + + +@overload +def select( + __table: FromClause[HasRowPos[Unpack[_Ts]]], # type: ignore[type-var] + __ent0: _TCCA[_T0], +) -> Select[Unpack[_Ts], _T0]: ... + + +@overload +def select( + __table: FromClause[HasRowPos[Unpack[_Ts]]], # type: ignore[type-var] + __ent0: _TCCA[_T0], + __ent1: _TCCA[_T1], +) -> Select[Unpack[_Ts], _T0, _T1]: ... + + +@overload +def select( + __table: FromClause[HasRowPos[Unpack[_Ts]]], # type: ignore[type-var] + __ent0: _TCCA[_T0], + __ent1: _TCCA[_T1], + __ent2: _TCCA[_T2], +) -> Select[Unpack[_Ts], _T0, _T1, _T2]: ... + + +@overload +def select( + __ent0: _TCCA[_T0], + __table: FromClause[HasRowPos[Unpack[_Ts]]], # type: ignore[type-var] +) -> Select[_T0, Unpack[_Ts]]: ... + + +@overload +def select( + __ent0: _TCCA[_T0], + __ent1: _TCCA[_T1], + __table: FromClause[HasRowPos[Unpack[_Ts]]], # type: ignore[type-var] +) -> Select[_T0, _T1, Unpack[_Ts]]: ... + + +@overload +def select( + __ent0: _TCCA[_T0], + __ent1: _TCCA[_T1], + __ent2: _TCCA[_T2], + __table: FromClause[HasRowPos[Unpack[_Ts]]], # type: ignore[type-var] +) -> Select[_T0, _T1, _T2, Unpack[_Ts]]: ... @overload @@ -564,8 +632,6 @@ def table(name: str, *columns: ColumnClause[Any], **kw: Any) -> TableClause: :param schema: The schema name for this table. - .. versionadded:: 1.3.18 :func:`_expression.table` can now - accept a ``schema`` argument. """ return TableClause(name, *columns, **kw) @@ -686,30 +752,78 @@ def union_all( def values( - *columns: ColumnClause[Any], + *columns: _OnlyColumnArgument[Any], name: Optional[str] = None, literal_binds: bool = False, ) -> Values: - r"""Construct a :class:`_expression.Values` construct. + r"""Construct a :class:`_expression.Values` construct representing the + SQL ``VALUES`` clause. - The column expressions and the actual data for - :class:`_expression.Values` are given in two separate steps. The - constructor receives the column expressions typically as - :func:`_expression.column` constructs, - and the data is then passed via the - :meth:`_expression.Values.data` method as a list, - which can be called multiple - times to add more data, e.g.:: + The column expressions and the actual data for :class:`_expression.Values` + are given in two separate steps. The constructor receives the column + expressions typically as :func:`_expression.column` constructs, and the + data is then passed via the :meth:`_expression.Values.data` method as a + list, which can be called multiple times to add more data, e.g.:: from sqlalchemy import column from sqlalchemy import values + from sqlalchemy import Integer + from sqlalchemy import String + + value_expr = ( + values( + column("id", Integer), + column("name", String), + ) + .data([(1, "name1"), (2, "name2")]) + .data([(3, "name3")]) + ) + + Would represent a SQL fragment like:: + + VALUES(1, "name1"), (2, "name2"), (3, "name3") + + The :class:`_sql.values` construct has an optional + :paramref:`_sql.values.name` field; when using this field, the + PostgreSQL-specific "named VALUES" clause may be generated:: value_expr = values( - column("id", Integer), - column("name", String), - name="my_values", + column("id", Integer), column("name", String), name="somename" ).data([(1, "name1"), (2, "name2"), (3, "name3")]) + When selecting from the above construct, the name and column names will + be listed out using a PostgreSQL-specific syntax:: + + >>> print(value_expr.select()) + SELECT somename.id, somename.name + FROM (VALUES (:param_1, :param_2), (:param_3, :param_4), + (:param_5, :param_6)) AS somename (id, name) + + For a more database-agnostic means of SELECTing named columns from a + VALUES expression, the :meth:`.Values.cte` method may be used, which + produces a named CTE with explicit column names against the VALUES + construct within; this syntax works on PostgreSQL, SQLite, and MariaDB:: + + value_expr = ( + values( + column("id", Integer), + column("name", String), + ) + .data([(1, "name1"), (2, "name2"), (3, "name3")]) + .cte() + ) + + Rendering as:: + + >>> print(value_expr.select()) + WITH anon_1(id, name) AS + (VALUES (:param_1, :param_2), (:param_3, :param_4), (:param_5, :param_6)) + SELECT anon_1.id, anon_1.name + FROM anon_1 + + .. versionadded:: 2.0.42 Added the :meth:`.Values.cte` method to + :class:`.Values` + :param \*columns: column expressions, typically composed using :func:`_expression.column` objects. @@ -721,5 +835,6 @@ def values( the data values inline in the SQL output, rather than using bound parameters. - """ + """ # noqa: E501 + return Values(*columns, literal_binds=literal_binds, name=name) diff --git a/lib/sqlalchemy/sql/_typing.py b/lib/sqlalchemy/sql/_typing.py index 6fef1766c6d..492601e5544 100644 --- a/lib/sqlalchemy/sql/_typing.py +++ b/lib/sqlalchemy/sql/_typing.py @@ -1,5 +1,5 @@ # sql/_typing.py -# Copyright (C) 2022-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2022-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -13,6 +13,7 @@ from typing import Dict from typing import Generic from typing import Iterable +from typing import Literal from typing import Mapping from typing import NoReturn from typing import Optional @@ -21,6 +22,7 @@ from typing import Set from typing import Type from typing import TYPE_CHECKING +from typing import TypeAlias from typing import TypeVar from typing import Union @@ -28,9 +30,7 @@ from .. import exc from .. import util from ..inspection import Inspectable -from ..util.typing import Literal from ..util.typing import TupleAny -from ..util.typing import TypeAlias from ..util.typing import TypeVarTuple from ..util.typing import Unpack @@ -40,6 +40,7 @@ from datetime import time from datetime import timedelta from decimal import Decimal + from typing import TypeGuard from uuid import UUID from .base import Executable @@ -72,12 +73,15 @@ from .sqltypes import TableValueType from .sqltypes import TupleType from .type_api import TypeEngine + from ..engine import Connection from ..engine import Dialect - from ..util.typing import TypeGuard + from ..engine import Engine + from ..engine.mock import MockConnection _T = TypeVar("_T", bound=Any) _T_co = TypeVar("_T_co", bound=Any, covariant=True) _Ts = TypeVarTuple("_Ts") +_Ts2 = TypeVarTuple("_Ts2") _CE = TypeVar("_CE", bound="ColumnElement[Any]") @@ -184,6 +188,19 @@ def dialect(self) -> Dialect: ... _T9 = TypeVar("_T9", bound=Any) +_OnlyColumnArgument = Union[ + "ColumnElement[_T]", + _HasClauseElement[_T], + roles.DMLColumnRole, +] +"""A narrow type that is looking for a ColumnClause (e.g. table column with a +name) or an ORM element that produces this. + +This is used for constructs that need a named column to represent a +position in a selectable, like TextClause().columns() or values(...). + +""" + _ColumnExpressionArgument = Union[ "ColumnElement[_T]", _HasClauseElement[_T], @@ -226,6 +243,7 @@ def dialect(self) -> Dialect: ... _FromClauseArgument = Union[ roles.FromClauseRole, + roles.TypedColumnsClauseRole[Any], Type[Any], Inspectable[_HasClauseElement[Any]], _HasClauseElement[Any], @@ -271,6 +289,7 @@ def dialect(self) -> Dialect: ... """ + _DMLKey = TypeVar("_DMLKey", bound=_DMLColumnArgument) _DMLColumnKeyMapping = Mapping[_DMLKey, Any] @@ -304,6 +323,8 @@ def dialect(self) -> Dialect: ... _AutoIncrementType = Union[bool, Literal["auto", "ignore_fk"]] +_CreateDropBind = Union["Engine", "Connection", "MockConnection"] + if TYPE_CHECKING: def is_sql_compiler(c: Compiled) -> TypeGuard[SQLCompiler]: ... @@ -335,11 +356,11 @@ def is_table_value_type( def is_selectable(t: Any) -> TypeGuard[Selectable]: ... def is_select_base( - t: Union[Executable, ReturnsRows] + t: Union[Executable, ReturnsRows], ) -> TypeGuard[SelectBase]: ... def is_select_statement( - t: Union[Executable, ReturnsRows] + t: Union[Executable, ReturnsRows], ) -> TypeGuard[Select[Unpack[TupleAny]]]: ... def is_table(t: FromClause) -> TypeGuard[TableClause]: ... diff --git a/lib/sqlalchemy/sql/_util_cy.py b/lib/sqlalchemy/sql/_util_cy.py index 101d1d102ed..7272bfefc38 100644 --- a/lib/sqlalchemy/sql/_util_cy.py +++ b/lib/sqlalchemy/sql/_util_cy.py @@ -1,21 +1,22 @@ # sql/_util_cy.py -# Copyright (C) 2010-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2010-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php +# mypy: disable-error-code="untyped-decorator" from __future__ import annotations from typing import Dict +from typing import Literal from typing import Tuple from typing import TYPE_CHECKING from typing import Union -from ..util.typing import Literal - if TYPE_CHECKING: from .cache_key import CacheConst + from ..engine.interfaces import _CoreSingleExecuteParams # START GENERATED CYTHON IMPORT # This section is automatically generated by the script tools/cython_imports.py @@ -30,7 +31,7 @@ def _is_compiled() -> bool: """Utility function to indicate if this module is compiled or not.""" - return cython.compiled # type: ignore[no-any-return] + return cython.compiled # type: ignore[no-any-return,unused-ignore] # END GENERATED CYTHON IMPORT @@ -68,13 +69,12 @@ def __missing__(self, key: str, /) -> str: return value +_AM_KEY = Union[int, str, "CacheConst"] +_AM_VALUE = Union[int, Literal[True], "_CoreSingleExecuteParams"] + + @cython.cclass -class anon_map( - Dict[ - Union[int, str, "Literal[CacheConst.NO_CACHE]"], - Union[int, Literal[True]], - ] -): +class anon_map(Dict[_AM_KEY, _AM_VALUE]): """A map that creates new keys for missing key access. Produces an incrementing sequence given a series of unique keys. @@ -95,11 +95,9 @@ def __cinit__(self): # type: ignore[no-untyped-def] else: _index: int = 0 # type: ignore[no-redef] - @cython.cfunc # type:ignore[misc] - @cython.inline # type:ignore[misc] - def _add_missing( - self: anon_map, key: Union[int, str, "Literal[CacheConst.NO_CACHE]"], / - ) -> int: + @cython.cfunc + @cython.inline + def _add_missing(self: anon_map, key: _AM_KEY, /) -> int: val: int = self._index self._index += 1 self_dict: dict = self # type: ignore[type-arg] @@ -117,11 +115,7 @@ def get_anon(self: anon_map, obj: object, /) -> Tuple[int, bool]: if cython.compiled: - def __getitem__( - self: anon_map, - key: Union[int, str, "Literal[CacheConst.NO_CACHE]"], - /, - ) -> Union[int, Literal[True]]: + def __getitem__(self: anon_map, key: _AM_KEY, /) -> _AM_VALUE: self_dict: dict = self # type: ignore[type-arg] if key in self_dict: @@ -129,7 +123,5 @@ def __getitem__( else: return self._add_missing(key) # type:ignore[no-any-return] - def __missing__( - self: anon_map, key: Union[int, str, "Literal[CacheConst.NO_CACHE]"], / - ) -> int: + def __missing__(self: anon_map, key: _AM_KEY, /) -> int: return self._add_missing(key) # type:ignore[no-any-return] diff --git a/lib/sqlalchemy/sql/annotation.py b/lib/sqlalchemy/sql/annotation.py index 0fb2390c11e..a66880868f6 100644 --- a/lib/sqlalchemy/sql/annotation.py +++ b/lib/sqlalchemy/sql/annotation.py @@ -1,5 +1,5 @@ # sql/annotation.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -24,6 +24,7 @@ from typing import cast from typing import Dict from typing import FrozenSet +from typing import Literal from typing import Mapping from typing import Optional from typing import overload @@ -39,7 +40,6 @@ from .visitors import ExternallyTraversible from .visitors import InternalTraversal from .. import util -from ..util.typing import Literal from ..util.typing import Self if TYPE_CHECKING: @@ -240,7 +240,7 @@ def _deannotate( # clone is used when we are also copying # the expression for a deep deannotation new = self._clone() - new._annotations = util.immutabledict() + new._annotations = util.EMPTY_DICT new.__dict__.pop("_annotations_cache_key", None) return new else: @@ -468,7 +468,9 @@ def clone(elem: SupportsAnnotations, **kw: Any) -> SupportsAnnotations: newelem = elem newelem._copy_internals( - clone=clone, ind_cols_on_fromclause=ind_cols_on_fromclause + clone=clone, + ind_cols_on_fromclause=ind_cols_on_fromclause, + _annotations_traversal=True, ) cloned_ids[id_] = newelem @@ -508,7 +510,7 @@ def clone(elem: SupportsAnnotations, **kw: Any) -> SupportsAnnotations: if key not in cloned: newelem = elem._deannotate(values=values, clone=True) - newelem._copy_internals(clone=clone) + newelem._copy_internals(clone=clone, _annotations_traversal=True) cloned[key] = newelem return newelem else: @@ -529,7 +531,7 @@ def _shallow_annotate(element: _SA, annotations: _AnnotationDict) -> _SA: structure wasting time. """ element = element._annotate(annotations) - element._copy_internals() + element._copy_internals(_annotations_traversal=True) return element diff --git a/lib/sqlalchemy/sql/base.py b/lib/sqlalchemy/sql/base.py index ee4037a2ffc..56bde75a127 100644 --- a/lib/sqlalchemy/sql/base.py +++ b/lib/sqlalchemy/sql/base.py @@ -1,14 +1,12 @@ # sql/base.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under # the MIT License: https://www.opensource.org/licenses/mit-license.php # mypy: allow-untyped-defs, allow-untyped-calls -"""Foundational utilities common to many sql modules. - -""" +"""Foundational utilities common to many sql modules.""" from __future__ import annotations @@ -22,8 +20,11 @@ from typing import Any from typing import Callable from typing import cast +from typing import Collection from typing import Dict +from typing import Final from typing import FrozenSet +from typing import Generator from typing import Generic from typing import Iterable from typing import Iterator @@ -40,6 +41,7 @@ from typing import Tuple from typing import Type from typing import TYPE_CHECKING +from typing import TypeGuard from typing import TypeVar from typing import Union @@ -55,10 +57,11 @@ from .. import event from .. import exc from .. import util +from ..util import EMPTY_DICT from ..util import HasMemoized as HasMemoized from ..util import hybridmethod +from ..util import warn_deprecated from ..util.typing import Self -from ..util.typing import TypeGuard from ..util.typing import TypeVarTuple from ..util.typing import Unpack @@ -69,6 +72,7 @@ from ._orm_types import DMLStrategyArgument from ._orm_types import SynchronizeSessionArgument from ._typing import _CLE + from .cache_key import CacheKey from .compiler import SQLCompiler from .dml import Delete from .dml import Insert @@ -87,9 +91,11 @@ from .selectable import _SelectIterable from .selectable import FromClause from .selectable import Select + from .visitors import anon_map from ..engine import Connection from ..engine import CursorResult from ..engine.interfaces import _CoreMultiExecuteParams + from ..engine.interfaces import _CoreSingleExecuteParams from ..engine.interfaces import _ExecuteOptions from ..engine.interfaces import _ImmutableExecuteOptions from ..engine.interfaces import CacheStats @@ -117,7 +123,7 @@ def __repr__(self): return f"_NoArg.{self.name}" -NO_ARG = _NoArg.NO_ARG +NO_ARG: Final = _NoArg.NO_ARG class _NoneName(Enum): @@ -125,7 +131,7 @@ class _NoneName(Enum): """indicate a 'deferred' name that was ultimately the value None.""" -_NONE_NAME = _NoneName.NONE_NAME +_NONE_NAME: Final = _NoneName.NONE_NAME _T = TypeVar("_T", bound=Any) @@ -160,7 +166,9 @@ def _from_column_default( ) -_never_select_column = operator.attrgetter("_omit_from_statements") +_never_select_column: operator.attrgetter[Any] = operator.attrgetter( + "_omit_from_statements" +) class _EntityNamespace(Protocol): @@ -195,12 +203,12 @@ class Immutable: __slots__ = () - _is_immutable = True + _is_immutable: bool = True - def unique_params(self, *optionaldict, **kwargs): + def unique_params(self, *optionaldict: Any, **kwargs: Any) -> NoReturn: raise NotImplementedError("Immutable objects do not support copying") - def params(self, *optionaldict, **kwargs): + def params(self, *optionaldict: Any, **kwargs: Any) -> NoReturn: raise NotImplementedError("Immutable objects do not support copying") def _clone(self: _Self, **kw: Any) -> _Self: @@ -215,7 +223,7 @@ def _copy_internals( class SingletonConstant(Immutable): """Represent SQL constants like NULL, TRUE, FALSE""" - _is_singleton_constant = True + _is_singleton_constant: bool = True _singleton: SingletonConstant @@ -227,7 +235,7 @@ def proxy_set(self) -> FrozenSet[ColumnElement[Any]]: raise NotImplementedError() @classmethod - def _create_singleton(cls): + def _create_singleton(cls) -> None: obj = object.__new__(cls) obj.__init__() # type: ignore @@ -296,17 +304,17 @@ def _generative( def _exclusive_against(*names: str, **kw: Any) -> Callable[[_Fn], _Fn]: - msgs = kw.pop("msgs", {}) + msgs: Dict[str, str] = kw.pop("msgs", {}) - defaults = kw.pop("defaults", {}) + defaults: Dict[str, str] = kw.pop("defaults", {}) - getters = [ + getters: List[Tuple[str, operator.attrgetter[Any], Optional[str]]] = [ (name, operator.attrgetter(name), defaults.get(name, None)) for name in names ] @util.decorator - def check(fn, *args, **kw): + def check(fn: _Fn, *args: Any, **kw: Any) -> Any: # make pylance happy by not including "self" in the argument # list self = args[0] @@ -355,12 +363,16 @@ def _cloned_intersection(a: Iterable[_CLE], b: Iterable[_CLE]) -> Set[_CLE]: The returned set is in terms of the entities present within 'a'. """ - all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b)) + all_overlap: Set[_CLE] = set(_expand_cloned(a)).intersection( + _expand_cloned(b) + ) return {elem for elem in a if all_overlap.intersection(elem._cloned_set)} def _cloned_difference(a: Iterable[_CLE], b: Iterable[_CLE]) -> Set[_CLE]: - all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b)) + all_overlap: Set[_CLE] = set(_expand_cloned(a)).intersection( + _expand_cloned(b) + ) return { elem for elem in a if not all_overlap.intersection(elem._cloned_set) } @@ -372,10 +384,12 @@ class _DialectArgView(MutableMapping[str, Any]): """ - def __init__(self, obj): + __slots__ = ("obj",) + + def __init__(self, obj: DialectKWArgs) -> None: self.obj = obj - def _key(self, key): + def _key(self, key: str) -> Tuple[str, str]: try: dialect, value_key = key.split("_", 1) except ValueError as err: @@ -383,7 +397,7 @@ def _key(self, key): else: return dialect, value_key - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: dialect, value_key = self._key(key) try: @@ -393,7 +407,7 @@ def __getitem__(self, key): else: return opt[value_key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any) -> None: try: dialect, value_key = self._key(key) except KeyError as err: @@ -403,17 +417,17 @@ def __setitem__(self, key, value): else: self.obj.dialect_options[dialect][value_key] = value - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: dialect, value_key = self._key(key) del self.obj.dialect_options[dialect][value_key] - def __len__(self): + def __len__(self) -> int: return sum( len(args._non_defaults) for args in self.obj.dialect_options.values() ) - def __iter__(self): + def __iter__(self) -> Generator[str, None, None]: return ( "%s_%s" % (dialect_name, value_name) for dialect_name in self.obj.dialect_options @@ -432,31 +446,31 @@ class _DialectArgDict(MutableMapping[str, Any]): """ - def __init__(self): - self._non_defaults = {} - self._defaults = {} + def __init__(self) -> None: + self._non_defaults: Dict[str, Any] = {} + self._defaults: Dict[str, Any] = {} - def __len__(self): + def __len__(self) -> int: return len(set(self._non_defaults).union(self._defaults)) - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(set(self._non_defaults).union(self._defaults)) - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: if key in self._non_defaults: return self._non_defaults[key] else: return self._defaults[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any) -> None: self._non_defaults[key] = value - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._non_defaults[key] @util.preload_module("sqlalchemy.dialects") -def _kw_reg_for_dialect(dialect_name): +def _kw_reg_for_dialect(dialect_name: str) -> Optional[Dict[Any, Any]]: dialect_cls = util.preloaded.dialects.registry.load(dialect_name) if dialect_cls.construct_arguments is None: return None @@ -478,12 +492,87 @@ class DialectKWArgs: __slots__ = () - _dialect_kwargs_traverse_internals = [ + _dialect_kwargs_traverse_internals: List[Tuple[str, Any]] = [ ("dialect_options", InternalTraversal.dp_dialect_options) ] + def get_dialect_option( + self, + dialect: Dialect, + argument_name: str, + *, + else_: Any = None, + deprecated_fallback: Optional[str] = None, + ) -> Any: + r"""Return the value of a dialect-specific option, or *else_* if + this dialect does not register the given argument. + + This is useful for DDL compilers that may be inherited by + third-party dialects whose ``construct_arguments`` do not + include the same set of keys as the parent dialect. + + :param dialect: The dialect for which to retrieve the option. + :param argument_name: The name of the argument to retrieve. + :param else\_: The value to return if the argument is not present. + :param deprecated_fallback: Optional dialect name to fall back to + if the argument is not present for the current dialect. If the + argument is present for the fallback dialect but not the current + dialect, a deprecation warning will be emitted. + + """ + + registry = DialectKWArgs._kw_registry[dialect.name] + if registry is None: + return else_ + + if argument_name in registry.get(self.__class__, {}): + if ( + deprecated_fallback is None + or dialect.name == deprecated_fallback + ): + return self.dialect_options[dialect.name][argument_name] + + # deprecated_fallback is present; need to look in two places + + # Current dialect has this option registered. + # Check if user explicitly set it. + if ( + dialect.name in self.dialect_options + and argument_name + in self.dialect_options[dialect.name]._non_defaults + ): + # User explicitly set this dialect's option - use it + return self.dialect_options[dialect.name][argument_name] + + # User didn't set current dialect's option. + # Check for deprecated fallback. + elif ( + deprecated_fallback in self.dialect_options + and argument_name + in self.dialect_options[deprecated_fallback]._non_defaults + ): + # User set fallback option but not current dialect's option + warn_deprecated( + f"Using '{deprecated_fallback}_{argument_name}' " + f"with the '{dialect.name}' dialect is deprecated; " + f"please additionally specify " + f"'{dialect.name}_{argument_name}'.", + version="2.1", + ) + return self.dialect_options[deprecated_fallback][argument_name] + + # Return default value + return self.dialect_options[dialect.name][argument_name] + else: + # Current dialect doesn't have the option registered at all. + # Don't warn - if a third-party dialect doesn't support an + # option, that's their choice, not a deprecation case. + return else_ + @classmethod - def argument_for(cls, dialect_name, argument_name, default): + def argument_for( + cls, dialect_name: str, argument_name: str, default: Any + ) -> None: """Add a new kind of dialect-specific keyword argument for this class. E.g.:: @@ -520,7 +609,9 @@ def argument_for(cls, dialect_name, argument_name, default): """ - construct_arg_dictionary = DialectKWArgs._kw_registry[dialect_name] + construct_arg_dictionary: Optional[Dict[Any, Any]] = ( + DialectKWArgs._kw_registry[dialect_name] + ) if construct_arg_dictionary is None: raise exc.ArgumentError( "Dialect '%s' does have keyword-argument " @@ -530,8 +621,8 @@ def argument_for(cls, dialect_name, argument_name, default): construct_arg_dictionary[cls] = {} construct_arg_dictionary[cls][argument_name] = default - @util.memoized_property - def dialect_kwargs(self): + @property + def dialect_kwargs(self) -> _DialectArgView: """A collection of keyword arguments specified as dialect-specific options to this construct. @@ -552,26 +643,29 @@ def dialect_kwargs(self): return _DialectArgView(self) @property - def kwargs(self): + def kwargs(self) -> _DialectArgView: """A synonym for :attr:`.DialectKWArgs.dialect_kwargs`.""" return self.dialect_kwargs - _kw_registry = util.PopulateDict(_kw_reg_for_dialect) + _kw_registry: util.PopulateDict[str, Optional[Dict[Any, Any]]] = ( + util.PopulateDict(_kw_reg_for_dialect) + ) - def _kw_reg_for_dialect_cls(self, dialect_name): + @classmethod + def _kw_reg_for_dialect_cls(cls, dialect_name: str) -> _DialectArgDict: construct_arg_dictionary = DialectKWArgs._kw_registry[dialect_name] d = _DialectArgDict() if construct_arg_dictionary is None: d._defaults.update({"*": None}) else: - for cls in reversed(self.__class__.__mro__): + for cls in reversed(cls.__mro__): if cls in construct_arg_dictionary: d._defaults.update(construct_arg_dictionary[cls]) return d @util.memoized_property - def dialect_options(self): + def dialect_options(self) -> util.PopulateDict[str, _DialectArgDict]: """A collection of keyword arguments specified as dialect-specific options to this construct. @@ -589,9 +683,7 @@ def dialect_options(self): """ - return util.PopulateDict( - util.portable_instancemethod(self._kw_reg_for_dialect_cls) - ) + return util.PopulateDict(self._kw_reg_for_dialect_cls) def _validate_dialect_kwargs(self, kwargs: Dict[str, Any]) -> None: # validate remaining kwargs that they all specify DB prefixes @@ -765,7 +857,7 @@ class InPlaceGenerative(HasMemoized): __slots__ = () - def _generate(self): + def _generate(self) -> Self: skip = self._memoized_keys # note __dict__ needs to be in __slots__ if this is used for k in skip: @@ -835,7 +927,7 @@ def __init_subclass__(cls) -> None: ) super().__init_subclass__() - def __init__(self, **kw): + def __init__(self, **kw: Any) -> None: self.__dict__.update(kw) def __add__(self, other): @@ -860,7 +952,7 @@ def __eq__(self, other): return False return True - def __repr__(self): + def __repr__(self) -> str: # TODO: fairly inefficient, used only in debugging right now. return "%s(%s)" % ( @@ -877,7 +969,7 @@ def isinstance(cls, klass: Type[Any]) -> bool: return issubclass(cls, klass) @hybridmethod - def add_to_element(self, name, value): + def add_to_element(self, name: str, value: str) -> Any: return self + {name: getattr(self, name) + value} @hybridmethod @@ -891,7 +983,7 @@ def _state_dict(cls) -> Mapping[str, Any]: return cls._state_dict_const @classmethod - def safe_merge(cls, other): + def safe_merge(cls, other: "Options") -> Any: d = other._state_dict() # only support a merge with another object of our class @@ -917,8 +1009,12 @@ def safe_merge(cls, other): @classmethod def from_execution_options( - cls, key, attrs, exec_options, statement_exec_options - ): + cls, + key: str, + attrs: set[str], + exec_options: Mapping[str, Any], + statement_exec_options: Mapping[str, Any], + ) -> Tuple["Options", Mapping[str, Any]]: """process Options argument in terms of execution options. @@ -957,7 +1053,7 @@ def from_execution_options( result[local] = statement_exec_options[argname] new_options = existing_options + result - exec_options = util.immutabledict().merge_with( + exec_options = util.EMPTY_DICT.merge_with( exec_options, {key: new_options} ) return new_options, exec_options @@ -978,28 +1074,32 @@ class CacheableOptions(Options, HasCacheKey): __slots__ = () @hybridmethod - def _gen_cache_key_inst(self, anon_map, bindparams): + def _gen_cache_key_inst( + self, anon_map: Any, bindparams: List[BindParameter[Any]] + ) -> Optional[Tuple[Any]]: return HasCacheKey._gen_cache_key(self, anon_map, bindparams) @_gen_cache_key_inst.classlevel - def _gen_cache_key(cls, anon_map, bindparams): + def _gen_cache_key( + cls, anon_map: "anon_map", bindparams: List[BindParameter[Any]] + ) -> Tuple[CacheableOptions, Any]: return (cls, ()) @hybridmethod - def _generate_cache_key(self): - return HasCacheKey._generate_cache_key_for_object(self) + def _generate_cache_key(self) -> Optional[CacheKey]: + return HasCacheKey._generate_cache_key(self) class ExecutableOption(HasCopyInternals): __slots__ = () - _annotations = util.EMPTY_DICT + _annotations: _ImmutableExecuteOptions = util.EMPTY_DICT - __visit_name__ = "executable_option" + __visit_name__: str = "executable_option" - _is_has_cache_key = False + _is_has_cache_key: bool = False - _is_core = True + _is_core: bool = True def _clone(self, **kw): """Create a shallow copy of this ExecutableOption.""" @@ -1030,6 +1130,10 @@ def ext(self, extension: SyntaxExtension) -> Self: :ref:`examples_syntax_extensions` + :func:`_mysql.limit` - DML LIMIT for MySQL + + :func:`_postgresql.distinct_on` - DISTINCT ON for PostgreSQL + .. versionadded:: 2.1 """ @@ -1102,6 +1206,8 @@ def apply_to_select(self, select_stmt: Select) -> None: :ref:`examples_syntax_extensions` + :meth:`.ext` + """ # noqa: E501 @@ -1164,13 +1270,13 @@ def append_replacing_same_type( self, existing: Sequence[ClauseElement] ) -> Sequence[ClauseElement]: """Utility function that can be used as - :paramref:`_sql.HasSyntaxExtensions.apply_extension_point.apply_fn` + :paramref:`_sql.Select.apply_syntax_extension_point.apply_fn` to remove any other element of the same type in existing and appending ``self`` to the list. This is equivalent to:: - stmt.apply_extension_point( + stmt.apply_syntax_extension_point( lambda existing: [ *(e for e in existing if not isinstance(e, ReplaceOfTypeExt)), self, @@ -1182,7 +1288,8 @@ def append_replacing_same_type( :ref:`examples_syntax_extensions` - :meth:`_sql.HasSyntaxExtensions.apply_syntax_extension_point` + :meth:`_sql.Select.apply_syntax_extension_point` and equivalents + in :class:`_dml.Insert`, :class:`_dml.Delete`, :class:`_dml.Update` """ # noqa: E501 cls = type(self) @@ -1207,8 +1314,7 @@ def apply_to_delete(self, delete_stmt: Delete) -> None: ) def apply_to_insert(self, insert_stmt: Insert) -> None: - """Apply this :class:`.SyntaxExtension` to an - :class:`_sql.Insert`""" + """Apply this :class:`.SyntaxExtension` to an :class:`_sql.Insert`""" raise NotImplementedError( f"Extension {type(self).__name__} cannot be applied to insert" ) @@ -1225,7 +1331,7 @@ class Executable(roles.StatementRole): supports_execution: bool = True _execution_options: _ImmutableExecuteOptions = util.EMPTY_DICT - _is_default_generator = False + _is_default_generator: bool = False _with_options: Tuple[ExecutableOption, ...] = () _compile_state_funcs: Tuple[ Tuple[Callable[[CompileState], None], Any], ... @@ -1241,13 +1347,13 @@ class Executable(roles.StatementRole): ("_propagate_attrs", ExtendedInternalTraversal.dp_propagate_attrs), ] - is_select = False - is_from_statement = False - is_update = False - is_insert = False - is_text = False - is_delete = False - is_dml = False + is_select: bool = False + is_from_statement: bool = False + is_update: bool = False + is_insert: bool = False + is_text: bool = False + is_delete: bool = False + is_dml: bool = False if TYPE_CHECKING: __visit_name__: str @@ -1261,8 +1367,11 @@ def _compile_w_cache( for_executemany: bool = False, schema_translate_map: Optional[SchemaTranslateMapType] = None, **kw: Any, - ) -> Tuple[ - Compiled, Optional[Sequence[BindParameter[Any]]], CacheStats + ) -> tuple[ + Compiled, + Sequence[BindParameter[Any]] | None, + _CoreSingleExecuteParams | None, + CacheStats, ]: ... def _execute_on_connection( @@ -1280,7 +1389,7 @@ def _execute_on_scalar( ) -> Any: ... @util.ro_non_memoized_property - def _all_selected_columns(self): + def _all_selected_columns(self) -> _SelectIterable: raise NotImplementedError() @property @@ -1507,8 +1616,6 @@ def _process_opt(conn, statement, multiparams, params, execution_options): def get_execution_options(self) -> _ExecuteOptions: """Get the non-SQL options which will take effect during execution. - .. versionadded:: 1.3 - .. seealso:: :meth:`.Executable.execution_options` @@ -1516,6 +1623,50 @@ def get_execution_options(self) -> _ExecuteOptions: return self._execution_options +class ExecutableStatement(Executable): + """Executable subclass that implements a lightweight version of ``params`` + that avoids a full cloned traverse. + + .. versionadded:: 2.1 + + """ + + _params: util.immutabledict[str, Any] = EMPTY_DICT + + _executable_traverse_internals = ( + Executable._executable_traverse_internals + + [("_params", InternalTraversal.dp_params)] + ) + + @_generative + def params( + self, + __optionaldict: _CoreSingleExecuteParams | None = None, + /, + **kwargs: Any, + ) -> Self: + """Return a copy with the provided bindparam values. + + Returns a copy of this Executable with bindparam values set + to the given dictionary:: + + >>> clause = column("x") + bindparam("foo") + >>> print(clause.compile().params) + {'foo': None} + >>> print(clause.params({"foo": 7}).compile().params) + {'foo': 7} + + """ + if __optionaldict: + kwargs.update(__optionaldict) + self._params = ( + util.immutabledict(kwargs) + if not self._params + else self._params | kwargs + ) + return self + + class SchemaEventTarget(event.EventTarget): """Base class for elements that are the targets of :class:`.DDLEvents` events. @@ -1537,10 +1688,21 @@ def _set_parent_with_dispatch( self.dispatch.after_parent_attach(self, parent) +class SchemaVisitable(SchemaEventTarget, visitors.Visitable): + """Base class for elements that are targets of a :class:`.SchemaVisitor`. + + .. versionadded:: 2.0.41 + + """ + + class SchemaVisitor(ClauseVisitor): - """Define the visiting for ``SchemaItem`` objects.""" + """Define the visiting for ``SchemaItem`` and more + generally ``SchemaVisitable`` objects. + + """ - __traverse_options__ = {"schema_visitor": True} + __traverse_options__: Dict[str, Any] = {"schema_visitor": True} class _SentinelDefaultCharacterization(Enum): @@ -1551,6 +1713,7 @@ class _SentinelDefaultCharacterization(Enum): SERVERSIDE = "serverside" IDENTITY = "identity" SEQUENCE = "sequence" + MONOTONIC_FUNCTION = "monotonic" class _SentinelColumnCharacterization(NamedTuple): @@ -1575,7 +1738,7 @@ class _ColumnMetrics(Generic[_COL_co]): def __init__( self, collection: ColumnCollection[Any, _COL_co], col: _COL_co - ): + ) -> None: self.column = col # proxy_index being non-empty means it was initialized. @@ -1585,10 +1748,10 @@ def __init__( for eps_col in col._expanded_proxy_set: pi[eps_col].add(self) - def get_expanded_proxy_set(self): + def get_expanded_proxy_set(self) -> FrozenSet[ColumnElement[Any]]: return self.column._expanded_proxy_set - def dispose(self, collection): + def dispose(self, collection: ColumnCollection[_COLKEY, _COL_co]) -> None: pi = collection._proxy_index if not pi: return @@ -1613,9 +1776,8 @@ def embedded( class ColumnCollection(Generic[_COLKEY, _COL_co]): - """Collection of :class:`_expression.ColumnElement` instances, - typically for - :class:`_sql.FromClause` objects. + """Base class for collection of :class:`_expression.ColumnElement` + instances, typically for :class:`_sql.FromClause` objects. The :class:`_sql.ColumnCollection` object is most commonly available as the :attr:`_schema.Table.c` or :attr:`_schema.Table.columns` collection @@ -1682,34 +1844,16 @@ class ColumnCollection(Generic[_COLKEY, _COL_co]): [Column('x', Integer(), table=None), Column('y', Integer(), table=None)] - The base :class:`_expression.ColumnCollection` object can store - duplicates, which can - mean either two columns with the same key, in which case the column - returned by key access is **arbitrary**:: - - >>> x1, x2 = Column("x", Integer), Column("x", Integer) - >>> cc = ColumnCollection(columns=[(x1.name, x1), (x2.name, x2)]) - >>> list(cc) - [Column('x', Integer(), table=None), - Column('x', Integer(), table=None)] - >>> cc["x"] is x1 - False - >>> cc["x"] is x2 - True - - Or it can also mean the same column multiple times. These cases are - supported as :class:`_expression.ColumnCollection` - is used to represent the columns in - a SELECT statement which may include duplicates. - - A special subclass :class:`.DedupeColumnCollection` exists which instead + The :class:`_expression.ColumnCollection` base class is read-only. + For mutation operations, the :class:`.WriteableColumnCollection` subclass + provides methods such as :meth:`.WriteableColumnCollection.add`. + A special subclass :class:`.DedupeColumnCollection` exists which maintains SQLAlchemy's older behavior of not allowing duplicates; this collection is used for schema level objects like :class:`_schema.Table` - and - :class:`.PrimaryKeyConstraint` where this deduping is helpful. The - :class:`.DedupeColumnCollection` class also has additional mutation methods - as the schema constructs have more use cases that require removal and - replacement of columns. + and :class:`.PrimaryKeyConstraint` where this deduping is helpful. + The :class:`.DedupeColumnCollection` class also has additional mutation + methods as the schema constructs have more use cases that require removal + and replacement of columns. .. versionchanged:: 1.4 :class:`_expression.ColumnCollection` now stores duplicate @@ -1718,27 +1862,27 @@ class ColumnCollection(Generic[_COLKEY, _COL_co]): former behavior in those cases where deduplication as well as additional replace/remove operations are needed. + .. versionchanged:: 2.1 :class:`_expression.ColumnCollection` is now + a read-only base class. Mutation operations are available through + :class:`.WriteableColumnCollection` and :class:`.DedupeColumnCollection` + subclasses. + """ - __slots__ = "_collection", "_index", "_colset", "_proxy_index" + __slots__ = ("_collection", "_index", "_colset", "_proxy_index") _collection: List[Tuple[_COLKEY, _COL_co, _ColumnMetrics[_COL_co]]] _index: Dict[Union[None, str, int], Tuple[_COLKEY, _COL_co]] - _proxy_index: Dict[ColumnElement[Any], Set[_ColumnMetrics[_COL_co]]] _colset: Set[_COL_co] + _proxy_index: Dict[ColumnElement[Any], Set[_ColumnMetrics[_COL_co]]] - def __init__( - self, columns: Optional[Iterable[Tuple[_COLKEY, _COL_co]]] = None - ): - object.__setattr__(self, "_colset", set()) - object.__setattr__(self, "_index", {}) - object.__setattr__( - self, "_proxy_index", collections.defaultdict(util.OrderedSet) + def __init__(self) -> None: + raise TypeError( + "ColumnCollection is an abstract base class and cannot be " + "instantiated directly. Use WriteableColumnCollection or " + "DedupeColumnCollection instead." ) - object.__setattr__(self, "_collection", []) - if columns: - self._initial_populate(columns) @util.preload_module("sqlalchemy.sql.elements") def __clause_element__(self) -> ClauseList: @@ -1750,11 +1894,6 @@ def __clause_element__(self) -> ClauseList: *self._all_columns, ) - def _initial_populate( - self, iter_: Iterable[Tuple[_COLKEY, _COL_co]] - ) -> None: - self._populate_separate_keys(iter_) - @property def _all_columns(self) -> List[_COL_co]: return [col for (_, col, _) in self._collection] @@ -1794,12 +1933,7 @@ def __getitem__(self, key: Union[str, int]) -> _COL_co: ... @overload def __getitem__( - self, key: Tuple[Union[str, int], ...] - ) -> ReadOnlyColumnCollection[_COLKEY, _COL_co]: ... - - @overload - def __getitem__( - self, key: slice + self, key: Union[Tuple[Union[str, int], ...], slice] ) -> ReadOnlyColumnCollection[_COLKEY, _COL_co]: ... def __getitem__( @@ -1815,7 +1949,7 @@ def __getitem__( else: cols = (self._index[sub_key] for sub_key in key) - return ColumnCollection(cols).as_readonly() + return WriteableColumnCollection(cols).as_readonly() else: return self._index[key][1] except KeyError as err: @@ -1840,7 +1974,7 @@ def __contains__(self, key: str) -> bool: else: return True - def compare(self, other: ColumnCollection[Any, Any]) -> bool: + def compare(self, other: ColumnCollection[_COLKEY, _COL_co]) -> bool: """Compare this :class:`_expression.ColumnCollection` to another based on the names of the keys""" @@ -1877,30 +2011,93 @@ def __str__(self) -> str: ", ".join(str(c) for c in self), ) - def __setitem__(self, key: str, value: Any) -> NoReturn: - raise NotImplementedError() + # https://github.com/python/mypy/issues/4266 + __hash__: Optional[int] = None # type: ignore - def __delitem__(self, key: str) -> NoReturn: - raise NotImplementedError() + def contains_column(self, col: ColumnElement[Any]) -> bool: + """Checks if a column object exists in this collection""" + if col not in self._colset: + if isinstance(col, str): + raise exc.ArgumentError( + "contains_column cannot be used with string arguments. " + "Use ``col_name in table.c`` instead." + ) + return False + else: + return True - def __setattr__(self, key: str, obj: Any) -> NoReturn: + def _as_readonly(self) -> ReadOnlyColumnCollection[_COLKEY, _COL_co]: raise NotImplementedError() - def clear(self) -> NoReturn: - """Dictionary clear() is not implemented for - :class:`_sql.ColumnCollection`.""" - raise NotImplementedError() + def corresponding_column( + self, column: _COL, require_embedded: bool = False + ) -> Optional[Union[_COL, _COL_co]]: + """Given a :class:`_expression.ColumnElement`, return the exported + :class:`_expression.ColumnElement` object from this + :class:`_expression.ColumnCollection` + which corresponds to that original :class:`_expression.ColumnElement` + via a common + ancestor column. - def remove(self, column: Any) -> None: - raise NotImplementedError() + :param column: the target :class:`_expression.ColumnElement` + to be matched. + + :param require_embedded: only return corresponding columns for + the given :class:`_expression.ColumnElement`, if the given + :class:`_expression.ColumnElement` + is actually present within a sub-element + of this :class:`_expression.Selectable`. + Normally the column will match if + it merely shares a common ancestor with one of the exported + columns of this :class:`_expression.Selectable`. - def update(self, iter_: Any) -> NoReturn: - """Dictionary update() is not implemented for - :class:`_sql.ColumnCollection`.""" + .. seealso:: + + :meth:`_expression.Selectable.corresponding_column` + - invokes this method + against the collection returned by + :attr:`_expression.Selectable.exported_columns`. + + .. versionchanged:: 1.4 the implementation for ``corresponding_column`` + was moved onto the :class:`_expression.ColumnCollection` itself. + + """ raise NotImplementedError() - # https://github.com/python/mypy/issues/4266 - __hash__ = None # type: ignore + +class WriteableColumnCollection(ColumnCollection[_COLKEY, _COL_co]): + """A :class:`_sql.ColumnCollection` that allows mutation operations. + + This is the writable form of :class:`_sql.ColumnCollection` that + implements methods such as :meth:`.add`, :meth:`.remove`, :meth:`.update`, + and :meth:`.clear`. + + This class is used internally for building column collections during + construction of SQL constructs. For schema-level objects that require + deduplication behavior, use :class:`.DedupeColumnCollection`. + + .. versionadded:: 2.1 + + """ + + __slots__ = () + + def __init__( + self, columns: Optional[Iterable[Tuple[_COLKEY, _COL_co]]] = None + ): + object.__setattr__(self, "_colset", set()) + object.__setattr__(self, "_index", {}) + object.__setattr__( + self, "_proxy_index", collections.defaultdict(util.OrderedSet) + ) + object.__setattr__(self, "_collection", []) + if columns: + self._initial_populate(columns) + + def _initial_populate( + self, iter_: Iterable[Tuple[_COLKEY, _COL_co]] + ) -> None: + self._populate_separate_keys(iter_) def _populate_separate_keys( self, iter_: Iterable[Tuple[_COLKEY, _COL_co]] @@ -1916,16 +2113,41 @@ def _populate_separate_keys( ) self._index.update({k: (k, col) for k, col, _ in reversed(collection)}) + def __getstate__(self) -> Dict[str, Any]: + return { + "_collection": [(k, c) for k, c, _ in self._collection], + "_index": self._index, + } + + def __setstate__(self, state: Dict[str, Any]) -> None: + object.__setattr__(self, "_index", state["_index"]) + object.__setattr__( + self, "_proxy_index", collections.defaultdict(util.OrderedSet) + ) + object.__setattr__( + self, + "_collection", + [ + (k, c, _ColumnMetrics(self, c)) + for (k, c) in state["_collection"] + ], + ) + object.__setattr__( + self, "_colset", {col for k, col, _ in self._collection} + ) + def add( - self, column: ColumnElement[Any], key: Optional[_COLKEY] = None + self, + column: ColumnElement[Any], + key: Optional[_COLKEY] = None, ) -> None: - """Add a column to this :class:`_sql.ColumnCollection`. + """Add a column to this :class:`_sql.WriteableColumnCollection`. .. note:: This method is **not normally used by user-facing code**, as the - :class:`_sql.ColumnCollection` is usually part of an existing - object such as a :class:`_schema.Table`. To add a + :class:`_sql.WriteableColumnCollection` is usually part of an + existing object such as a :class:`_schema.Table`. To add a :class:`_schema.Column` to an existing :class:`_schema.Table` object, use the :meth:`_schema.Table.append_column` method. @@ -1948,52 +2170,21 @@ def add( (colkey, _column, _ColumnMetrics(self, _column)) ) self._colset.add(_column._deannotate()) + self._index[l] = (colkey, _column) if colkey not in self._index: self._index[colkey] = (colkey, _column) - def __getstate__(self) -> Dict[str, Any]: - return { - "_collection": [(k, c) for k, c, _ in self._collection], - "_index": self._index, - } - - def __setstate__(self, state: Dict[str, Any]) -> None: - object.__setattr__(self, "_index", state["_index"]) - object.__setattr__( - self, "_proxy_index", collections.defaultdict(util.OrderedSet) - ) - object.__setattr__( - self, - "_collection", - [ - (k, c, _ColumnMetrics(self, c)) - for (k, c) in state["_collection"] - ], - ) - object.__setattr__( - self, "_colset", {col for k, col, _ in self._collection} - ) - - def contains_column(self, col: ColumnElement[Any]) -> bool: - """Checks if a column object exists in this collection""" - if col not in self._colset: - if isinstance(col, str): - raise exc.ArgumentError( - "contains_column cannot be used with string arguments. " - "Use ``col_name in table.c`` instead." - ) - return False - else: - return True + def _as_readonly(self) -> ReadOnlyColumnCollection[_COLKEY, _COL_co]: + return ReadOnlyColumnCollection(self) def as_readonly(self) -> ReadOnlyColumnCollection[_COLKEY, _COL_co]: """Return a "read only" form of this - :class:`_sql.ColumnCollection`.""" + :class:`_sql.WriteableColumnCollection`.""" - return ReadOnlyColumnCollection(self) + return self._as_readonly() - def _init_proxy_index(self): + def _init_proxy_index(self) -> None: """populate the "proxy index", if empty. proxy index is added in 2.0 to provide more efficient operation @@ -2029,27 +2220,8 @@ def corresponding_column( via a common ancestor column. - :param column: the target :class:`_expression.ColumnElement` - to be matched. - - :param require_embedded: only return corresponding columns for - the given :class:`_expression.ColumnElement`, if the given - :class:`_expression.ColumnElement` - is actually present within a sub-element - of this :class:`_expression.Selectable`. - Normally the column will match if - it merely shares a common ancestor with one of the exported - columns of this :class:`_expression.Selectable`. - - .. seealso:: - - :meth:`_expression.Selectable.corresponding_column` - - invokes this method - against the collection returned by - :attr:`_expression.Selectable.exported_columns`. - - .. versionchanged:: 1.4 the implementation for ``corresponding_column`` - was moved onto the :class:`_expression.ColumnCollection` itself. + See :meth:`.ColumnCollection.corresponding_column` for parameter + information. """ # TODO: cython candidate @@ -2129,7 +2301,7 @@ def corresponding_column( _NAMEDCOL = TypeVar("_NAMEDCOL", bound="NamedColumn[Any]") -class DedupeColumnCollection(ColumnCollection[str, _NAMEDCOL]): +class DedupeColumnCollection(WriteableColumnCollection[str, _NAMEDCOL]): """A :class:`_expression.ColumnCollection` that maintains deduplicating behavior. @@ -2143,7 +2315,11 @@ class DedupeColumnCollection(ColumnCollection[str, _NAMEDCOL]): """ def add( # type: ignore[override] - self, column: _NAMEDCOL, key: Optional[str] = None + self, + column: _NAMEDCOL, + key: Optional[str] = None, + *, + index: Optional[int] = None, ) -> None: if key is not None and column.key != key: raise exc.ArgumentError( @@ -2163,21 +2339,42 @@ def add( # type: ignore[override] if existing is column: return - self.replace(column) + self.replace(column, index=index) # pop out memoized proxy_set as this # operation may very well be occurring # in a _make_proxy operation util.memoized_property.reset(column, "proxy_set") else: - self._append_new_column(key, column) + self._append_new_column(key, column, index=index) + + def _append_new_column( + self, key: str, named_column: _NAMEDCOL, *, index: Optional[int] = None + ) -> None: + collection_length = len(self._collection) + + if index is None: + l = collection_length + else: + if index < 0: + index = max(0, collection_length + index) + l = index + + if index is None: + self._collection.append( + (key, named_column, _ColumnMetrics(self, named_column)) + ) + else: + self._collection.insert( + index, (key, named_column, _ColumnMetrics(self, named_column)) + ) - def _append_new_column(self, key: str, named_column: _NAMEDCOL) -> None: - l = len(self._collection) - self._collection.append( - (key, named_column, _ColumnMetrics(self, named_column)) - ) self._colset.add(named_column._deannotate()) + + if index is not None: + for idx in reversed(range(index, collection_length)): + self._index[idx + 1] = self._index[idx] + self._index[l] = (key, named_column) self._index[key] = (key, named_column) @@ -2237,7 +2434,9 @@ def remove(self, column: _NAMEDCOL) -> None: def replace( self, column: _NAMEDCOL, + *, extra_remove: Optional[Iterable[_NAMEDCOL]] = None, + index: Optional[int] = None, ) -> None: """add the given column to this collection, removing unaliased versions of this column as well as existing columns with the @@ -2269,14 +2468,15 @@ def replace( remove_col.add(self._index[column.key][1]) if not remove_col: - self._append_new_column(column.key, column) + self._append_new_column(column.key, column, index=index) return new_cols: List[Tuple[str, _NAMEDCOL, _ColumnMetrics[_NAMEDCOL]]] = [] - replaced = False - for k, col, metrics in self._collection: + replace_index = None + + for idx, (k, col, metrics) in enumerate(self._collection): if col in remove_col: - if not replaced: - replaced = True + if replace_index is None: + replace_index = idx new_cols.append( (column.key, column, _ColumnMetrics(self, column)) ) @@ -2290,8 +2490,26 @@ def replace( for metrics in self._proxy_index.get(rc, ()): metrics.dispose(self) - if not replaced: - new_cols.append((column.key, column, _ColumnMetrics(self, column))) + if replace_index is None: + if index is not None: + new_cols.insert( + index, (column.key, column, _ColumnMetrics(self, column)) + ) + + else: + new_cols.append( + (column.key, column, _ColumnMetrics(self, column)) + ) + elif index is not None: + to_move = new_cols[replace_index] + effective_positive_index = ( + index if index >= 0 else max(0, len(new_cols) + index) + ) + new_cols.insert(index, to_move) + if replace_index > effective_positive_index: + del new_cols[replace_index + 1] + else: + del new_cols[replace_index] self._colset.add(column._deannotate()) self._collection[:] = new_cols @@ -2309,35 +2527,49 @@ class ReadOnlyColumnCollection( ): __slots__ = ("_parent",) - def __init__(self, collection): + _parent: WriteableColumnCollection[_COLKEY, _COL_co] + + def __init__( + self, collection: WriteableColumnCollection[_COLKEY, _COL_co] + ): object.__setattr__(self, "_parent", collection) - object.__setattr__(self, "_colset", collection._colset) object.__setattr__(self, "_index", collection._index) object.__setattr__(self, "_collection", collection._collection) + object.__setattr__(self, "_colset", collection._colset) object.__setattr__(self, "_proxy_index", collection._proxy_index) - def __getstate__(self): + def _as_readonly(self) -> ReadOnlyColumnCollection[_COLKEY, _COL_co]: + return self + + def __getstate__(self) -> Dict[str, ColumnCollection[_COLKEY, _COL_co]]: return {"_parent": self._parent} - def __setstate__(self, state): + def __setstate__(self, state: Dict[str, Any]) -> None: parent = state["_parent"] self.__init__(parent) # type: ignore - def add(self, column: Any, key: Any = ...) -> Any: - self._readonly() + def corresponding_column( + self, column: _COL, require_embedded: bool = False + ) -> Optional[Union[_COL, _COL_co]]: + """Given a :class:`_expression.ColumnElement`, return the exported + :class:`_expression.ColumnElement` object from this + :class:`_expression.ColumnCollection` + which corresponds to that original :class:`_expression.ColumnElement` + via a common + ancestor column. - def extend(self, elements: Any) -> NoReturn: - self._readonly() + See :meth:`.ColumnCollection.corresponding_column` for parameter + information. - def remove(self, item: Any) -> NoReturn: - self._readonly() + """ + return self._parent.corresponding_column(column, require_embedded) class ColumnSet(util.OrderedSet["ColumnClause[Any]"]): - def contains_column(self, col): + def contains_column(self, col: ColumnClause[Any]) -> bool: return col in self - def extend(self, cols): + def extend(self, cols: Iterable[Any]) -> None: for col in cols: self.add(col) @@ -2349,12 +2581,12 @@ def __eq__(self, other): l.append(c == local) return elements.and_(*l) - def __hash__(self): # type: ignore[override] + def __hash__(self) -> int: # type: ignore[override] return hash(tuple(x for x in self)) def _entity_namespace( - entity: Union[_HasEntityNamespace, ExternallyTraversible] + entity: Union[_HasEntityNamespace, ExternallyTraversible], ) -> _EntityNamespace: """Return the nearest .entity_namespace for the given entity. @@ -2372,11 +2604,34 @@ def _entity_namespace( raise +@overload def _entity_namespace_key( entity: Union[_HasEntityNamespace, ExternallyTraversible], key: str, - default: Union[SQLCoreOperations[Any], _NoArg] = NO_ARG, -) -> SQLCoreOperations[Any]: +) -> SQLCoreOperations[Any]: ... + + +@overload +def _entity_namespace_key( + entity: Union[_HasEntityNamespace, ExternallyTraversible], + key: str, + default: _NoArg, +) -> SQLCoreOperations[Any]: ... + + +@overload +def _entity_namespace_key( + entity: Union[_HasEntityNamespace, ExternallyTraversible], + key: str, + default: _T, +) -> Union[SQLCoreOperations[Any], _T]: ... + + +def _entity_namespace_key( + entity: Union[_HasEntityNamespace, ExternallyTraversible], + key: str, + default: Union[SQLCoreOperations[Any], _T, _NoArg] = NO_ARG, +) -> Union[SQLCoreOperations[Any], _T]: """Return an entry from an entity_namespace. @@ -2395,3 +2650,50 @@ def _entity_namespace_key( raise exc.InvalidRequestError( 'Entity namespace for "%s" has no property "%s"' % (entity, key) ) from err + + +def _entity_namespace_key_search_all( + entities: Collection[Any], + key: str, +) -> SQLCoreOperations[Any]: + """Search multiple entities for a key, raise if ambiguous or not found. + + This is used by filter_by() to search across all FROM clause entities + when a single entity doesn't have the requested attribute. + + .. versionadded:: 2.1 + + Raises: + AmbiguousColumnError: If key exists in multiple entities + InvalidRequestError: If key doesn't exist in any entity + """ + + match_: SQLCoreOperations[Any] | None = None + + for entity in entities: + ns = _entity_namespace(entity) + # Check if the attribute exists + if hasattr(ns, key): + if match_ is not None: + entity_desc = ", ".join(str(e) for e in list(entities)[:3]) + if len(entities) > 3: + entity_desc += f", ... ({len(entities)} total)" + raise exc.AmbiguousColumnError( + f'Attribute name "{key}" is ambiguous; it exists in ' + f"multiple FROM clause entities ({entity_desc}). " + f"Use filter() with explicit column references instead " + f"of filter_by()." + ) + match_ = getattr(ns, key) + + if match_ is None: + # No entity has this attribute + entity_desc = ", ".join(str(e) for e in list(entities)[:3]) + if len(entities) > 3: + entity_desc += f", ... ({len(entities)} total)" + raise exc.InvalidRequestError( + f'None of the FROM clause entities have a property "{key}". ' + f"Searched entities: {entity_desc}" + ) + + return match_ diff --git a/lib/sqlalchemy/sql/cache_key.py b/lib/sqlalchemy/sql/cache_key.py index 5ac11878bac..fbf02f893dc 100644 --- a/lib/sqlalchemy/sql/cache_key.py +++ b/lib/sqlalchemy/sql/cache_key.py @@ -1,5 +1,5 @@ # sql/cache_key.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -13,9 +13,11 @@ from typing import Any from typing import Callable from typing import Dict +from typing import Final from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import MutableMapping from typing import NamedTuple from typing import Optional @@ -32,7 +34,6 @@ from .. import util from ..inspection import inspect from ..util import HasMemoized -from ..util.typing import Literal if typing.TYPE_CHECKING: from .elements import BindParameter @@ -50,9 +51,10 @@ def __call__( class CacheConst(enum.Enum): NO_CACHE = 0 + PARAMS = 1 -NO_CACHE = CacheConst.NO_CACHE +NO_CACHE: Final = CacheConst.NO_CACHE _CacheKeyTraversalType = Union[ @@ -384,21 +386,11 @@ def _generate_cache_key(self) -> Optional[CacheKey]: return None else: assert key is not None - return CacheKey(key, bindparams) - - @classmethod - def _generate_cache_key_for_object( - cls, obj: HasCacheKey - ) -> Optional[CacheKey]: - bindparams: List[BindParameter[Any]] = [] - - _anon_map = anon_map() - key = obj._gen_cache_key(_anon_map, bindparams) - if NO_CACHE in _anon_map: - return None - else: - assert key is not None - return CacheKey(key, bindparams) + return CacheKey( + key, + bindparams, + _anon_map.get(CacheConst.PARAMS), # type: ignore[arg-type] + ) class HasCacheKeyTraverse(HasTraverseInternals, HasCacheKey): @@ -432,6 +424,7 @@ class CacheKey(NamedTuple): key: Tuple[Any, ...] bindparams: Sequence[BindParameter[Any]] + params: _CoreSingleExecuteParams | None # can't set __hash__ attribute because it interferes # with namedtuple @@ -485,8 +478,8 @@ def __ne__(self, other: Any) -> bool: @classmethod def _diff_tuples(cls, left: CacheKey, right: CacheKey) -> str: - ck1 = CacheKey(left, []) - ck2 = CacheKey(right, []) + ck1 = CacheKey(left, [], None) + ck2 = CacheKey(right, [], None) return ck1._diff(ck2) def _whats_different(self, other: CacheKey) -> Iterator[str]: @@ -516,7 +509,7 @@ def _whats_different(self, other: CacheKey) -> Iterator[str]: e2, ) else: - pickup_index = stack.pop(-1) + stack.pop(-1) break def _diff(self, other: CacheKey) -> str: @@ -1053,5 +1046,21 @@ def visit_dml_multi_values( anon_map[NO_CACHE] = True return () + def visit_params( + self, + attrname: str, + obj: Any, + parent: Any, + anon_map: anon_map, + bindparams: List[BindParameter[Any]], + ) -> Tuple[Any, ...]: + if obj: + if CacheConst.PARAMS in anon_map: + to_set = anon_map[CacheConst.PARAMS] | obj + else: + to_set = obj + anon_map[CacheConst.PARAMS] = to_set + return () + _cache_key_traversal_visitor = _CacheKeyTraversal() diff --git a/lib/sqlalchemy/sql/coercions.py b/lib/sqlalchemy/sql/coercions.py index fc3614c06ba..b73eb6de7b6 100644 --- a/lib/sqlalchemy/sql/coercions.py +++ b/lib/sqlalchemy/sql/coercions.py @@ -1,5 +1,5 @@ # sql/coercions.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -19,6 +19,7 @@ from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import NoReturn from typing import Optional from typing import overload @@ -39,7 +40,6 @@ from .. import exc from .. import inspection from .. import util -from ..util.typing import Literal if typing.TYPE_CHECKING: # elements lambdas schema selectable are set by __init__ @@ -52,6 +52,7 @@ from ._typing import _DDLColumnArgument from ._typing import _DMLTableArgument from ._typing import _FromClauseArgument + from ._typing import _OnlyColumnArgument from .base import SyntaxExtension from .dml import _DMLTableElement from .elements import BindParameter @@ -76,7 +77,7 @@ _T = TypeVar("_T", bound=Any) -def _is_literal(element): +def _is_literal(element: Any) -> bool: """Return whether or not the element is a "literal" in the context of a SQL expression construct. @@ -221,7 +222,7 @@ def expect( @overload def expect( role: Type[roles.LabeledColumnExprRole[Any]], - element: _ColumnExpressionArgument[_T], + element: _ColumnExpressionArgument[_T] | _OnlyColumnArgument[_T], **kw: Any, ) -> NamedColumn[_T]: ... @@ -795,6 +796,10 @@ def _raise_for_expected(self, element, argname=None, resolved=None, **kw): ) +class TStringElementImpl(ExpressionElementImpl, RoleImpl): + __slots__ = () + + class BinaryElementImpl(ExpressionElementImpl, RoleImpl): __slots__ = () @@ -852,7 +857,7 @@ def _warn_for_implicit_coercion(self, elem): ) @util.preload_module("sqlalchemy.sql.elements") - def _literal_coercion(self, element, *, expr, operator, **kw): + def _literal_coercion(self, element, *, expr, operator, **kw): # type: ignore[override] # noqa: E501 if util.is_non_string_iterable(element): non_literal_expressions: Dict[ Optional[_ColumnExpressionArgument[Any]], @@ -1178,21 +1183,11 @@ def _post_coercion( if resolved is not original_element and not isinstance( original_element, str ): - # use same method as Connection uses; this will later raise - # ObjectNotExecutableError + # use same method as Connection uses try: original_element._execute_on_connection - except AttributeError: - util.warn_deprecated( - "Object %r should not be used directly in a SQL statement " - "context, such as passing to methods such as " - "session.execute(). This usage will be disallowed in a " - "future release. " - "Please use Core select() / update() / delete() etc. " - "with Session.execute() and other statement execution " - "methods." % original_element, - "1.4", - ) + except AttributeError as err: + raise exc.ObjectNotExecutableError(original_element) from err return resolved diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index 32043dd7bb4..b8781d00bab 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -1,5 +1,5 @@ # sql/compiler.py -# Copyright (C) 2005-2025 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under @@ -40,10 +40,12 @@ from typing import cast from typing import ClassVar from typing import Dict +from typing import Final from typing import FrozenSet from typing import Iterable from typing import Iterator from typing import List +from typing import Literal from typing import Mapping from typing import MutableMapping from typing import NamedTuple @@ -76,19 +78,14 @@ from .base import _from_objects from .base import _NONE_NAME from .base import _SentinelDefaultCharacterization -from .base import Executable from .base import NO_ARG -from .elements import ClauseElement from .elements import quoted_name -from .schema import Column from .sqltypes import TupleType -from .type_api import TypeEngine from .visitors import prefix_anon_map -from .visitors import Visitable from .. import exc from .. import util from ..util import FastIntFlag -from ..util.typing import Literal +from ..util.typing import Self from ..util.typing import TupleAny from ..util.typing import Unpack @@ -96,18 +93,40 @@ from .annotation import _AnnotationDict from .base import _AmbiguousTableNameMap from .base import CompileState + from .base import Executable + from .base import ExecutableStatement from .cache_key import CacheKey + from .ddl import _TableViaSelect + from .ddl import CreateTableAs + from .ddl import CreateView from .ddl import ExecutableDDLElement + from .dml import Delete from .dml import Insert + from .dml import Update from .dml import UpdateBase + from .dml import UpdateDMLState from .dml import ValuesBase from .elements import _truncated_label + from .elements import BinaryExpression from .elements import BindParameter + from .elements import ClauseElement from .elements import ColumnClause from .elements import ColumnElement + from .elements import False_ from .elements import Label + from .elements import Null + from .elements import True_ from .functions import Function + from .schema import CheckConstraint + from .schema import Column + from .schema import Constraint + from .schema import ForeignKeyConstraint + from .schema import IdentityOptions + from .schema import Index + from .schema import PrimaryKeyConstraint from .schema import Table + from .schema import UniqueConstraint + from .selectable import _ColumnsClauseElement from .selectable import AliasedReturnsRows from .selectable import CompoundSelectState from .selectable import CTE @@ -117,6 +136,10 @@ from .selectable import Select from .selectable import SelectState from .type_api import _BindProcessorType + from .type_api import TypeDecorator + from .type_api import TypeEngine + from .type_api import UserDefinedType + from .visitors import Visitable from ..engine.cursor import CursorResultMetaData from ..engine.interfaces import _CoreSingleExecuteParams from ..engine.interfaces import _DBAPIAnyExecuteParams @@ -128,6 +151,7 @@ from ..engine.interfaces import Dialect from ..engine.interfaces import SchemaTranslateMapType + _FromHintsType = Dict["FromClause", str] RESERVED_WORDS = { @@ -238,6 +262,9 @@ r"^(?:RESTRICT|CASCADE|SET NULL|NO ACTION|SET DEFAULT)$", re.I ) FK_INITIALLY = re.compile(r"^(?:DEFERRED|IMMEDIATE)$", re.I) +_WINDOW_EXCLUDE_RE = re.compile( + r"^(?:CURRENT ROW|GROUP|TIES|NO OTHERS)$", re.I +) BIND_PARAMS = re.compile(r"(? None: def _init_compiler_cls(cls): pass - def _execute_on_connection( - self, connection, distilled_params, execution_options - ): - if self.can_execute: - return connection._execute_compiled( - self, distilled_params, execution_options - ) - else: - raise exc.ObjectNotExecutableError(self.statement) - def visit_unsupported_compilation(self, element, err, **kw): raise exc.UnsupportedCompilationError(self, type(element)) from err @property - def sql_compiler(self): + def sql_compiler(self) -> SQLCompiler: """Return a Compiled that is capable of processing SQL expressions. If this compiler is one, it would likely just return 'self'. @@ -994,6 +1048,39 @@ def self_group(self, **kw): return self +class aggregate_orderby_inline( + roles.BinaryElementRole[Any], elements.CompilerColumnElement +): + """produce ORDER BY inside of function argument lists""" + + __visit_name__ = "aggregate_orderby_inline" + __slots__ = "element", "aggregate_order_by" + + def __init__(self, element, orderby): + self.element = element + self.aggregate_order_by = orderby + + def __iter__(self): + return iter(self.element) + + @property + def proxy_set(self): + return self.element.proxy_set + + @property + def type(self): + return self.element.type + + def self_group(self, **kw): + return self + + def _with_binary_element_type(self, type_): + return aggregate_orderby_inline( + self.element._with_binary_element_type(type_), + self.aggregate_order_by, + ) + + class ilike_case_insensitive( roles.BinaryElementRole[Any], elements.CompilerColumnElement ): @@ -1328,6 +1415,8 @@ class SQLCompiler(Compiled): _positional_pattern = re.compile( f"{_pyformat_pattern.pattern}|{_post_compile_pattern.pattern}" ) + _collect_params: Final[bool] + _collected_params: util.immutabledict[str, Any] @classmethod def _init_compiler_cls(cls): @@ -1427,6 +1516,11 @@ def __init__( # dialect.label_length or dialect.max_identifier_length self.truncated_names: Dict[Tuple[str, str], str] = {} self._truncated_counters: Dict[str, int] = {} + if not cache_key: + self._collect_params = True + self._collected_params = util.EMPTY_DICT + else: + self._collect_params = False # type: ignore[misc] Compiled.__init__(self, dialect, statement, **kwargs) @@ -1491,8 +1585,6 @@ def insert_single_values_expr(self) -> Optional[str]: a VALUES expression, the string is assigned here, where it can be used for insert batching schemes to rewrite the VALUES expression. - .. versionadded:: 1.3.8 - .. versionchanged:: 2.0 This collection is no longer used by SQLAlchemy's built-in dialects, in favor of the currently internal ``_insertmanyvalues`` collection that is used only by @@ -1553,19 +1645,6 @@ def current_executable(self): by a ``visit_`` method, as it is not guaranteed to be assigned nor guaranteed to correspond to the current statement being compiled. - .. versionadded:: 1.3.21 - - For compatibility with previous versions, use the following - recipe:: - - statement = getattr(self, "current_executable", False) - if statement is False: - statement = self.stack[-1]["selectable"] - - For versions 1.4 and above, ensure only .current_executable - is used; the format of "self.stack" may change. - - """ try: return self.stack[-1]["selectable"] @@ -1580,6 +1659,13 @@ def prefetch(self): def _global_attributes(self) -> Dict[Any, Any]: return {} + def _add_to_params(self, item: ExecutableStatement) -> None: + # assumes that this is called before traversing the statement + # so the call happens outer to inner, meaning that existing params + # take precedence + if item._params: + self._collected_params = item._params | self._collected_params + @util.memoized_instancemethod def _init_cte_state(self) -> MutableMapping[CTE, str]: """Initialize collections related to CTEs only if @@ -1793,7 +1879,7 @@ def is_subquery(self): return len(self.stack) > 1 @property - def sql_compiler(self): + def sql_compiler(self) -> Self: return self def construct_expanded_state( @@ -1827,8 +1913,19 @@ def construct_params( _group_number: Optional[int] = None, _check: bool = True, _no_postcompile: bool = False, + _collected_params: _CoreSingleExecuteParams | None = None, ) -> _MutableCoreSingleExecuteParams: """return a dictionary of bind parameter keys and values""" + if _collected_params is not None: + assert not self._collect_params + elif self._collect_params: + _collected_params = self._collected_params + + if _collected_params: + if not params: + params = _collected_params + else: + params = {**_collected_params, **params} if self._render_postcompile and not _no_postcompile: assert self._post_compile_expanded_state is not None @@ -2299,10 +2396,7 @@ def get(lastrowid, parameters): @util.memoized_property @util.preload_module("sqlalchemy.engine.result") def _inserted_primary_key_from_returning_getter(self): - if typing.TYPE_CHECKING: - from ..engine import result - else: - result = util.preloaded.engine_result + result = util.preloaded.engine_result assert self.compile_state is not None statement = self.compile_state.statement @@ -2344,7 +2438,7 @@ def get(row, parameters): return get - def default_from(self): + def default_from(self) -> str: """Called when a SELECT statement has no froms, and no FROM clause is to be appended. @@ -2524,8 +2618,15 @@ def visit_label( within_columns_clause=False, render_label_as_label=None, result_map_targets=(), + within_tstring=False, **kw, ): + if within_tstring: + raise exc.CompileError( + "Using label() directly inside tstring is not supported " + "as it is ambiguous how the label expression should be " + "rendered without knowledge of how it's being used in SQL" + ) # only render labels within the columns clause # or ORDER BY clause of a select. dialect-specific compilers # can modify this behavior. @@ -2660,6 +2761,9 @@ def escape_literal_column(self, text): return text def visit_textclause(self, textclause, add_to_result_map=None, **kw): + if self._collect_params: + self._add_to_params(textclause) + def do_bindparam(m): name = m.group(1) if name in textclause._bindparams: @@ -2684,9 +2788,28 @@ def do_bindparam(m): ), ) + def visit_tstring(self, tstring, add_to_result_map=None, **kw): + if self._collect_params: + self._add_to_params(tstring) + + if not self.stack: + self.isplaintext = True + + if add_to_result_map: + # tstring() object is present in the columns clause of a + # select(). Add a no-name entry to the result map so that + # row[tstring()] produces a result + add_to_result_map(None, None, (tstring,), sqltypes.NULLTYPE) + + # Process each part and concatenate + kw["within_tstring"] = True + return "".join(self.process(part, **kw) for part in tstring.parts) + def visit_textual_select( self, taf, compound_index=None, asfrom=False, **kw ): + if self._collect_params: + self._add_to_params(taf) toplevel = not self.stack entry = self._default_stack_entry if toplevel else self.stack[-1] @@ -2736,16 +2859,16 @@ def visit_textual_select( return text - def visit_null(self, expr, **kw): + def visit_null(self, expr: Null, **kw: Any) -> str: return "NULL" - def visit_true(self, expr, **kw): + def visit_true(self, expr: True_, **kw: Any) -> str: if self.dialect.supports_native_boolean: return "true" else: return "1" - def visit_false(self, expr, **kw): + def visit_false(self, expr: False_, **kw: Any) -> str: if self.dialect.supports_native_boolean: return "false" else: @@ -2781,6 +2904,9 @@ def visit_tuple(self, clauselist, **kw): def visit_element_list(self, element, **kw): return self._generate_delimited_list(element.clauses, " ", **kw) + def visit_order_by_list(self, element, **kw): + return self._generate_delimited_list(element.clauses, ", ", **kw) + def visit_clauselist(self, clauselist, **kw): sep = clauselist.operator if sep is None: @@ -2842,30 +2968,24 @@ def visit_cast(self, cast, **kwargs): def visit_frame_clause(self, frameclause, **kw): - if frameclause.lower_type is elements._FrameClauseType.RANGE_UNBOUNDED: + if frameclause.lower_type is elements.FrameClauseType.UNBOUNDED: left = "UNBOUNDED PRECEDING" - elif frameclause.lower_type is elements._FrameClauseType.RANGE_CURRENT: + elif frameclause.lower_type is elements.FrameClauseType.CURRENT: left = "CURRENT ROW" else: - val = self.process(frameclause.lower_integer_bind, **kw) - if ( - frameclause.lower_type - is elements._FrameClauseType.RANGE_PRECEDING - ): + val = self.process(frameclause.lower_bind, **kw) + if frameclause.lower_type is elements.FrameClauseType.PRECEDING: left = f"{val} PRECEDING" else: left = f"{val} FOLLOWING" - if frameclause.upper_type is elements._FrameClauseType.RANGE_UNBOUNDED: + if frameclause.upper_type is elements.FrameClauseType.UNBOUNDED: right = "UNBOUNDED FOLLOWING" - elif frameclause.upper_type is elements._FrameClauseType.RANGE_CURRENT: + elif frameclause.upper_type is elements.FrameClauseType.CURRENT: right = "CURRENT ROW" else: - val = self.process(frameclause.upper_integer_bind, **kw) - if ( - frameclause.upper_type - is elements._FrameClauseType.RANGE_PRECEDING - ): + val = self.process(frameclause.upper_bind, **kw) + if frameclause.upper_type is elements.FrameClauseType.PRECEDING: right = f"{val} PRECEDING" else: right = f"{val} FOLLOWING" @@ -2878,9 +2998,16 @@ def visit_over(self, over, **kwargs): range_ = f"RANGE BETWEEN {self.process(over.range_, **kwargs)}" elif over.rows is not None: range_ = f"ROWS BETWEEN {self.process(over.rows, **kwargs)}" + elif over.groups is not None: + range_ = f"GROUPS BETWEEN {self.process(over.groups, **kwargs)}" else: range_ = None + if range_ is not None and over.exclude is not None: + range_ += " EXCLUDE " + self.preparer.validate_sql_phrase( + over.exclude, _WINDOW_EXCLUDE_RE + ) + return "%s OVER (%s)" % ( text, " ".join( @@ -2909,6 +3036,62 @@ def visit_funcfilter(self, funcfilter, **kwargs): funcfilter.criterion._compiler_dispatch(self, **kwargs), ) + def visit_aggregateorderby(self, aggregateorderby, **kwargs): + if self.dialect.aggregate_order_by_style is AggregateOrderByStyle.NONE: + raise exc.CompileError( + "this dialect does not support " + "ORDER BY within an aggregate function" + ) + elif ( + self.dialect.aggregate_order_by_style + is AggregateOrderByStyle.INLINE + ): + new_fn = aggregateorderby.element._clone() + new_fn.clause_expr = elements.Grouping( + aggregate_orderby_inline( + new_fn.clause_expr.element, aggregateorderby.order_by + ) + ) + + return new_fn._compiler_dispatch(self, **kwargs) + else: + return self.visit_withingroup(aggregateorderby, **kwargs) + + def visit_aggregate_orderby_inline(self, element, **kw): + return "%s ORDER BY %s" % ( + self.process(element.element, **kw), + self.process(element.aggregate_order_by, **kw), + ) + + def visit_aggregate_strings_func(self, fn, *, use_function_name, **kw): + # aggreagate_order_by attribute is present if visit_function + # gave us a Function with aggregate_orderby_inline() as the inner + # contents + order_by = getattr(fn.clauses, "aggregate_order_by", None) + + literal_exec = dict(kw) + literal_exec["literal_execute"] = True + + # break up the function into its components so we can apply + # literal_execute to the second argument (the delimiter) + cl = list(fn.clauses) + expr, delimiter = cl[0:2] + if ( + order_by is not None + and self.dialect.aggregate_order_by_style + is AggregateOrderByStyle.INLINE + ): + return ( + f"{use_function_name}({expr._compiler_dispatch(self, **kw)}, " + f"{delimiter._compiler_dispatch(self, **literal_exec)} " + f"ORDER BY {order_by._compiler_dispatch(self, **kw)})" + ) + else: + return ( + f"{use_function_name}({expr._compiler_dispatch(self, **kw)}, " + f"{delimiter._compiler_dispatch(self, **literal_exec)})" + ) + def visit_extract(self, extract, **kwargs): field = self.extract_map.get(extract.field, extract.field) return "EXTRACT(%s FROM %s)" % ( @@ -2927,6 +3110,8 @@ def visit_function( add_to_result_map: Optional[_ResultMapAppender] = None, **kwargs: Any, ) -> str: + if self._collect_params: + self._add_to_params(func) if add_to_result_map is not None: add_to_result_map(func.name, func.name, (func.name,), func.type) @@ -2934,6 +3119,8 @@ def visit_function( text: str + kwargs["within_aggregate_function"] = True + if disp: text = disp(func, **kwargs) else: @@ -2976,12 +3163,14 @@ def visit_sequence(self, sequence, **kw): % self.dialect.name ) - def function_argspec(self, func, **kwargs): + def function_argspec(self, func: Function[Any], **kwargs: Any) -> str: return func.clause_expr._compiler_dispatch(self, **kwargs) def visit_compound_select( self, cs, asfrom=False, compound_index=None, **kwargs ): + if self._collect_params: + self._add_to_params(cs) toplevel = not self.stack compile_state = cs._compile_state_factory(cs, self, **kwargs) @@ -3058,6 +3247,10 @@ def _get_operator_dispatch(self, operator_, qualifier1, qualifier2): ) return getattr(self, attrname, None) + def _get_custom_operator_dispatch(self, operator_, qualifier1): + attrname = "visit_%s_op_%s" % (operator_.visit_name, qualifier1) + return getattr(self, attrname, None) + def visit_unary( self, unary, add_to_result_map=None, result_map_targets=(), **kw ): @@ -3066,6 +3259,16 @@ def visit_unary( kw["add_to_result_map"] = add_to_result_map kw["result_map_targets"] = result_map_targets + if unary.operator is operators.distinct_op and not kw.get( + "within_aggregate_function", False + ): + util.warn( + "Column-expression-level unary distinct() " + "should not be used outside of an aggregate " + "function. For general 'SELECT DISTINCT' support" + "use select().distinct()." + ) + if unary.operator: if unary.modifier: raise exc.CompileError( @@ -3422,6 +3625,11 @@ def visit_mod_binary(self, binary, operator, **kw): ) def visit_custom_op_binary(self, element, operator, **kw): + if operator.visit_name: + disp = self._get_custom_operator_dispatch(operator, "binary") + if disp: + return disp(element, operator, **kw) + kw["eager_grouping"] = operator.eager_grouping return self._generate_generic_binary( element, @@ -3430,18 +3638,32 @@ def visit_custom_op_binary(self, element, operator, **kw): ) def visit_custom_op_unary_operator(self, element, operator, **kw): + if operator.visit_name: + disp = self._get_custom_operator_dispatch(operator, "unary") + if disp: + return disp(element, operator, **kw) + return self._generate_generic_unary_operator( element, self.escape_literal_column(operator.opstring) + " ", **kw ) def visit_custom_op_unary_modifier(self, element, operator, **kw): + if operator.visit_name: + disp = self._get_custom_operator_dispatch(operator, "unary") + if disp: + return disp(element, operator, **kw) + return self._generate_generic_unary_modifier( element, " " + self.escape_literal_column(operator.opstring), **kw ) def _generate_generic_binary( - self, binary, opstring, eager_grouping=False, **kw - ): + self, + binary: BinaryExpression[Any], + opstring: str, + eager_grouping: bool = False, + **kw: Any, + ) -> str: _in_operator_expression = kw.get("_in_operator_expression", False) kw["_in_operator_expression"] = True @@ -3610,24 +3832,40 @@ def visit_not_between_op_binary(self, binary, operator, **kw): **kw, ) - def visit_regexp_match_op_binary(self, binary, operator, **kw): + def visit_regexp_match_op_binary( + self, binary: BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: raise exc.CompileError( "%s dialect does not support regular expressions" % self.dialect.name ) - def visit_not_regexp_match_op_binary(self, binary, operator, **kw): + def visit_not_regexp_match_op_binary( + self, binary: BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: raise exc.CompileError( "%s dialect does not support regular expressions" % self.dialect.name ) - def visit_regexp_replace_op_binary(self, binary, operator, **kw): + def visit_regexp_replace_op_binary( + self, binary: BinaryExpression[Any], operator: Any, **kw: Any + ) -> str: raise exc.CompileError( "%s dialect does not support regular expression replacements" % self.dialect.name ) + def visit_dmltargetcopy(self, element, *, bindmarkers=None, **kw): + if bindmarkers is None: + raise exc.CompileError( + "DML target objects may only be used with " + "compiled INSERT or UPDATE statements" + ) + + bindmarkers[element.column.key] = element + return f"__BINDMARKER_~~{element.column.key}~~" + def visit_bindparam( self, bindparam, @@ -3636,8 +3874,19 @@ def visit_bindparam( skip_bind_expression=False, literal_execute=False, render_postcompile=False, + is_upsert_set=False, **kwargs, ): + # Detect parametrized bindparams in upsert SET clause for issue #13130 + if ( + is_upsert_set + and bindparam.value is None + and bindparam.callable is None + and self._insertmanyvalues is not None + ): + self._insertmanyvalues = self._insertmanyvalues._replace( + has_upsert_bound_parameters=True + ) if not skip_bind_expression: impl = bindparam.type.dialect_impl(self.dialect) @@ -3829,7 +4078,9 @@ def render_literal_bindparam( else: return self.render_literal_value(value, bindparam.type) - def render_literal_value(self, value, type_): + def render_literal_value( + self, value: Any, type_: sqltypes.TypeEngine[Any] + ) -> str: """Render the value of a bind parameter as a quoted literal. This is used for statement sections that do not accept bind parameters @@ -4119,7 +4370,7 @@ def visit_cte( if cte.recursive: self.ctes_recursive = True text = self.preparer.format_alias(cte, cte_name) - if cte.recursive: + if cte.recursive or cte.element.name_cte_columns: col_source = cte.element # TODO: can we get at the .columns_plus_names collection @@ -4188,7 +4439,7 @@ def visit_cte( if self.preparer._requires_quotes(cte_name): cte_name = self.preparer.quote(cte_name) text += self.get_render_as_alias_suffix(cte_name) - return text + return text # type: ignore[no-any-return] else: return self.preparer.format_alias(cte, cte_name) @@ -4216,6 +4467,7 @@ def visit_alias( lateral=False, enclosing_alias=None, from_linter=None, + within_tstring=False, **kwargs, ): if lateral: @@ -4250,9 +4502,9 @@ def visit_alias( inner = "(%s)" % (inner,) return inner else: - enclosing_alias = kwargs["enclosing_alias"] = alias + kwargs["enclosing_alias"] = alias - if asfrom or ashint: + if asfrom or ashint or within_tstring: if isinstance(alias.name, elements._truncated_label): alias_name = self._truncated_identifier("alias", alias.name) else: @@ -4260,7 +4512,7 @@ def visit_alias( if ashint: return self.preparer.format_alias(alias, alias_name) - elif asfrom: + elif asfrom or within_tstring: if from_linter: from_linter.froms[alias._de_clone()] = alias_name @@ -4340,7 +4592,13 @@ def _render_values(self, element, **kw): ) return f"VALUES {tuples}" - def visit_values(self, element, asfrom=False, from_linter=None, **kw): + def visit_values( + self, element, asfrom=False, from_linter=None, visiting_cte=None, **kw + ): + + if element._independent_ctes: + self._dispatch_independent_ctes(element, kw) + v = self._render_values(element, **kw) if element._unnamed: @@ -4361,7 +4619,12 @@ def visit_values(self, element, asfrom=False, from_linter=None, **kw): name if name is not None else "(unnamed VALUES element)" ) - if name: + if visiting_cte is not None and visiting_cte.element is element: + if element._is_lateral: + raise exc.CompileError( + "Can't use a LATERAL VALUES expression inside of a CTE" + ) + elif name: kw["include_table"] = False v = "%s(%s)%s (%s)" % ( lateral, @@ -4493,6 +4756,8 @@ def add_to_result_map(keyname, name, objects, type_): "_label_select_column is only relevant within " "the columns clause of a SELECT or RETURNING" ) + result_expr: elements.Label[Any] | _CompileLabel + if isinstance(column, elements.Label): if col_expr is not column: result_expr = _CompileLabel( @@ -4545,10 +4810,55 @@ def add_to_result_map(keyname, name, objects, type_): elif isinstance(column, elements.TextClause): render_with_label = False elif isinstance(column, elements.UnaryExpression): - render_with_label = column.wraps_column_expression or asfrom + # unary expression. notes added as of #12681 + # + # By convention, the visit_unary() method + # itself does not add an entry to the result map, and relies + # upon either the inner expression creating a result map + # entry, or if not, by creating a label here that produces + # the result map entry. Where that happens is based on whether + # or not the element immediately inside the unary is a + # NamedColumn subclass or not. + # + # Now, this also impacts how the SELECT is written; if + # we decide to generate a label here, we get the usual + # "~(x+y) AS anon_1" thing in the columns clause. If we + # don't, we don't get an AS at all, we get like + # "~table.column". + # + # But here is the important thing as of modernish (like 1.4) + # versions of SQLAlchemy - **whether or not the AS