diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 00000000000..27b6ee0a0b4 --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,175 @@ +name: build-and-release + +on: + push: + tags: + - 'AliSQL-8.0.*' + - 'AliSQL-5.7.*' + +permissions: + contents: write + +jobs: + parse: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.p.outputs.version }} + major: ${{ steps.p.outputs.major }} + minor: ${{ steps.p.outputs.minor }} + patch: ${{ steps.p.outputs.patch }} + extra: ${{ steps.p.outputs.extra }} + steps: + - id: p + run: | + set -eu + TAG="${GITHUB_REF_NAME}" + VERSION="${TAG#AliSQL-}" + MAJOR="${VERSION%%.*}" + REST="${VERSION#*.}" + MINOR="${REST%%.*}" + REST="${REST#*.}" + PATCH="${REST%%-*}" + EXTRA="${REST#*-}" + { + echo "version=$VERSION" + echo "major=$MAJOR" + echo "minor=$MINOR" + echo "patch=$PATCH" + echo "extra=$EXTRA" + } >> "$GITHUB_OUTPUT" + echo "Parsed: version=$VERSION major=$MAJOR minor=$MINOR patch=$PATCH extra=$EXTRA" + + build: + needs: parse + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runner: ubuntu-24.04 + cmake_arch: x86_64 + - arch: aarch64 + runner: ubuntu-24.04-arm + cmake_arch: aarch64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Build inside oraclelinux:7 + env: + MAJOR: ${{ needs.parse.outputs.major }} + MINOR: ${{ needs.parse.outputs.minor }} + PATCH: ${{ needs.parse.outputs.patch }} + EXTRA: ${{ needs.parse.outputs.extra }} + CMAKE_ARCH: ${{ matrix.cmake_arch }} + run: | + set -eux + mkdir -p "${GITHUB_WORKSPACE}/out" + docker run --rm -i \ + -v "${GITHUB_WORKSPACE}:/AliSQL" \ + -v "${GITHUB_WORKSPACE}/out:/out" \ + -e MAJOR -e MINOR -e PATCH -e EXTRA -e CMAKE_ARCH \ + oraclelinux:7 \ + bash -ex <<'INNER' + + # ---- Toolchain repo (devtoolset-10) ---- + cat >/etc/yum.repos.d/ol7-scl.repo <<'REPO' + [ol7_software_collections] + name=Oracle Linux 7 Software Collections ($basearch) + baseurl=https://yum.oracle.com/repo/OracleLinux/OL7/SoftwareCollections/$basearch/ + gpgcheck=1 + enabled=1 + gpgkey=https://yum.oracle.com/RPM-GPG-KEY-oracle-ol7 + REPO + + yum clean all + yum makecache + yum install -y git wget \ + devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-binutils \ + openssl openssl-devel ncurses-devel libaio-devel perl-IPC-Cmd bison + + # ---- CMake ---- + cd /opt + wget -q "https://github.com/Kitware/CMake/releases/download/v3.28.6/cmake-3.28.6-linux-${CMAKE_ARCH}.tar.gz" + tar -xzf "cmake-3.28.6-linux-${CMAKE_ARCH}.tar.gz" + export PATH="/opt/cmake-3.28.6-linux-${CMAKE_ARCH}/bin:$PATH" + cmake --version + + # ---- Inject version from tag ---- + # 8.0 branch uses MYSQL_VERSION; 5.7 branch uses VERSION; same internal format. + cd /AliSQL + VERSION_FILE="" + for f in MYSQL_VERSION VERSION; do + [ -f "$f" ] && VERSION_FILE="$f" && break + done + if [ -z "$VERSION_FILE" ]; then + echo "ERROR: neither MYSQL_VERSION nor VERSION found in source root" >&2 + exit 1 + fi + sed -i \ + -e "s/^MYSQL_VERSION_MAJOR=.*/MYSQL_VERSION_MAJOR=${MAJOR}/" \ + -e "s/^MYSQL_VERSION_MINOR=.*/MYSQL_VERSION_MINOR=${MINOR}/" \ + -e "s/^MYSQL_VERSION_PATCH=.*/MYSQL_VERSION_PATCH=${PATCH}/" \ + -e "s/^MYSQL_VERSION_EXTRA=.*/MYSQL_VERSION_EXTRA=${EXTRA}/" \ + "$VERSION_FILE" + echo "--- ${VERSION_FILE} after injection ---" + cat "$VERSION_FILE" + + # ---- Build & install ---- + source scl_source enable devtoolset-10 + sh build.sh -t release -d /opt/alisql + make install + + # ---- Package ---- + cd /opt + VER="${MAJOR}.${MINOR}.${PATCH}-${EXTRA}" + GLIBC="glibc$(ldd --version | awk 'NR==1{print $NF}')" + ARCH="$(uname -m)" + PKG="alisql-${VER}-linux-${GLIBC}-${ARCH}.tar.xz" + TOPDIR="${PKG%.tar.xz}" + XZ_OPT='-T0 -9' tar -cJf "/out/${PKG}" \ + --transform="s,^alisql,${TOPDIR}," \ + --exclude='alisql/mysql-test' \ + --exclude='alisql/run' \ + --exclude='alisql/var' \ + --exclude='alisql/LICENSE-test' \ + --exclude='alisql/LICENSE.router' \ + --exclude='alisql/README-test' \ + --exclude='alisql/README.router' \ + --exclude='alisql/mysqlrouter-log-rotate' \ + alisql/ + ls -lh "/out/${PKG}" + INNER + + - uses: actions/upload-artifact@v4 + with: + name: alisql-${{ matrix.arch }} + path: ${{ github.workspace }}/out/alisql-*.tar.xz + retention-days: 7 + + release: + needs: [parse, build] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + + - name: List artifacts + run: ls -lh artifacts/ + + - name: Create release with tarballs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + VERSION: ${{ needs.parse.outputs.version }} + run: | + gh release create "$TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --title "$TAG" \ + --notes "Auto-built AliSQL ${VERSION} (oraclelinux:7, glibc 2.17). x86_64 and aarch64 binaries attached." \ + artifacts/*.tar.xz diff --git a/.github/workflows/trigger-alisql-docker-build.yml b/.github/workflows/trigger-alisql-docker-build.yml new file mode 100644 index 00000000000..62ce01a3acb --- /dev/null +++ b/.github/workflows/trigger-alisql-docker-build.yml @@ -0,0 +1,31 @@ +name: trigger-alisql-docker-build + +on: + release: + types: [published] + +jobs: + trigger: + runs-on: ubuntu-latest + if: | + startsWith(github.event.release.tag_name, 'AliSQL-8.0.') || + startsWith(github.event.release.tag_name, 'AliSQL-5.7.') + steps: + - name: Push tag to alisql-docker + env: + PAT: ${{ secrets.DOCKER_REPO_PAT }} + TAG: ${{ github.event.release.tag_name }} + run: | + set -eux + git clone --depth 1 "https://x-access-token:${PAT}@github.com/p1p1bear/alisql-docker.git" work + cd work + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + if git ls-remote --tags origin "refs/tags/${TAG}" | grep -q "refs/tags/${TAG}$"; then + echo "Tag ${TAG} already exists on alisql-docker, skipping." + exit 0 + fi + + git tag "${TAG}" + git push origin "${TAG}" diff --git a/README.md b/README.md index bb1e316aa44..daba747e687 100644 --- a/README.md +++ b/README.md @@ -1,95 +1,278 @@ -# AliSQL +

+ AliSQL Logo +

-AliSQL is Alibaba's MySQL branch, forked from official MySQL and used extensively in Alibaba Group's production environment. It includes various performance optimizations, stability improvements, and features tailored for large-scale applications. +

AliSQL

-- [AliSQL](#alisql) - - [Version Information](#version-information) - - [Features](#features) - - [Roadmap](#roadmap) - - [Getting Started](#getting-started) - - [Support](#support) - - [Contributing](#contributing) - - [License](#license) - - [See Also](#see-also) +

+ Alibaba's Enterprise MySQL Branch with DuckDB OLAP & Native Vector Search +

-## Version Information +

+ Battle-tested in Alibaba's production environment, powering millions of databases +

-- **AliSQL Version**: 8.0.44 (LTS) -- **Based on**: MySQL 8.0.44 +

+ GitHub Stars + GitHub Forks + License + MySQL Version +

-## Features +

+ Features • + Quick Start • + Docs • + Roadmap • + Contributing +

-- **[DuckDB Storage Engine](./wiki/duckdb/duckdb.md)**:AliSQL integrates DuckDB as a native storage engine, allowing users to operate DuckDB with the same experience as MySQL. By leveraging AliSQL for rapid deployment of DuckDB service nodes, users can easily achieve lightweight analytical capabilities. +

+ 简体中文 | English +

-## Roadmap -- **[Vector Storage](https://www.alibabacloud.com/help/en/rds/apsaradb-rds-for-mysql/vector-storage-1?spm=a2c63.p38356.help-menu-26090.d_3_3_0.6bb8d111D06xOW)** *(planned)*:AliSQL natively supports enterprise-grade vector processing for up to 16,383 dimensions. By integrating a highly optimized HNSW algorithm for high-performance Approximate Nearest Neighbor (ANN) search, AliSQL empowers users to build AI-driven applications—such as semantic search and recommendation systems—seamlessly using standard SQL interfaces. +## Why AliSQL? + +AliSQL brings enterprise-grade capabilities to MySQL, combining the reliability of InnoDB OLTP with DuckDB's blazing-fast analytics and native vector search — all through familiar MySQL interfaces. + + + + + + + +
+ +### 200x Faster Analytics + +DuckDB columnar engine delivers **200x speedup** on analytical queries compared to InnoDB + + + +### Native Vector Search + +Built-in HNSW algorithm supporting up to **16,383 dimensions** for AI/ML workloads + + -- **[DDL Optimization](https://www.alibabacloud.com/help/en/rds/apsaradb-rds-for-mysql/alisql-ddl-best-practices?spm=a2c63.p38356.help-menu-26090.d_2_8_0.1f7a28a5F1ZVeK)** *(planned)*:AliSQL delivers a faster, safer, and lighter DDL experience through innovations such as enhanced Instant DDL, parallel B+tree construction, a non-blocking lock mechanism, and real-time DDL apply—significantly improving schema change efficiency and virtually eliminating replication lag. +### 100% MySQL Compatible -- **[RTO Optimization](https://www.alibabacloud.com/help/en/rds/apsaradb-rds-for-mysql/best-practices-for-rto-optimization-in-alisql?spm=a3c0i.36496430.J_TlTAa0s_LXHOq4tuiO-gv.1.43c56e9bd5YdDQ&scm=20140722.S_help@@%E6%96%87%E6%A1%A3@@2880006._.ID_help@@%E6%96%87%E6%A1%A3@@2880006-RL_RDSMySQLRTO-LOC_2024SPAllResult-OR_ser-PAR1_0bc3b4af17685488697221621e29f2-V_4-PAR3_r-RE_new5-P0_0-P1_0)** *(planned)*:AliSQL deeply optimizes the end-to-end crash recovery path to accelerate instance startup, shorten RTO, and restore service quickly. +Use your existing MySQL tools, drivers, and SQL — zero learning curve -- **[Replication Optimization](https://www.alibabacloud.com/help/en/rds/apsaradb-rds-for-mysql/replication-optimization/?spm=a2c63.p38356.help-menu-26090.d_2_6.48a125033Ze9gw)** *(palned)*: AliSQL significantly boosts replication throughput and minimizes lag by implementing Binlog Parallel Flush, Binlog in Redo, and specialized optimizations for large transactions and DDL operations. +
-## Getting Started -**Prerequisites**: -- [CMake](https://cmake.org) 3.x or higher -- Python3 -- C++17 compliant compiler (GCC 7+ or Clang 5+) +## Key Features -**Build Instructions**: +| Feature | Description | Status | +|---------|-------------|--------| +| **DuckDB Storage Engine** | Columnar OLAP engine with automatic compression, perfect for analytics workloads | Available | +| **Vector Index (VIDX)** | Native vector storage & ANN search with HNSW, supports COSINE & EUCLIDEAN distance | Available | +| **DDL Optimization** | Instant DDL, parallel B+tree construction, non-blocking locks | Planned | +| **RTO Optimization** | Accelerated crash recovery for faster instance startup | Planned | +| **Replication Boost** | Binlog Parallel Flush, Binlog in Redo, large transaction optimization | Planned | + +## Quick Start + +### Option 1: Build from Source ```bash # Clone the repository git clone https://github.com/alibaba/AliSQL.git cd AliSQL -# Build the project (release build) -sh build.sh -t release -d /path/to/install/dir +# Build (release mode) +sh build.sh -t release -d ~/alisql -# For development/debugging (debug build) -sh build.sh -t debug -d /path/to/install/dir - -# Install the built MySQL server +# Install make install ``` -**Build Options**: -- `-t release|debug`: Build type (default: debug) -- `-d `: Installation directory (default: /usr/local/alisql or $HOME/alisql) -- `-s `: Server suffix (default: alisql-dev) -- `-g asan|tsan`: Enable sanitizer -- `-c`: Enable GCC coverage (gcov) -- `-h, --help`: Show help +### Option 2: Set Up a DuckDB Analytical Node -## Support -- **GitHub Issues**: [https://github.com/alibaba/AliSQL/issues](https://github.com/alibaba/AliSQL/issues) -- **Alibaba Cloud RDS**: [DuckDB-based Analytical Instance](https://help.aliyun.com/zh/rds/apsaradb-rds-for-mysql/duckdb-based-analytical-instance/) +> **Step-by-step guide:** [How to set up a DuckDB node](./wiki/duckdb/how-to-setup-duckdb-node-en.md) -> For DuckDB-specific support, see the [DuckDB Support Options](https://duckdblabs.com/support/). +### Initialize & Start Server -## Contributing +```bash +# Initialize data directory +~/alisql/bin/mysqld --initialize-insecure --datadir=~/alisql/data -AliSQL 8.0 became an open-source project in December 2025 and is actively maintained by engineers at Alibaba Group. +# Start the server +~/alisql/bin/mysqld --datadir=~/alisql/data +``` -Contributions are welcome! Please: -1. Fork the repository -2. Create a feature branch -3. Make your changes with appropriate tests -4. Submit a pull request +## Usage Examples -For bug reports and feature requests, please use the [GitHub Issues](https://github.com/alibaba/AliSQL/issues) page. +### DuckDB for Analytics -## License +```sql +-- Create an analytical table with DuckDB engine +CREATE TABLE sales_analytics ( + sale_date DATE, + product_id INT, + revenue DECIMAL(10,2), + quantity INT +) ENGINE=DuckDB; -This project is licensed under the GPL-2.0 license. See the [LICENSE](LICENSE) file for details. +-- Run complex analytics (200x faster than InnoDB!) +SELECT + DATE_FORMAT(sale_date, '%Y-%m') as month, + SUM(revenue) as total_revenue, + COUNT(*) as transactions +FROM sales_analytics +GROUP BY month +ORDER BY total_revenue DESC; +``` -AliSQL is based on MySQL, which is licensed under GPL-2.0. The DuckDB integration follows the same licensing terms. +### Vector Search for AI Applications -## See Also -- [AliSQL Release Notes](./wiki/changes-in-alisql-8.0.44.md) -- [DuckDB Storage Engine in AliSQL](./wiki/duckdb/duckdb.md) +```sql +-- Create a table with vector column +CREATE TABLE embeddings ( + id INT PRIMARY KEY, + content TEXT, + embedding VECTOR(768) -- 768-dimensional vectors +) ENGINE=InnoDB; + +-- Create HNSW index for fast ANN search +CREATE VECTOR INDEX idx_embedding ON embeddings(embedding); + +-- Find similar items using cosine distance +SELECT id, content, + COSINE_DISTANCE(embedding, '[0.1, 0.2, ...]') as distance +FROM embeddings +ORDER BY distance +LIMIT 10; +``` + +## Build Options + +| Option | Description | Default | +|--------|-------------|---------| +| `-t release\|debug` | Build type | `debug` | +| `-d ` | Installation directory | `/usr/local/alisql` | +| `-g asan\|tsan` | Enable sanitizer (memory/thread) | disabled | +| `-c` | Enable code coverage (gcov) | disabled | + +**Prerequisites:** CMake 3.x+, Python 3, GCC 7+ or Clang 5+ + +## Roadmap + +``` +Q4 2025 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + [x] DuckDB Storage Engine [x] Vector Index (VIDX) [x] Open Source + +2026 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + [ ] DDL Optimization [ ] RTO Optimization [ ] Replication Boost + - Instant DDL - Fast Crash Recovery - Binlog Parallel Flush + - Parallel B+tree - Minimize RTO - Binlog in Redo + - Non-blocking Locks - Large TX Optimization +``` + +## Documentation + +| Document | Description | +|----------|-------------| +| [DuckDB Integration Guide](./wiki/duckdb/duckdb-en.md) | Complete guide for DuckDB storage engine | +| [Vector Index Guide](./wiki/vidx/vidx_readme.md) | Native vector storage and ANN search | +| [Release Notes](./wiki/changes-in-alisql-8.0.44.md) | What's new in AliSQL 8.0.44 | +| [Setup DuckDB Node](./wiki/duckdb/how-to-setup-duckdb-node-en.md) | Quick setup guide for analytics | + +**External Resources:** - [MySQL 8.0 Documentation](https://dev.mysql.com/doc/refman/8.0/en/) -- [MySQL 8.0 Github Repository](https://github.com/mysql/mysql-server) -- [DuckDB Official Documentation](https://duckdb.org/docs/stable/) -- [DuckDB GitHub Repository](https://github.com/duckdb/duckdb) +- [DuckDB Official Docs](https://duckdb.org/docs/stable/) - [Detailed Article (Chinese)](https://mp.weixin.qq.com/s/_YmlV3vPc9CksumXvXWBEw) + +## Contributing + +AliSQL became open source in December 2025 and is actively maintained by Alibaba Cloud Database Team. + +We welcome contributions of all kinds! + +1. **Fork** the repository +2. **Create** a feature branch (`git checkout -b feature/amazing-feature`) +3. **Commit** your changes (`git commit -m 'Add amazing feature'`) +4. **Push** to the branch (`git push origin feature/amazing-feature`) +5. **Open** a Pull Request + +For bugs and feature requests, please use [GitHub Issues](https://github.com/alibaba/AliSQL/issues). + +## Related Tools + +### RDSAI CLI — AI-Powered Database Assistant + +

+ RDSAI CLI + Python 3.13+ +

+ +[RDSAI CLI](https://github.com/aliyun/rdsai-cli) is a next-generation, AI-powered CLI that transforms how you interact with AliSQL and MySQL databases. Describe your intent in **natural language**, and the AI agent handles the rest. + +```bash +# Install +curl -LsSf https://raw.githubusercontent.com/aliyun/rdsai-cli/main/install.sh | sh + +# Connect and ask in natural language +rdsai --host localhost -u root -p secret -D mydb +mysql> analyze index usage on users table +mysql> show me slow queries from the last hour +mysql> why this query is slow: SELECT * FROM users WHERE name LIKE '%john%' +``` + +**Key Features:** +- Natural language to SQL conversion (English/中文) +- AI-powered query optimization and diagnostics +- Execution plan analysis with `Ctrl+E` +- Multi-model LLM support (Qwen, OpenAI, DeepSeek, Anthropic, etc.) +- Performance benchmarking with automated analysis + +👉 **[Get Started with RDSAI CLI](https://github.com/aliyun/rdsai-cli)** + +## Community & Support + + + + + + +
+ +**GitHub Issues** + +For bug reports & feature requests + +[Open an Issue](https://github.com/alibaba/AliSQL/issues) + + + +**Alibaba Cloud RDS** + +Managed DuckDB analytical instances + +[Learn More](https://help.aliyun.com/zh/rds/apsaradb-rds-for-mysql/duckdb-based-analytical-instance/) + +
+ +## License + +AliSQL is licensed under **GPL-2.0**, the same license as MySQL. + +See the [LICENSE](LICENSE) file for details. + +## Star History + +

+ + Star History Chart + +

+ +

+ Made with care by Alibaba Cloud Database Team +

+ +

+ GitHub • + MySQL • + DuckDB +

diff --git a/README_zh.md b/README_zh.md new file mode 100644 index 00000000000..dde06fd448e --- /dev/null +++ b/README_zh.md @@ -0,0 +1,278 @@ +

+ AliSQL Logo +

+ +

AliSQL

+ +

+ 阿里巴巴企业级 MySQL 分支 - 集成 DuckDB OLAP 引擎与原生向量搜索 +

+ +

+ 经阿里巴巴生产环境大规模验证，支撑数百万数据库实例稳定运行 +

+ +

+ GitHub Stars + GitHub Forks + License + MySQL Version +

+ +

+ 特性 • + 快速开始 • + 文档 • + 路线图 • + 贡献 +

+ +

+ 简体中文 | English +

+ +## 为什么选择 AliSQL？ + +AliSQL 为 MySQL 带来企业级能力，将 InnoDB 可靠的 OLTP 性能与 DuckDB 极速的分析能力和原生向量搜索相结合 - 全部通过熟悉的 MySQL 接口使用。 + + + + + + + +
+ +### 分析性能提升 200 倍 + +DuckDB 列式引擎相比 InnoDB，分析查询性能提升 **200 倍** + + + +### 原生向量搜索 + +内置 HNSW 算法，支持高达 **16,383 维**向量，满足 AI/ML 工作负载 + + + +### 100% MySQL 兼容 + +使用现有的 MySQL 工具、驱动和 SQL - 零学习成本 + +
+ +## 核心特性 + +| 特性 | 描述 | 状态 | +|------|------|------| +| **DuckDB 存储引擎** | 列式 OLAP 引擎，支持自动压缩，专为分析场景设计 | 已发布 | +| **向量索引 (VIDX)** | 原生向量存储与 ANN 搜索，基于 HNSW 算法，支持余弦和欧氏距离 | 已发布 | +| **DDL 优化** | Instant DDL、并行 B+树构建、非阻塞锁机制 | 规划中 | +| **RTO 优化** | 加速崩溃恢复，缩短实例启动时间 | 规划中 | +| **复制增强** | Binlog 并行刷盘、Binlog in Redo、大事务优化 | 规划中 | + +## 快速开始 + +### 方式一：从源码构建 + +```bash +# 克隆仓库 +git clone https://github.com/alibaba/AliSQL.git +cd AliSQL + +# 构建（release 模式） +sh build.sh -t release -d ~/alisql + +# 安装 +make install +``` + +### 方式二：搭建 DuckDB 分析节点 + +> **详细指南：** [如何搭建 DuckDB 节点](./wiki/duckdb/how-to-setup-duckdb-node-cn.md) + +### 初始化并启动服务 + +```bash +# 初始化数据目录 +~/alisql/bin/mysqld --initialize-insecure --datadir=~/alisql/data + +# 启动服务 +~/alisql/bin/mysqld --datadir=~/alisql/data +``` + +## 使用示例 + +### 使用 DuckDB 进行数据分析 + +```sql +-- 创建使用 DuckDB 引擎的分析表 +CREATE TABLE sales_analytics ( + sale_date DATE, + product_id INT, + revenue DECIMAL(10,2), + quantity INT +) ENGINE=DuckDB; + +-- 执行复杂分析查询（比 InnoDB 快 200 倍！） +SELECT + DATE_FORMAT(sale_date, '%Y-%m') as month, + SUM(revenue) as total_revenue, + COUNT(*) as transactions +FROM sales_analytics +GROUP BY month +ORDER BY total_revenue DESC; +``` + +### 使用向量搜索构建 AI 应用 + +```sql +-- 创建包含向量列的表 +CREATE TABLE embeddings ( + id INT PRIMARY KEY, + content TEXT, + embedding VECTOR(768) -- 768 维向量 +) ENGINE=InnoDB; + +-- 创建 HNSW 索引以加速 ANN 搜索 +CREATE VECTOR INDEX idx_embedding ON embeddings(embedding); + +-- 使用余弦距离查找相似项 +SELECT id, content, + COSINE_DISTANCE(embedding, '[0.1, 0.2, ...]') as distance +FROM embeddings +ORDER BY distance +LIMIT 10; +``` + +## 构建选项 + +| 选项 | 描述 | 默认值 | +|------|------|--------| +| `-t release\|debug` | 构建类型 | `debug` | +| `-d <目录>` | 安装目录 | `/usr/local/alisql` | +| `-g asan\|tsan` | 启用内存/线程检测器 | 禁用 | +| `-c` | 启用代码覆盖率 (gcov) | 禁用 | + +**前置依赖：** CMake 3.x+、Python 3、GCC 7+ 或 Clang 5+ + +## 路线图 + +``` +2025 Q4 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + [x] DuckDB 存储引擎 [x] 向量索引 (VIDX) [x] 开源发布 + +2026 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + [ ] DDL 优化 [ ] RTO 优化 [ ] 复制增强 + - Instant DDL - 快速崩溃恢复 - Binlog 并行刷盘 + - 并行 B+树构建 - 最小化 RTO - Binlog in Redo + - 非阻塞锁机制 - 大事务优化 +``` + +## 文档 + +| 文档 | 描述 | +|------|------| +| [DuckDB 集成指南](./wiki/duckdb/duckdb-cn.md) | DuckDB 存储引擎完整使用指南 | +| [向量索引指南](./wiki/vidx/vidx_readme.md) | 原生向量存储与 ANN 搜索 | +| [发布说明](./wiki/changes-in-alisql-8.0.44.md) | AliSQL 8.0.44 新特性 | +| [搭建 DuckDB 节点](./wiki/duckdb/how-to-setup-duckdb-node-cn.md) | 快速搭建分析节点指南 | + +**外部资源：** +- [MySQL 8.0 官方文档](https://dev.mysql.com/doc/refman/8.0/en/) +- [DuckDB 官方文档](https://duckdb.org/docs/stable/) +- [技术详解文章](https://mp.weixin.qq.com/s/_YmlV3vPc9CksumXvXWBEw) + +## 参与贡献 + +AliSQL 于 2025 年 12 月正式开源，由阿里云数据库团队持续维护。 + +我们欢迎各种形式的贡献！ + +1. **Fork** 本仓库 +2. **创建** 功能分支 (`git checkout -b feature/amazing-feature`) +3. **提交** 你的修改 (`git commit -m 'Add amazing feature'`) +4. **推送** 到分支 (`git push origin feature/amazing-feature`) +5. **发起** Pull Request + +如有 Bug 反馈或功能建议，请通过 [GitHub Issues](https://github.com/alibaba/AliSQL/issues) 提交。 + +## 相关工具 + +### RDSAI CLI — AI 驱动的数据库助手 + +

+ RDSAI CLI + Python 3.13+ +

+ +[RDSAI CLI](https://github.com/aliyun/rdsai-cli) 是新一代 AI 驱动的数据库命令行工具，让你可以用**自然语言**与 AliSQL 和 MySQL 数据库交互。AI 代理会帮你完成 SQL 生成、执行计划分析、诊断优化等工作。 + +```bash +# 安装 +curl -LsSf https://raw.githubusercontent.com/aliyun/rdsai-cli/main/install.sh | sh + +# 连接数据库，使用自然语言查询 +rdsai --host localhost -u root -p secret -D mydb +mysql> 分析 users 表的索引使用情况 +mysql> 显示过去一小时的慢查询 +mysql> 为什么这个查询很慢: SELECT * FROM users WHERE name LIKE '%john%' +``` + +**核心功能：** +- 自然语言转 SQL（支持中英文） +- AI 驱动的查询优化与诊断分析 +- 按 `Ctrl+E` 即时分析执行计划 +- 多模型 LLM 支持（通义千问、OpenAI、DeepSeek、Anthropic 等） +- 自动化性能基准测试与分析报告 + +👉 **[立即体验 RDSAI CLI](https://github.com/aliyun/rdsai-cli)** + +## 社区与支持 + + + + + + +
+ +**GitHub Issues** + +Bug 反馈与功能建议 + +[提交 Issue](https://github.com/alibaba/AliSQL/issues) + + + +**阿里云 RDS** + +托管的 DuckDB 分析型实例 + +[了解更多](https://help.aliyun.com/zh/rds/apsaradb-rds-for-mysql/duckdb-based-analytical-instance/) + +
+ +## 开源协议 + +AliSQL 采用 **GPL-2.0** 协议开源，与 MySQL 保持一致。 + +详见 [LICENSE](LICENSE) 文件。 + +## Star 趋势 + +

+ + Star History Chart + +

+ +

+ 由 阿里云数据库团队 精心打造 +

+ +

+ GitHub • + MySQL • + DuckDB +

diff --git a/alisql-logo.png b/alisql-logo.png new file mode 100644 index 00000000000..5017714471f Binary files /dev/null and b/alisql-logo.png differ diff --git a/build.sh b/build.sh index 3e8836e0698..262e4650d46 100644 --- a/build.sh +++ b/build.sh @@ -212,6 +212,7 @@ cmake_args=( -DWITH_EXTRA_CHARSETS=all -DWITH_ZLIB=bundled -DWITH_ZSTD=bundled + -DWITH_TIRPC=bundled -DWITH_MYISAM_STORAGE_ENGINE=1 -DWITH_INNOBASE_STORAGE_ENGINE=1 -DWITH_CSV_STORAGE_ENGINE=1 diff --git a/config.h.cmake b/config.h.cmake index 4069116ebbb..4ab276cde45 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -44,6 +44,7 @@ #cmakedefine HAVE_EXECINFO_H 1 #cmakedefine HAVE_FPU_CONTROL_H 1 #cmakedefine HAVE_GRP_H 1 +#cmakedefine HAVE_IMMINTRIN_H 1 #cmakedefine HAVE_LANGINFO_H 1 #cmakedefine HAVE_MALLOC_H 1 #cmakedefine HAVE_NETINET_IN_H 1 diff --git a/configure.cmake b/configure.cmake index 3ac3f4d5fe6..46da4811bef 100644 --- a/configure.cmake +++ b/configure.cmake @@ -195,6 +195,7 @@ CHECK_INCLUDE_FILES (endian.h HAVE_ENDIAN_H) CHECK_INCLUDE_FILES (execinfo.h HAVE_EXECINFO_H) CHECK_INCLUDE_FILES (fpu_control.h HAVE_FPU_CONTROL_H) CHECK_INCLUDE_FILES (grp.h HAVE_GRP_H) +CHECK_INCLUDE_FILES (immintrin.h HAVE_IMMINTRIN_H) # vidx CHECK_INCLUDE_FILES (langinfo.h HAVE_LANGINFO_H) CHECK_INCLUDE_FILES (malloc.h HAVE_MALLOC_H) CHECK_INCLUDE_FILES (netinet/in.h HAVE_NETINET_IN_H) diff --git a/include/m_ctype.h b/include/m_ctype.h index 31f7e5ad68e..182c5700b6b 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -782,4 +782,14 @@ static inline bool is_supported_parser_charset(const CHARSET_INFO *cs) { return (cs->mbminlen == 1); } +static inline void my_ci_hash_sort(CHARSET_INFO *ci, const uchar *key, + size_t len, ulong *nr1, ulong *nr2) { + (ci->coll->hash_sort)(ci, key, len, nr1, nr2); +} + +extern "C" { +void my_hash_sort_bin(const CHARSET_INFO *cs [[maybe_unused]], const uchar *key, + size_t len, uint64 *nr1, uint64 *nr2); +} // extern "C" + #endif // M_CTYPE_INCLUDED diff --git a/include/my_base.h b/include/my_base.h index b98b1acfefd..6642cf71144 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -520,7 +520,8 @@ enum ha_base_keytype { /* The combination of the above can be used for key type comparison. */ #define HA_KEYFLAG_MASK \ (HA_NOSAME | HA_PACK_KEY | HA_AUTO_KEY | HA_BINARY_PACK_KEY | HA_FULLTEXT | \ - HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY) + HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY | \ + HA_VECTOR) /** Fulltext index uses [pre]parser */ #define HA_USES_PARSER (1 << 14) @@ -555,6 +556,7 @@ enum ha_base_keytype { constexpr const ulong HA_INDEX_USES_ENGINE_ATTRIBUTE{1UL << 20}; constexpr const ulong HA_INDEX_USES_SECONDARY_ENGINE_ATTRIBUTE{1UL << 21}; +constexpr const ulong HA_VECTOR{1UL << 22}; /* These flags can be added to key-seg-flag */ diff --git a/include/my_bit.h b/include/my_bit.h index 231c10bd1a7..6ac588a9533 100644 --- a/include/my_bit.h +++ b/include/my_bit.h @@ -51,6 +51,45 @@ static inline uint my_bit_log2(ulong value) { return bit; } +/* + my_bit_log2_xxx() + + In the given value, find the highest bit set, + which is the smallest X that satisfies the condition: (2^X >= value). + Can be used as a reverse operation for (1<> 4)) + 4 + : my_bit_log2_hex_digit(value); +} +static inline constexpr uint my_bit_log2_uint16(uint16 value) { + return value & 0xFF00 ? my_bit_log2_uint8((uint8)(value >> 8)) + 8 + : my_bit_log2_uint8((uint8)value); +} +static inline constexpr uint my_bit_log2_uint32(uint32 value) { + return value & 0xFFFF0000UL ? my_bit_log2_uint16((uint16)(value >> 16)) + 16 + : my_bit_log2_uint16((uint16)value); +} +static inline constexpr uint my_bit_log2_uint64(ulonglong value) { + return value & 0xFFFFFFFF00000000ULL + ? my_bit_log2_uint32((uint32)(value >> 32)) + 32 + : my_bit_log2_uint32((uint32)value); +} + static inline uint my_count_bits(ulonglong v) { #if SIZEOF_LONG_LONG > 4 /* The following code is a bit faster on 16 bit machines than if we would diff --git a/include/my_byteorder.h b/include/my_byteorder.h index 12112975e53..cd42b83d351 100644 --- a/include/my_byteorder.h +++ b/include/my_byteorder.h @@ -322,6 +322,11 @@ inline uchar *store32be(uchar *ptr, uint32 val) { return pointer_cast(store32be(pointer_cast(ptr), val)); } +/* convenience helpers */ +static inline float get_float(const void *from) { + return float4get(((const uchar *)from)); +} + #endif /* __cplusplus */ #endif /* MY_BYTEORDER_INCLUDED */ diff --git a/include/my_sys.h b/include/my_sys.h index 081ce35ca3e..8b9f6251307 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -789,11 +789,13 @@ extern bool my_init_dynamic_array(DYNAMIC_ARRAY *array, PSI_memory_key key, #define dynamic_element(array, array_index, type) \ ((type)((array)->buffer) + (array_index)) +#define reset_dynamic(array) ((array)->elements = 0) /* Some functions are still in use in C++, because HASH uses DYNAMIC_ARRAY */ extern bool insert_dynamic(DYNAMIC_ARRAY *array, const void *element); extern void *alloc_dynamic(DYNAMIC_ARRAY *array); extern void delete_dynamic(DYNAMIC_ARRAY *array); +extern void *pop_dynamic(DYNAMIC_ARRAY *); extern bool init_dynamic_string(DYNAMIC_STRING *str, const char *init_str, size_t init_alloc); diff --git a/include/vidx/SIMD.h b/include/vidx/SIMD.h new file mode 100644 index 00000000000..d13f668a2f3 --- /dev/null +++ b/include/vidx/SIMD.h @@ -0,0 +1,55 @@ +#ifndef SIMD_INCLUDED +#define SIMD_INCLUDED + +/* +MIT License + +Copyright (c) 2023 Sasha Krassovsky + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// https://save-buffer.github.io/bloom_filter.html + +/* + Use gcc function multiversioning to optimize for a specific CPU with run-time + detection. Works only for x86, for other architectures we provide only one + implementation for now. +*/ +#define DEFAULT_IMPLEMENTATION +#if __GNUC__ > 7 +#ifdef __x86_64__ +#ifdef HAVE_IMMINTRIN_H +#include +#undef DEFAULT_IMPLEMENTATION +#define DEFAULT_IMPLEMENTATION __attribute__((target("default"))) +#define AVX2_IMPLEMENTATION __attribute__((target("avx2,avx,fma"))) +#if __GNUC__ > 9 +#define AVX512_IMPLEMENTATION __attribute__((target("avx512f,avx512bw"))) +#endif +#endif +#endif +#ifdef __aarch64__ +#include +#undef DEFAULT_IMPLEMENTATION +#define NEON_IMPLEMENTATION +#endif +#endif + +#endif /* SIMD_INCLUDED */ diff --git a/include/vidx/bloom_filters.h b/include/vidx/bloom_filters.h new file mode 100644 index 00000000000..53381dab57f --- /dev/null +++ b/include/vidx/bloom_filters.h @@ -0,0 +1,417 @@ +#ifndef BLOOM_FILTERS_INCLUDED +#define BLOOM_FILTERS_INCLUDED + +/* +MIT License + +Copyright (c) 2023 Sasha Krassovsky + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// https://save-buffer.github.io/bloom_filter.html + +#pragma once +#include +#include +#include + +#include "my_bit.h" +#include "my_inttypes.h" // rds uintxx_t +#include "vidx/SIMD.h" + +template +struct PatternedSimdBloomFilter { + PatternedSimdBloomFilter(int n, float eps) : n(n), epsilon(eps) { + m = ComputeNumBits(); + int log_num_blocks = my_bit_log2_uint32(m) + 1 - rotate_bits; + num_blocks = (1ULL << log_num_blocks); + bv.resize(num_blocks); + } + + uint32_t ComputeNumBits() { + double bits_per_val = -1.44 * std::log2(epsilon); + return std::max(512, + static_cast(bits_per_val * n + 0.5)); + } + +#ifdef AVX2_IMPLEMENTATION + AVX2_IMPLEMENTATION + __m256i CalcHash(__m256i vecData) { + // (almost) xxHash parallel version, 64bit input, 64bit output, seed=0 + static constexpr __m256i rotl48 = { + 0x0504030201000706ULL, 0x0D0C0B0A09080F0EULL, 0x1514131211101716ULL, + 0x1D1C1B1A19181F1EULL}; + static constexpr __m256i rotl24 = { + 0x0201000706050403ULL, + 0x0A09080F0E0D0C0BULL, + 0x1211101716151413ULL, + 0x1A19181F1E1D1C1BULL, + }; + static constexpr uint64_t prime_mx2 = 0x9FB21C651E98DF25ULL; + static constexpr uint64_t bitflip = 0xC73AB174C5ECD5A2ULL; + __m256i step1 = _mm256_xor_si256(vecData, _mm256_set1_epi64x(bitflip)); + __m256i step2 = _mm256_shuffle_epi8(step1, rotl48); + __m256i step3 = _mm256_shuffle_epi8(step1, rotl24); + __m256i step4 = _mm256_xor_si256(step1, _mm256_xor_si256(step2, step3)); + __m256i step5 = _mm256_mul_epi32(step4, _mm256_set1_epi64x(prime_mx2)); + __m256i step6 = _mm256_srli_epi64(step5, 35); + __m256i step7 = _mm256_add_epi64(step6, _mm256_set1_epi64x(8)); + __m256i step8 = _mm256_xor_si256(step5, step7); + __m256i step9 = _mm256_mul_epi32(step8, _mm256_set1_epi64x(prime_mx2)); + return _mm256_xor_si256(step9, _mm256_srli_epi64(step9, 28)); + } + + AVX2_IMPLEMENTATION + __m256i GetBlockIdx(__m256i vecHash) { + __m256i vecNumBlocksMask = _mm256_set1_epi64x(num_blocks - 1); + __m256i vecBlockIdx = + _mm256_srli_epi64(vecHash, mask_idx_bits + rotate_bits); + return _mm256_and_si256(vecBlockIdx, vecNumBlocksMask); + } + + AVX2_IMPLEMENTATION + __m256i ConstructMask(__m256i vecHash) { + __m256i vecMaskIdxMask = _mm256_set1_epi64x((1 << mask_idx_bits) - 1); + __m256i vecMaskMask = _mm256_set1_epi64x((1ull << bits_per_mask) - 1); + __m256i vec64 = _mm256_set1_epi64x(64); + + __m256i vecMaskIdx = _mm256_and_si256(vecHash, vecMaskIdxMask); + __m256i vecMaskByteIdx = _mm256_srli_epi64(vecMaskIdx, 3); + __m256i vecMaskBitIdx = + _mm256_and_si256(vecMaskIdx, _mm256_set1_epi64x(0x7)); + __m256i vecRawMasks = + _mm256_i64gather_epi64((const longlong *)masks, vecMaskByteIdx, 1); + __m256i vecUnrotated = _mm256_and_si256( + _mm256_srlv_epi64(vecRawMasks, vecMaskBitIdx), vecMaskMask); + + __m256i vecRotation = + _mm256_and_si256(_mm256_srli_epi64(vecHash, mask_idx_bits), + _mm256_set1_epi64x((1 << rotate_bits) - 1)); + __m256i vecShiftUp = _mm256_sllv_epi64(vecUnrotated, vecRotation); + __m256i vecShiftDown = + _mm256_srlv_epi64(vecUnrotated, _mm256_sub_epi64(vec64, vecRotation)); + return _mm256_or_si256(vecShiftDown, vecShiftUp); + } + + AVX2_IMPLEMENTATION + void Insert(const T **data) { + __m256i vecDataA = + _mm256_loadu_si256(reinterpret_cast<__m256i *>(data + 0)); + __m256i vecDataB = + _mm256_loadu_si256(reinterpret_cast<__m256i *>(data + 4)); + + __m256i vecHashA = CalcHash(vecDataA); + __m256i vecHashB = CalcHash(vecDataB); + + __m256i vecMaskA = ConstructMask(vecHashA); + __m256i vecMaskB = ConstructMask(vecHashB); + + __m256i vecBlockIdxA = GetBlockIdx(vecHashA); + __m256i vecBlockIdxB = GetBlockIdx(vecHashB); + + uint64_t block0 = _mm256_extract_epi64(vecBlockIdxA, 0); + uint64_t block1 = _mm256_extract_epi64(vecBlockIdxA, 1); + uint64_t block2 = _mm256_extract_epi64(vecBlockIdxA, 2); + uint64_t block3 = _mm256_extract_epi64(vecBlockIdxA, 3); + uint64_t block4 = _mm256_extract_epi64(vecBlockIdxB, 0); + uint64_t block5 = _mm256_extract_epi64(vecBlockIdxB, 1); + uint64_t block6 = _mm256_extract_epi64(vecBlockIdxB, 2); + uint64_t block7 = _mm256_extract_epi64(vecBlockIdxB, 3); + + bv[block0] |= _mm256_extract_epi64(vecMaskA, 0); + bv[block1] |= _mm256_extract_epi64(vecMaskA, 1); + bv[block2] |= _mm256_extract_epi64(vecMaskA, 2); + bv[block3] |= _mm256_extract_epi64(vecMaskA, 3); + bv[block4] |= _mm256_extract_epi64(vecMaskB, 0); + bv[block5] |= _mm256_extract_epi64(vecMaskB, 1); + bv[block6] |= _mm256_extract_epi64(vecMaskB, 2); + bv[block7] |= _mm256_extract_epi64(vecMaskB, 3); + } + + AVX2_IMPLEMENTATION + uint8_t Query(T **data) { + __m256i vecDataA = + _mm256_loadu_si256(reinterpret_cast<__m256i *>(data + 0)); + __m256i vecDataB = + _mm256_loadu_si256(reinterpret_cast<__m256i *>(data + 4)); + + __m256i vecHashA = CalcHash(vecDataA); + __m256i vecHashB = CalcHash(vecDataB); + + __m256i vecMaskA = ConstructMask(vecHashA); + __m256i vecMaskB = ConstructMask(vecHashB); + + __m256i vecBlockIdxA = GetBlockIdx(vecHashA); + __m256i vecBlockIdxB = GetBlockIdx(vecHashB); + + __m256i vecBloomA = + _mm256_i64gather_epi64(bv.data(), vecBlockIdxA, sizeof(longlong)); + __m256i vecBloomB = + _mm256_i64gather_epi64(bv.data(), vecBlockIdxB, sizeof(longlong)); + __m256i vecCmpA = + _mm256_cmpeq_epi64(_mm256_and_si256(vecMaskA, vecBloomA), vecMaskA); + __m256i vecCmpB = + _mm256_cmpeq_epi64(_mm256_and_si256(vecMaskB, vecBloomB), vecMaskB); + uint32_t res_a = static_cast(_mm256_movemask_epi8(vecCmpA)); + uint32_t res_b = static_cast(_mm256_movemask_epi8(vecCmpB)); + uint64_t res_bytes = res_a | (static_cast(res_b) << 32); + uint8_t res_bits = static_cast( + _mm256_movemask_epi8(_mm256_set1_epi64x(res_bytes)) & 0xff); + return res_bits; + } + + /* AVX-512 version can be (and was) implemented, but the speedup is, + basically, unnoticeable, well below the noise level */ +#endif + +#ifdef NEON_IMPLEMENTATION + uint64x2_t CalcHash(uint64x2_t vecData) { + static constexpr uint64_t prime_mx2 = 0x9FB21C651E98DF25ULL; + static constexpr uint64_t bitflip = 0xC73AB174C5ECD5A2ULL; + uint64x2_t step1 = veorq_u64(vecData, vdupq_n_u64(bitflip)); + uint64x2_t step2 = + veorq_u64(vshrq_n_u64(step1, 48), vshlq_n_u64(step1, 16)); + uint64x2_t step3 = + veorq_u64(vshrq_n_u64(step1, 24), vshlq_n_u64(step1, 40)); + uint64x2_t step4 = veorq_u64(step1, veorq_u64(step2, step3)); + uint64x2_t step5; + step5 = vsetq_lane_u64(vgetq_lane_u64(step4, 0) * prime_mx2, step4, 0); + step5 = vsetq_lane_u64(vgetq_lane_u64(step4, 1) * prime_mx2, step5, 1); + uint64x2_t step6 = vshrq_n_u64(step5, 35); + uint64x2_t step7 = vaddq_u64(step6, vdupq_n_u64(8)); + uint64x2_t step8 = veorq_u64(step5, step7); + uint64x2_t step9; + step9 = vsetq_lane_u64(vgetq_lane_u64(step8, 0) * prime_mx2, step8, 0); + step9 = vsetq_lane_u64(vgetq_lane_u64(step8, 1) * prime_mx2, step9, 1); + return veorq_u64(step9, vshrq_n_u64(step9, 28)); + } + + uint64x2_t GetBlockIdx(uint64x2_t vecHash) { + uint64x2_t vecNumBlocksMask = vdupq_n_u64(num_blocks - 1); + uint64x2_t vecBlockIdx = vshrq_n_u64(vecHash, mask_idx_bits + rotate_bits); + return vandq_u64(vecBlockIdx, vecNumBlocksMask); + } + + uint64x2_t ConstructMask(uint64x2_t vecHash) { + uint64x2_t vecMaskIdxMask = vdupq_n_u64((1 << mask_idx_bits) - 1); + uint64x2_t vecMaskMask = vdupq_n_u64((1ull << bits_per_mask) - 1); + + uint64x2_t vecMaskIdx = vandq_u64(vecHash, vecMaskIdxMask); + uint64x2_t vecMaskByteIdx = vshrq_n_u64(vecMaskIdx, 3); + /* + Shift right in NEON is implemented as shift left by a negative value. + Do the negation here. + */ + int64x2_t vecMaskBitIdx = vsubq_s64( + vdupq_n_s64(0), + vreinterpretq_s64_u64(vandq_u64(vecMaskIdx, vdupq_n_u64(0x7)))); + uint64x2_t vecRawMasks = vdupq_n_u64(*reinterpret_cast( + masks + vgetq_lane_u64(vecMaskByteIdx, 0))); + vecRawMasks = vsetq_lane_u64(*reinterpret_cast( + masks + vgetq_lane_u64(vecMaskByteIdx, 1)), + vecRawMasks, 1); + uint64x2_t vecUnrotated = + vandq_u64(vshlq_u64(vecRawMasks, vecMaskBitIdx), vecMaskMask); + + int64x2_t vecRotation = + vreinterpretq_s64_u64(vandq_u64(vshrq_n_u64(vecHash, mask_idx_bits), + vdupq_n_u64((1 << rotate_bits) - 1))); + uint64x2_t vecShiftUp = vshlq_u64(vecUnrotated, vecRotation); + uint64x2_t vecShiftDown = + vshlq_u64(vecUnrotated, vsubq_s64(vecRotation, vdupq_n_s64(64))); + return vorrq_u64(vecShiftDown, vecShiftUp); + } + + void Insert(const T **data) { + uint64x2_t vecDataA = vld1q_u64(reinterpret_cast(data + 0)); + uint64x2_t vecDataB = vld1q_u64(reinterpret_cast(data + 2)); + uint64x2_t vecDataC = vld1q_u64(reinterpret_cast(data + 4)); + uint64x2_t vecDataD = vld1q_u64(reinterpret_cast(data + 6)); + + uint64x2_t vecHashA = CalcHash(vecDataA); + uint64x2_t vecHashB = CalcHash(vecDataB); + uint64x2_t vecHashC = CalcHash(vecDataC); + uint64x2_t vecHashD = CalcHash(vecDataD); + + uint64x2_t vecMaskA = ConstructMask(vecHashA); + uint64x2_t vecMaskB = ConstructMask(vecHashB); + uint64x2_t vecMaskC = ConstructMask(vecHashC); + uint64x2_t vecMaskD = ConstructMask(vecHashD); + + uint64x2_t vecBlockIdxA = GetBlockIdx(vecHashA); + uint64x2_t vecBlockIdxB = GetBlockIdx(vecHashB); + uint64x2_t vecBlockIdxC = GetBlockIdx(vecHashC); + uint64x2_t vecBlockIdxD = GetBlockIdx(vecHashD); + + uint64_t block0 = vgetq_lane_u64(vecBlockIdxA, 0); + uint64_t block1 = vgetq_lane_u64(vecBlockIdxA, 1); + uint64_t block2 = vgetq_lane_u64(vecBlockIdxB, 0); + uint64_t block3 = vgetq_lane_u64(vecBlockIdxB, 1); + uint64_t block4 = vgetq_lane_u64(vecBlockIdxC, 0); + uint64_t block5 = vgetq_lane_u64(vecBlockIdxC, 1); + uint64_t block6 = vgetq_lane_u64(vecBlockIdxD, 0); + uint64_t block7 = vgetq_lane_u64(vecBlockIdxD, 1); + + bv[block0] |= vgetq_lane_u64(vecMaskA, 0); + bv[block1] |= vgetq_lane_u64(vecMaskA, 1); + bv[block2] |= vgetq_lane_u64(vecMaskB, 0); + bv[block3] |= vgetq_lane_u64(vecMaskB, 1); + bv[block4] |= vgetq_lane_u64(vecMaskC, 0); + bv[block5] |= vgetq_lane_u64(vecMaskC, 1); + bv[block6] |= vgetq_lane_u64(vecMaskD, 0); + bv[block7] |= vgetq_lane_u64(vecMaskD, 1); + } + + uint8_t Query(T **data) { + uint64x2_t vecDataA = vld1q_u64(reinterpret_cast(data + 0)); + uint64x2_t vecDataB = vld1q_u64(reinterpret_cast(data + 2)); + uint64x2_t vecDataC = vld1q_u64(reinterpret_cast(data + 4)); + uint64x2_t vecDataD = vld1q_u64(reinterpret_cast(data + 6)); + + uint64x2_t vecHashA = CalcHash(vecDataA); + uint64x2_t vecHashB = CalcHash(vecDataB); + uint64x2_t vecHashC = CalcHash(vecDataC); + uint64x2_t vecHashD = CalcHash(vecDataD); + + uint64x2_t vecMaskA = ConstructMask(vecHashA); + uint64x2_t vecMaskB = ConstructMask(vecHashB); + uint64x2_t vecMaskC = ConstructMask(vecHashC); + uint64x2_t vecMaskD = ConstructMask(vecHashD); + + uint64x2_t vecBlockIdxA = GetBlockIdx(vecHashA); + uint64x2_t vecBlockIdxB = GetBlockIdx(vecHashB); + uint64x2_t vecBlockIdxC = GetBlockIdx(vecHashC); + uint64x2_t vecBlockIdxD = GetBlockIdx(vecHashD); + + uint64x2_t vecBloomA = vdupq_n_u64(bv[vgetq_lane_u64(vecBlockIdxA, 0)]); + vecBloomA = + vsetq_lane_u64(bv[vgetq_lane_u64(vecBlockIdxA, 1)], vecBloomA, 1); + uint64x2_t vecBloomB = vdupq_n_u64(bv[vgetq_lane_u64(vecBlockIdxB, 0)]); + vecBloomB = + vsetq_lane_u64(bv[vgetq_lane_u64(vecBlockIdxB, 1)], vecBloomB, 1); + uint64x2_t vecBloomC = vdupq_n_u64(bv[vgetq_lane_u64(vecBlockIdxC, 0)]); + vecBloomC = + vsetq_lane_u64(bv[vgetq_lane_u64(vecBlockIdxC, 1)], vecBloomC, 1); + uint64x2_t vecBloomD = vdupq_n_u64(bv[vgetq_lane_u64(vecBlockIdxD, 0)]); + vecBloomD = + vsetq_lane_u64(bv[vgetq_lane_u64(vecBlockIdxD, 1)], vecBloomD, 1); + + uint64x2_t vecCmpA = vceqq_u64(vandq_u64(vecMaskA, vecBloomA), vecMaskA); + uint64x2_t vecCmpB = vceqq_u64(vandq_u64(vecMaskB, vecBloomB), vecMaskB); + uint64x2_t vecCmpC = vceqq_u64(vandq_u64(vecMaskC, vecBloomC), vecMaskC); + uint64x2_t vecCmpD = vceqq_u64(vandq_u64(vecMaskD, vecBloomD), vecMaskD); + + return (vgetq_lane_u64(vecCmpA, 0) & 0x01) | + (vgetq_lane_u64(vecCmpA, 1) & 0x02) | + (vgetq_lane_u64(vecCmpB, 0) & 0x04) | + (vgetq_lane_u64(vecCmpB, 1) & 0x08) | + (vgetq_lane_u64(vecCmpC, 0) & 0x10) | + (vgetq_lane_u64(vecCmpC, 1) & 0x20) | + (vgetq_lane_u64(vecCmpD, 0) & 0x40) | + (vgetq_lane_u64(vecCmpD, 1) & 0x80); + } +#endif + + /******************************************************** + ********* non-SIMD fallback version ********************/ + +#ifdef DEFAULT_IMPLEMENTATION + uint64_t CalcHash_1(const T *data) { + static constexpr uint64_t prime_mx2 = 0x9FB21C651E98DF25ULL; + static constexpr uint64_t bitflip = 0xC73AB174C5ECD5A2ULL; + uint64_t step1 = ((intptr)data) ^ bitflip; + uint64_t step2 = (step1 >> 48) ^ (step1 << 16); + uint64_t step3 = (step1 >> 24) ^ (step1 << 40); + uint64_t step4 = step1 ^ step2 ^ step3; + uint64_t step5 = step4 * prime_mx2; + uint64_t step6 = step5 >> 35; + uint64_t step7 = step6 + 8; + uint64_t step8 = step5 ^ step7; + uint64_t step9 = step8 * prime_mx2; + return step9 ^ (step9 >> 28); + } + + uint64_t GetBlockIdx_1(uint64_t hash) { + uint64_t blockIdx = hash >> (mask_idx_bits + rotate_bits); + return blockIdx & (num_blocks - 1); + } + + uint64_t ConstructMask_1(uint64_t hash) { + uint64_t maskIdxMask = (1 << mask_idx_bits) - 1; + uint64_t maskMask = (1ULL << bits_per_mask) - 1; + uint64_t maskIdx = hash & maskIdxMask; + uint64_t maskByteIdx = maskIdx >> 3; + uint64_t maskBitIdx = maskIdx & 7; + uint64_t rawMask = *(uint64_t *)(masks + maskByteIdx); + uint64_t unrotated = (rawMask >> maskBitIdx) & maskMask; + uint64_t rotation = (hash >> mask_idx_bits) & ((1 << rotate_bits) - 1); + return rotation ? (unrotated << rotation) | (unrotated >> (64 - rotation)) + : unrotated; + } + + DEFAULT_IMPLEMENTATION + void Insert(const T **data) { + for (size_t i = 0; i < 8; i++) { + uint64_t hash = CalcHash_1(data[i]); + uint64_t mask = ConstructMask_1(hash); + bv[GetBlockIdx_1(hash)] |= mask; + } + } + + DEFAULT_IMPLEMENTATION + uint8_t Query(T **data) { + uint8_t res_bits = 0; + for (size_t i = 0; i < 8; i++) { + uint64_t hash = CalcHash_1(data[i]); + uint64_t mask = ConstructMask_1(hash); + if ((bv[GetBlockIdx_1(hash)] & mask) == mask) res_bits |= 1 << i; + } + return res_bits; + } +#endif + + int n; + float epsilon; + + uint64_t num_blocks; + uint32_t m; + // calculated from the upstream MaskTable and hard-coded + static constexpr int log_num_masks = 10; + static constexpr int bits_per_mask = 57; + const uint8_t masks[136] = { + 0x00, 0x04, 0x01, 0x04, 0x00, 0x20, 0x01, 0x00, 0x00, 0x02, 0x08, 0x00, + 0x02, 0x42, 0x00, 0x00, 0x04, 0x00, 0x00, 0x84, 0x80, 0x00, 0x04, 0x00, + 0x02, 0x00, 0x00, 0x21, 0x00, 0x08, 0x00, 0x14, 0x00, 0x00, 0x40, 0x00, + 0x10, 0x00, 0xa8, 0x00, 0x00, 0x00, 0x00, 0x10, 0x04, 0x40, 0x01, 0x00, + 0x40, 0x00, 0x00, 0x08, 0x01, 0x02, 0x80, 0x00, 0x00, 0x01, 0x00, 0x06, + 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x0c, 0x10, 0x00, 0x10, 0x00, 0x00, + 0x10, 0x08, 0x01, 0x10, 0x00, 0x00, 0x10, 0x20, 0x00, 0x01, 0x20, 0x00, + 0x02, 0x40, 0x00, 0x00, 0x02, 0x40, 0x01, 0x00, 0x40, 0x00, 0x00, 0x0a, + 0x00, 0x02, 0x01, 0x80, 0x00, 0x00, 0x10, 0x08, 0x00, 0x06, 0x00, 0x04, + 0x00, 0x00, 0x50, 0x00, 0x08, 0x10, 0x20, 0x00, 0x00, 0x80, 0x00, 0x10, + 0x10, 0x04, 0x04, 0x00, 0x00, 0x00, 0x20, 0x20, 0x08, 0x08, 0x02, 0x00, + 0x00, 0x00, 0x40, 0x00}; + std::vector bv; + + static constexpr int mask_idx_bits = log_num_masks; + static constexpr int rotate_bits = 6; +}; + +#endif /* BLOOM_FILTERS_INCLUDED */ diff --git a/include/vidx/hash.h b/include/vidx/hash.h new file mode 100644 index 00000000000..93aad190af5 --- /dev/null +++ b/include/vidx/hash.h @@ -0,0 +1,121 @@ +/* Copyright (c) 2000, 2023, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/* Dynamic hashing of record with different key-length */ + +#ifndef _hash_h +#define _hash_h + +#include "my_sys.h" /* DYNAMIC_ARRAY */ + +/* + This forward declaration is used from C files where the real + definition is included before. Since C does not allow repeated + typedef declarations, even when identical, the definition may not be + repeated. +*/ +#ifdef __cplusplus +extern "C" { +#endif + +/* + Overhead to store an element in hash + Can be used to approximate memory consumption for a hash + */ +#define HASH_OVERHEAD (sizeof(char *) * 2) + +/* flags for hash_init */ +#define HASH_UNIQUE 1 /* hash_insert fails on duplicate key */ +#define HASH_THREAD_SPECIFIC 2 /* Mark allocated memory THREAD_SPECIFIC */ + +typedef char my_bool; + +/* Define some general constants */ +#ifndef TRUE +#define TRUE (1) /* Logical true */ +#define FALSE (0) /* Logical false */ +#endif + +typedef uint32 my_hash_value_type; +typedef const uchar *(*my_hash_get_key)(const void *, size_t *, my_bool); +typedef my_hash_value_type (*my_hash_function)(CHARSET_INFO *, const uchar *, + size_t); +typedef void (*my_hash_free_key)(void *); +typedef my_bool (*my_hash_walk_action)(void *, void *); + +typedef struct st_hash { + size_t key_offset, key_length; /* Length of key if const length */ + size_t blength; + ulong records; + uint flags; + DYNAMIC_ARRAY array; /* Place for hash_keys */ + my_hash_get_key get_key; + my_hash_function hash_function; + void (*free)(void *); + CHARSET_INFO *charset; +} HASH; + +/* A search iterator state */ +typedef uint HASH_SEARCH_STATE; + +#define my_hash_init(A, B, C, D, E, F, G, H, I) \ + my_hash_init2(A, B, 0, C, D, E, F, G, 0, H, I) +my_bool my_hash_init2(PSI_memory_key psi_key, HASH *hash, size_t growth_size, + CHARSET_INFO *charset, size_t default_array_elements, + size_t key_offset, size_t key_length, + my_hash_get_key get_key, my_hash_function hash_function, + void (*free_element)(void *), uint flags); +void my_hash_free(HASH *tree); +void my_hash_reset(HASH *hash); +uchar *my_hash_element(const HASH *hash, size_t idx); +uchar *my_hash_search(const HASH *info, const uchar *key, size_t length); +uchar *my_hash_search_using_hash_value(const HASH *info, + my_hash_value_type hash_value, + const uchar *key, size_t length); +my_hash_value_type my_hash_sort(CHARSET_INFO *cs, const uchar *key, + size_t length); +#define my_calc_hash(A, B, C) my_hash_sort((A)->charset, B, C) +uchar *my_hash_first(const HASH *info, const uchar *key, size_t length, + HASH_SEARCH_STATE *state); +uchar *my_hash_first_from_hash_value(const HASH *info, + my_hash_value_type hash_value, + const uchar *key, size_t length, + HASH_SEARCH_STATE *state); +uchar *my_hash_next(const HASH *info, const uchar *key, size_t length, + HASH_SEARCH_STATE *state); +my_bool my_hash_insert(HASH *info, const uchar *data); +my_bool my_hash_delete(HASH *hash, uchar *record); +my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key, + size_t old_key_length); +void my_hash_replace(HASH *hash, HASH_SEARCH_STATE *state, uchar *new_row); +my_bool my_hash_check(HASH *hash); /* Only in debug library */ +my_bool my_hash_iterate(HASH *hash, my_hash_walk_action action, void *argument); + +#define my_hash_clear(H) bzero((char *)(H), sizeof(*(H))) +#define my_hash_inited(H) ((H)->blength != 0) +#define my_hash_init_opt(A, B, C, D, E, F, G, H, I) \ + (!my_hash_inited(B) && my_hash_init(A, B, C, D, E, F, G, H, I)) + +#ifdef __cplusplus +} +#endif +#endif diff --git a/include/vidx/my_atomic_wrapper.h b/include/vidx/my_atomic_wrapper.h new file mode 100644 index 00000000000..fa815de7aa0 --- /dev/null +++ b/include/vidx/my_atomic_wrapper.h @@ -0,0 +1,100 @@ +/* + Copyright (c) 2001, 2023, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + Without limiting anything contained in the foregoing, this file, + which is part of C Driver for MySQL (Connector/C), is also subject to the + Universal FOSS Exception, version 1.0, a copy of which can be found at + http://oss.oracle.com/licenses/universal-foss-exception. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#pragma once +#ifdef __cplusplus +#include +/** + A wrapper for std::atomic, defaulting to std::memory_order_relaxed. + + When it comes to atomic loads or stores at std::memory_order_relaxed + on IA-32 or AMD64, this wrapper is only introducing some constraints + to the C++ compiler, to prevent some optimizations of loads or + stores. + + On POWER and ARM, atomic loads and stores involve different instructions + from normal loads and stores and will thus incur some overhead. + + Because atomic read-modify-write operations will always incur + overhead, we intentionally do not define + operator++(), operator--(), operator+=(), operator-=(), or similar, + to make the overhead stand out in the users of this code. +*/ +template +class Atomic_relaxed { + std::atomic m; + + public: + Atomic_relaxed(const Atomic_relaxed &rhs) { + m.store(rhs, std::memory_order_relaxed); + } + Atomic_relaxed(Type val) : m(val) {} + Atomic_relaxed() = default; + + Type load(std::memory_order o = std::memory_order_relaxed) const { + return m.load(o); + } + void store(Type i, std::memory_order o = std::memory_order_relaxed) { + m.store(i, o); + } + operator Type() const { return m.load(); } + Type operator=(const Type i) { + store(i); + return i; + } + Type operator=(const Atomic_relaxed &rhs) { return *this = Type{rhs}; } + Type operator+=(const Type i) { return fetch_add(i); } + Type fetch_add(const Type i, + std::memory_order o = std::memory_order_relaxed) { + return m.fetch_add(i, o); + } + Type fetch_sub(const Type i, + std::memory_order o = std::memory_order_relaxed) { + return m.fetch_sub(i, o); + } + Type fetch_xor(const Type i, + std::memory_order o = std::memory_order_relaxed) { + return m.fetch_xor(i, o); + } + Type fetch_and(const Type i, + std::memory_order o = std::memory_order_relaxed) { + return m.fetch_and(i, o); + } + Type fetch_or(const Type i, std::memory_order o = std::memory_order_relaxed) { + return m.fetch_or(i, o); + } + bool compare_exchange_strong( + Type &i1, const Type i2, std::memory_order o1 = std::memory_order_relaxed, + std::memory_order o2 = std::memory_order_relaxed) { + return m.compare_exchange_strong(i1, i2, o1, o2); + } + Type exchange(const Type i, std::memory_order o = std::memory_order_relaxed) { + return m.exchange(i, o); + } +}; +#endif /* __cplusplus */ diff --git a/include/vidx/my_cmp.h b/include/vidx/my_cmp.h new file mode 100644 index 00000000000..fc8895e25e6 --- /dev/null +++ b/include/vidx/my_cmp.h @@ -0,0 +1,38 @@ +/* + Copyright (c) 2001, 2023, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + Without limiting anything contained in the foregoing, this file, + which is part of C Driver for MySQL (Connector/C), is also subject to the + Universal FOSS Exception, version 1.0, a copy of which can be found at + http://oss.oracle.com/licenses/universal-foss-exception. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif +typedef int (*qsort_cmp)(const void *, const void *); +typedef int (*qsort_cmp2)(void *param, const void *a, const void *b); +#ifdef __cplusplus +} +#endif diff --git a/include/vidx/sql_hset.h b/include/vidx/sql_hset.h new file mode 100644 index 00000000000..2575aaf408a --- /dev/null +++ b/include/vidx/sql_hset.h @@ -0,0 +1,142 @@ +#ifndef SQL_HSET_INCLUDED +#define SQL_HSET_INCLUDED +/* Copyright (c) 2010, 2023, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + */ + +#include "vidx/hash.h" + +/** + A type-safe wrapper around mysys HASH. +*/ + +template +class Hash_set { + public: + enum { START_SIZE = 8 }; + /** + Constructs an empty unique hash. + */ + Hash_set(PSI_memory_key psi_key, + const uchar *(*K)(const void *, size_t *, my_bool), + CHARSET_INFO *cs = &my_charset_bin) { + my_hash_init(psi_key, &m_hash, cs, START_SIZE, 0, 0, K, 0, HASH_UNIQUE); + } + + Hash_set(PSI_memory_key psi_key, CHARSET_INFO *charset, + ulong default_array_elements, size_t key_offset, size_t key_length, + my_hash_get_key get_key, void (*free_element)(void *), uint flags) { + my_hash_init(psi_key, &m_hash, charset, default_array_elements, key_offset, + key_length, get_key, free_element, flags); + } + /** + Destroy the hash by freeing the buckets table. Does + not call destructors for the elements. + */ + ~Hash_set() { my_hash_free(&m_hash); } + /** + Insert a single value into a hash. Does not tell whether + the value was inserted -- if an identical value existed, + it is not replaced. + + @retval TRUE Out of memory. + @retval FALSE OK. The value either was inserted or existed + in the hash. + */ + bool insert(const T *value) { + return my_hash_insert(&m_hash, reinterpret_cast(value)); + } + bool remove(const T *value) { + return my_hash_delete(&m_hash, + reinterpret_cast(const_cast(value))); + } + T *find(const void *key, size_t klen) const { + return (T *)my_hash_search(&m_hash, reinterpret_cast(key), + klen); + } + + T *find(const T *other) const { + size_t klen; + const uchar *key = + m_hash.get_key(reinterpret_cast(other), &klen, false); + return find(key, klen); + } + /** Is this hash set empty? */ + bool is_empty() const { return m_hash.records == 0; } + /** Returns the number of unique elements. */ + size_t size() const { return static_cast(m_hash.records); } + /** Erases all elements from the container */ + void clear() { my_hash_reset(&m_hash); } + const T *at(size_t i) const { + return reinterpret_cast( + my_hash_element(const_cast(&m_hash), i)); + } + /** An iterator over hash elements. Is not insert-stable. */ + class Iterator; + using value_type = T; + using iterator = Iterator; + using const_iterator = const Iterator; + + Iterator begin() const { return Iterator(*this, 0); } + Iterator end() const { return Iterator(*this, m_hash.records); } + + class Iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = T; + using difference_type = std::ptrdiff_t; + using pointer = T *; + using reference = T &; + + Iterator(const Hash_set &hash_set, uint idx = 0) + : m_hash(&hash_set.m_hash), m_idx(idx) {} + + Iterator &operator++() { + m_idx++; + return *this; + } + + T &operator*() { + return *reinterpret_cast(my_hash_element(m_hash, m_idx)); + } + + T *operator->() { + return reinterpret_cast(my_hash_element(m_hash, m_idx)); + } + + bool operator==(const typename Hash_set::iterator &rhs) { + return m_idx == rhs.m_idx && m_hash == rhs.m_hash; + } + bool operator!=(const typename Hash_set::iterator &rhs) { + return m_idx != rhs.m_idx || m_hash != rhs.m_hash; + } + + private: + const HASH *m_hash; + uint m_idx; + }; + + private: + HASH m_hash; +}; + +#endif // SQL_HSET_INCLUDED diff --git a/include/vidx/sql_queue.h b/include/vidx/sql_queue.h new file mode 100644 index 00000000000..c980235da5a --- /dev/null +++ b/include/vidx/sql_queue.h @@ -0,0 +1,87 @@ +/* + Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights + reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + 02110-1301 USA + */ + +#ifndef QUEUE_INCLUDED +#define QUEUE_INCLUDED + +#include "sql/psi_memory_key.h" // key_memory_vidx_mem +#include "storage/myisam/queues.h" + +#define set_if_bigger(a, b) \ + do { \ + if ((a) < (b)) (a) = (b); \ + } while (0) +#define set_if_smaller(a, b) \ + do { \ + if ((a) > (b)) (a) = (b); \ + } while (0) + +/* Attention, the QUEUE's elements in storage/myisam/queues.cc is started from +0 */ +#define queue_first_element(queue) 0 +#define queue_remove_top(queue_arg) \ + queue_remove((queue_arg), queue_first_element(queue_arg)) + +/** + A typesafe wrapper of QUEUE, a priority heap +*/ +template +class Queue { + public: + Queue() { m_queue.root = 0; } + ~Queue() { delete_queue(&m_queue); } + int init(uint max_elements, bool max_at_top, queue_compare compare, + Param *param = 0) { + return init_queue(&m_queue, key_memory_vidx_mem, max_elements, 0, + max_at_top, compare, (void *)param); + } + + size_t elements() const { return m_queue.elements; } + bool is_inited() const { return is_queue_inited(&m_queue); } + bool is_full() const { return queue_is_full((QUEUE *)(&m_queue)); } + bool is_empty() const { return elements() == 0; } + Element *top() const { return (Element *)queue_top(&m_queue); } + + void push(const Element *element) { + queue_insert(&m_queue, (uchar *)element); + } + void safe_push(const Element *element) { + if (is_full()) m_queue.elements--; // remove one of the furthest elements + queue_insert(&m_queue, (uchar *)element); + } + Element *pop() { return (Element *)queue_remove_top(&m_queue); } + void clear() { queue_remove_all(&m_queue); } + void propagate_top() { queue_replaced(&m_queue); } + void replace_top(const Element *element) { + queue_top(&m_queue) = (uchar *)element; + propagate_top(); + } + + private: + QUEUE m_queue; +}; + +#endif diff --git a/include/vidx/vidx_common.h b/include/vidx/vidx_common.h new file mode 100644 index 00000000000..5855c403d85 --- /dev/null +++ b/include/vidx/vidx_common.h @@ -0,0 +1,71 @@ +#ifndef VIDX_COMMON_INCLUDED +#define VIDX_COMMON_INCLUDED + +/* Copyright (c) 2025, 2025, Alibaba and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include +#include +#include + +#include "my_inttypes.h" + +/* RDS comment version. The comment having version bigger than it should not be +treated as comment. */ +#define RDS_COMMENT_VERSION 99999 + +#define RDS_COMMENT_VIDX_START "/*!99999 " +#define RDS_COMMENT_VIDX_END " */" + +namespace vidx { +static constexpr uint32_t MAX_DIMENSIONS = 16383; +static constexpr uint32_t VECTOR_PRECISION = sizeof(float); + +static const char *distance_names[] = {"EUCLIDEAN", "COSINE", nullptr}; +static constexpr uint METRIC_DEF = 0; +static constexpr uint METRIC_MAX = + sizeof(distance_names) / sizeof(distance_names[0]) - 1; + +namespace hnsw { +static constexpr uint M_DEF = 6; +static constexpr uint M_MAX = 200; +static constexpr uint M_MIN = 3; + +static inline bool validate_index_option_m(const uint option) { + return option <= M_MAX && option >= M_MIN; +} +} // namespace hnsw + +static inline bool validate_index_option_distance(const uint option) { + return option <= METRIC_MAX; +} + +static inline uint32_t get_dimensions_low(const uint32_t length, + const uint32_t precision) { + if (length % precision > 0) { + return UINT_MAX32; + } + return length / precision; +} +} // namespace vidx + +#endif /* VIDX_COMMON_INCLUDED */ diff --git a/include/vidx/vidx_field.h b/include/vidx/vidx_field.h new file mode 100644 index 00000000000..78530bc2f0c --- /dev/null +++ b/include/vidx/vidx_field.h @@ -0,0 +1,77 @@ +#ifndef VIDX_FIELD_INCLUDED +#define VIDX_FIELD_INCLUDED + +/* Copyright (c) 2025, 2025, Alibaba and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include "sql/field.h" +#include "vidx/vidx_common.h" + +namespace vidx { +class Field_vector : public Field_varstring { + public: + static uint32 dimension_bytes(uint32 dimensions) { + return VECTOR_PRECISION * dimensions; + } + + Field_vector(uchar *ptr_arg, uint32 len_arg, uint length_bytes_arg, + uchar *null_ptr_arg, uchar null_bit_arg, uchar auto_flags_arg, + const char *field_name_arg, TABLE_SHARE *share) + : Field_varstring(ptr_arg, len_arg, length_bytes_arg, null_ptr_arg, + null_bit_arg, auto_flags_arg, field_name_arg, share, + &my_charset_bin) {} + + Field_vector(uint32 len_arg, bool is_nullable_arg, const char *field_name_arg, + TABLE_SHARE *share) + : Field_varstring(len_arg, is_nullable_arg, field_name_arg, share, + &my_charset_bin) {} + + Field_vector(const Field_vector &field) : Field_varstring(field) {} + + uint32 get_dimensions() const; + + void sql_type(String &res) const final { + const CHARSET_INFO *cs = res.charset(); + size_t length = cs->cset->snprintf( + cs, res.ptr(), res.alloced_length(), + RDS_COMMENT_VIDX_START "vector(%u)" RDS_COMMENT_VIDX_END + " varbinary(%u)", + get_dimensions(), VECTOR_PRECISION * get_dimensions()); + res.length(length); + } + Field_vector *clone(MEM_ROOT *mem_root) const final { + assert(type() == MYSQL_TYPE_VARCHAR); + return new (mem_root) Field_vector(*this); + } + using Field_varstring::store; + type_conversion_status store(double nr) final; + type_conversion_status store(longlong nr, bool unsigned_val) final; + type_conversion_status store_decimal(const my_decimal *) final; + type_conversion_status store(const char *from, size_t length, + const CHARSET_INFO *cs) final; + uint is_equal(const Create_field *new_field) const final; + String *val_str(String *, String *) const final; + bool is_vector() const final { return true; } +}; +} // namespace vidx + +#endif /* VIDX_FIELD_INCLUDED */ diff --git a/include/vidx/vidx_func.h b/include/vidx/vidx_func.h new file mode 100644 index 00000000000..e5ac9625ef5 --- /dev/null +++ b/include/vidx/vidx_func.h @@ -0,0 +1,132 @@ +#ifndef VIDX_FUNC_INCLUDED +#define VIDX_FUNC_INCLUDED + +/* Copyright (c) 2025, 2025, Alibaba and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include "vidx/vidx_common.h" + +class ORDER; + +namespace vidx { +enum distance_kind { EUCLIDEAN, COSINE, AUTO }; + +class Item_func_vec_distance : public Item_real_func { + public: + Item_func_vec_distance(const POS &pos, Item *a, Item *b) + : Item_real_func(pos, a, b), kind(AUTO) {} + + Item_func_vec_distance(const POS &pos, Item *a, Item *b, distance_kind c) + : Item_real_func(pos, a, b), kind(c) {} + + const char *func_name() const override { + static LEX_CSTRING name[3] = {{STRING_WITH_LEN("VEC_DISTANCE_EUCLIDEAN")}, + {STRING_WITH_LEN("VEC_DISTANCE_COSINE")}, + {STRING_WITH_LEN("VEC_DISTANCE")}}; + return name[kind].str; + } + + bool resolve_type(THD *thd) override; + int get_key(); + double val_real() override; + enum Functype functype() const override { return VECTOR_DISTANCE_FUNC; } + ha_rows get_limit() const { return m_limit; } + void set_limit(const ha_rows &limit) { m_limit = limit; } + Item *get_const_arg() const { return const_arg; } + + private: + bool check_args(); + + distance_kind kind; + double (*calc_distance_func)(float *v1, float *v2, size_t v_len); + ha_rows m_limit = 0; + Item_field *field_arg = nullptr; + Item *const_arg = nullptr; +}; + +class Item_func_vec_distance_euclidean final : public Item_func_vec_distance { + public: + Item_func_vec_distance_euclidean(const POS &pos, Item *a, Item *b) + : Item_func_vec_distance(pos, a, b, distance_kind::EUCLIDEAN) {} +}; + +class Item_func_vec_distance_cosine final : public Item_func_vec_distance { + public: + Item_func_vec_distance_cosine(const POS &pos, Item *a, Item *b) + : Item_func_vec_distance(pos, a, b, distance_kind::COSINE) {} +}; + +class Item_func_vec_fromtext final : public Item_str_func { + String buffer; + + public: + Item_func_vec_fromtext(const POS &pos, Item *a) : Item_str_func(pos, a) {} + bool resolve_type(THD *thd) override; + const char *func_name() const override { return "VEC_FromText"; } + String *val_str(String *str) override; +}; + +class Item_func_vec_totext final : public Item_str_func { + static const uint32_t per_value_chars = 16; + static const uint32_t max_output_bytes = + (MAX_DIMENSIONS * Item_func_vec_totext::per_value_chars); + String buffer; + + public: + Item_func_vec_totext(const POS &pos, Item *a) : Item_str_func(pos, a) { + collation.set(&my_charset_utf8mb4_0900_bin); + } + bool resolve_type(THD *thd) override; + const char *func_name() const override { return "VEC_ToText"; } + String *val_str(String *str) override; +}; + +class Item_func_vector_dim : public Item_int_func { + String value; + + public: + Item_func_vector_dim(const POS &pos, Item *a) : Item_int_func(pos, a) {} + longlong val_int() override; + const char *func_name() const override { return "vector_dim"; } + bool resolve_type(THD *thd) override { + if (param_type_is_default(thd, 0, 1, MYSQL_TYPE_VARCHAR)) { + return true; + } + bool valid_type = (args[0]->data_type() == MYSQL_TYPE_VARCHAR) || + (args[0]->result_type() == STRING_RESULT && + args[0]->collation.collation == &my_charset_bin); + if (!valid_type) { + my_error(ER_WRONG_ARGUMENTS, MYF(0), func_name()); + return true; + } + max_length = 10; + return false; + } +}; + +static inline bool check_item_func_vec_distance(const Item *item) { + return item->type() == Item::FUNC_ITEM && + ((Item_func *)item)->functype() == Item_func::VECTOR_DISTANCE_FUNC; +} +} // namespace vidx + +#endif /* VIDX_FUNC_INCLUDED */ diff --git a/include/vidx/vidx_hnsw.h b/include/vidx/vidx_hnsw.h new file mode 100644 index 00000000000..bba5bdd1668 --- /dev/null +++ b/include/vidx/vidx_hnsw.h @@ -0,0 +1,56 @@ +#ifndef VIDX_HNSW_INCLUDED +#define VIDX_HNSW_INCLUDED + +/* Copyright (c) 2025, 2025, Alibaba and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +class THD; +class KEY; +struct TABLE; + +namespace dd { +class Table; +} // namespace dd + +namespace vidx { +namespace hnsw { +static constexpr uint32_t DEF_CACHE_SIZE = 16 * 1024 * 1024; +static constexpr uint max_ef = 10000; + +extern ulonglong max_cache_size; +extern void *trx_handler; + +std::unique_ptr create_dd_table(THD *thd, const char *table_name, + KEY *key, dd::Table *dd_table, + TABLE *table, const char *db_name, + const uint tref_len); +int mhnsw_insert(TABLE *table, KEY *keyinfo); +int mhnsw_read_first(TABLE *table, KEY *keyinfo, Item *dist); +int mhnsw_read_next(TABLE *table); +int mhnsw_read_end(TABLE *table); +int mhnsw_invalidate(TABLE *table, const uchar *rec, KEY *keyinfo); +int mhnsw_delete_all(TABLE *table, KEY *keyinfo); +void mhnsw_free(TABLE_SHARE *share); +} // namespace hnsw +} // namespace vidx + +#endif /* VIDX_HNSW_INCLUDED */ diff --git a/include/vidx/vidx_index.h b/include/vidx/vidx_index.h new file mode 100644 index 00000000000..0e8de4789e8 --- /dev/null +++ b/include/vidx/vidx_index.h @@ -0,0 +1,145 @@ +#ifndef VIDX_INDEX_INCLUDED +#define VIDX_INDEX_INCLUDED + +/* Copyright (c) 2025, 2025, Alibaba and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include "my_inttypes.h" // uintxx_t +#include "sql/dd/properties.h" // dd::Properties +#include "sql/dd/types/table.h" // dd::Table +#include "sql/key.h" // KEY + +#include "vidx/vidx_common.h" + +class THD; +struct TABLE; +struct handlerton; +struct TABLE_SHARE; +struct st_plugin_int; +class JOIN_TAB; +class ORDER; +class Alter_info; + +namespace dd { +class Schema; +} // namespace dd + +namespace vidx { +constexpr uint32_t DATA_ROW_ID_LEN = 6; + +class Item_func_vec_distance; + +extern st_plugin_int *vidx_plugin; +extern bool feature_disabled; + +bool check_vector_ddl_and_rewrite_sql(THD *thd, Alter_info *alter_info, + KEY *key_info, const uint key_count, + TABLE *table); + +namespace hnsw { +uint index_options_print(const uint distance, const uint m, char *buf, + uint buf_len); + +bool copy_index_option_m(THD *thd, uint *to, const uint from); +} // namespace hnsw + +bool copy_index_option_distance(THD *thd, uint *to, const uint from); + +/* Create the auxiliary table for the vector index. +@param[in] thd Thread context. +@param[in] key The vector key. +@param[in] dd_table The dd table object describing the base table. +@param[in] table The base table +@param[in] db_name The DB name. +@param[in] old_table_id The old table id before truncate, or + dd::INVALID_OBJECT_ID if new create. +@return true if failed. */ +bool create_table(THD *thd, KEY *key, dd::Table *dd_table, TABLE *table, + const char *db_name, const uint64_t old_table_id); + +/* Drop the auxiliary table for the vector index. +@param[in] thd Thread context. +@param[in] dd_table The dd table object describing the base table. +@param[in] db_name The DB name. +@return true if failed. */ +bool delete_table(THD *thd, const dd::Table *dd_table, const char *db_name); + +/* Rename the auxiliary table for the vector index. +@param[in] thd Thread context. +@param[in] dd_table The dd table object describing the base table. +@param[in] base The base handlerton. +@param[in] new_schema The dd schema object. +@param[in] old_db The old DB name before rename. +@param[in] new_db The new DB name after rename. +@param[in] flags flags. See also mysql_rename_table(). +@return true if failed. */ +bool rename_table(THD *thd, dd::Table *dd_table, handlerton *base, + const dd::Schema &new_schema, const char *old_db, + const char *new_db, uint flags); + +/* Build the info of vector key. +@param[in] thd Thread context. +@param[in] share The table share. +@param[in] dd_table The dd table object describing the base table. +@param[in] nr The number of the vector key. +@return true if failed. */ +bool build_hlindex_key(THD *thd, TABLE_SHARE *share, const dd::Table *dd_table, + const uint nr); + +/* Test if ORDER BY is a single distance function(ORDER BY VEC_DISTANCE), +sort order is descending, and vector index is more efficient than original +access path. +@param[in] tab JOIN_TAB to check +@param[in] order pointer to ORDER struct. +@param[in] limit maximum number of rows to select. +@param[out] order_idx idx choosen. +@return True if vector index is enabled and efficient, otherwise False. */ +bool test_if_cheaper_vector_ordering(JOIN_TAB *tab, ORDER *order, ha_rows limit, + int *order_idx); + +/* Check if the key is a vector key. +@param[in] key The vector key. +@return true if the key is a vector key. */ +static inline bool key_is_vector(KEY *key) { + return key != nullptr && (key->flags & HA_VECTOR); +} + +/* Check the option "__hlindexes__" of the dd table exists and is not empty. +@param[in] dd_table The dd table. +@return true if "__hlindexes__" exists and is not empty, otherwise false. */ +static inline bool dd_table_has_hlindexes(const dd::Table *dd_table) { + return dd_table->options().exists("__hlindexes__"); +} + +/* Check if the dd table is a vector table. +@param[in] dd_table The dd table. +@return true if the dd table is a vector table, otherwise false. */ +static inline bool dd_table_is_hlindex(const dd::Table *dd_table) { + assert(dd_table->options().exists("__vector_column__") == + dd_table->options().exists("__vector_m__")); + assert(dd_table->options().exists("__vector_column__") == + dd_table->options().exists("__vector_distance__")); + return dd_table->options().exists("__vector_column__"); +} +} // namespace vidx + +#endif /* VIDX_INDEX_INCLUDED */ diff --git a/mysql-test/r/all_persisted_variables.result b/mysql-test/r/all_persisted_variables.result index 15e373f3d79..fc69409bfec 100644 --- a/mysql-test/r/all_persisted_variables.result +++ b/mysql-test/r/all_persisted_variables.result @@ -40,7 +40,7 @@ include/assert.inc [Expect 500+ variables in the table. Due to open Bugs, we are # Test SET PERSIST -include/assert.inc [Expect 476 persisted variables in the table.] +include/assert.inc [Expect 481 persisted variables in the table.] ************************************************************ * 3. Restart server, it must preserve the persisted variable @@ -48,9 +48,9 @@ include/assert.inc [Expect 476 persisted variables in the table.] ************************************************************ # restart -include/assert.inc [Expect 476 persisted variables in persisted_variables table.] -include/assert.inc [Expect 476 persisted variables shown as PERSISTED in variables_info table.] -include/assert.inc [Expect 476 persisted variables with matching peristed and global values.] +include/assert.inc [Expect 481 persisted variables in persisted_variables table.] +include/assert.inc [Expect 481 persisted variables shown as PERSISTED in variables_info table.] +include/assert.inc [Expect 481 persisted variables with matching peristed and global values.] ************************************************************ * 4. Test RESET PERSIST IF EXISTS. Verify persisted variable diff --git a/mysql-test/r/comments.result b/mysql-test/r/comments.result index 8e0904709ec..17bce93040f 100644 --- a/mysql-test/r/comments.result +++ b/mysql-test/r/comments.result @@ -10,11 +10,9 @@ ERROR 42000: Query was empty select 1 /*!32301 +1 */; 1 +1 2 -select 1 /*!999999 +1 */; +select 1 /*!99998 +1 */; 1 1 -Warnings: -Warning 4164 Immediately starting the version comment after the version number is deprecated and may change behavior in a future release. Please insert a white-space character after the version number. select 1--1; 1--1 2 @@ -32,12 +30,12 @@ select 1/*!2*/; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '2*/' at line 1 select 1/*!000002*/; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '2*/' at line 1 -select 1/*!999992*/; +select 1/*!99992*/; 1 1 Warnings: Warning 4164 Immediately starting the version comment after the version number is deprecated and may change behavior in a future release. Please insert a white-space character after the version number. -select 1 + /*!00000 2 */ + 3 /*!99999 noise*/ + 4; +select 1 + /*!00000 2 */ + 3 /*!99998 noise*/ + 4; 1 + 2 + 3 + 4 10 drop table if exists table_28779; @@ -58,7 +56,7 @@ drop table table_28779; # # WL#12099: Deprecate nested comments in 8.0 # -SELECT 1 /*!99999 /* */ */; +SELECT 1 /*!99998 /* */ */; 1 1 Warnings: @@ -91,6 +89,7 @@ DO 1 /*!80034 +1*/; # Form feed (0x0c). Should pass without warning. DO 1 /*!80034 +1*/; # Carriage return (0x0d). Should pass without warning. -DO 1 /*!80034 +1*/; +DO 1 /*!80034 ++1*/; # Space (0x20). Should pass without warning. DO 1 /*!80034 +1*/; diff --git a/mysql-test/r/derived_condition_pushdown.result b/mysql-test/r/derived_condition_pushdown.result index e978151b295..c30fd2b2671 100644 --- a/mysql-test/r/derived_condition_pushdown.result +++ b/mysql-test/r/derived_condition_pushdown.result @@ -249,7 +249,7 @@ FROM (SELECT f1 as i, f2 as j FROM t1 ) as dt1 , (SELECT f1 as l, f2 as m, f3 as n FROM t1) as dt2 WHERE i > 1 and i+j > 40 and m < 20 and m+i > 20; EXPLAIN --> Filter: ((dt2.m + dt1.i) > 20) (rows=24) +-> Filter: ((dt2.`m` + dt1.i) > 20) (rows=24) -> Inner hash join (no condition) (rows=24) -> Table scan on dt1 (rows=6) -> Materialize (rows=6) diff --git a/mysql-test/r/information_schema_keywords.result b/mysql-test/r/information_schema_keywords.result index 74ba19d332c..27e3e00fd2e 100644 --- a/mysql-test/r/information_schema_keywords.result +++ b/mysql-test/r/information_schema_keywords.result @@ -99,6 +99,7 @@ CONTAINS 0 CONTEXT 0 CONTINUE 1 CONVERT 1 +COSINE 0 CPU 0 CREATE 1 CROSS 1 @@ -143,6 +144,7 @@ DIRECTORY 0 DISABLE 0 DISCARD 0 DISK 0 +DISTANCE 0 DISTINCT 1 DISTINCTROW 1 DIV 1 @@ -171,6 +173,7 @@ ERROR 0 ERRORS 0 ESCAPE 0 ESCAPED 1 +EUCLIDEAN 0 EVENT 0 EVENTS 0 EVERY 0 @@ -328,6 +331,7 @@ LONGBLOB 1 LONGTEXT 1 LOOP 1 LOW_PRIORITY 1 +M 0 MASTER 0 MASTER_AUTO_POSITION 0 MASTER_BIND 1 @@ -730,6 +734,7 @@ VARCHARACTER 1 VARIABLES 0 VARYING 1 VCPU 0 +VECTOR 1 VIEW 0 VIRTUAL 1 VISIBLE 0 diff --git a/mysql-test/r/information_schema_parameters.result b/mysql-test/r/information_schema_parameters.result index f1b7232a9ae..7ffc059be2b 100644 --- a/mysql-test/r/information_schema_parameters.result +++ b/mysql-test/r/information_schema_parameters.result @@ -2,7 +2,7 @@ USE INFORMATION_SCHEMA; SHOW CREATE TABLE INFORMATION_SCHEMA.PARAMETERS; View Create View character_set_client collation_connection -PARAMETERS CREATE ALGORITHM=UNDEFINED DEFINER=`mysql.infoschema`@`localhost` SQL SECURITY DEFINER VIEW `PARAMETERS` AS select `cat`.`name` AS `SPECIFIC_CATALOG`,`sch`.`name` AS `SPECIFIC_SCHEMA`,`rtn`.`name` AS `SPECIFIC_NAME`,if((`rtn`.`type` = 'FUNCTION'),(`prm`.`ordinal_position` - 1),`prm`.`ordinal_position`) AS `ORDINAL_POSITION`,if(((`rtn`.`type` = 'FUNCTION') and (`prm`.`ordinal_position` = 1)),NULL,`prm`.`mode`) AS `PARAMETER_MODE`,if(((`rtn`.`type` = 'FUNCTION') and (`prm`.`ordinal_position` = 1)),NULL,`prm`.`name`) AS `PARAMETER_NAME`,substring_index(substring_index(`prm`.`data_type_utf8`,'(',1),' ',1) AS `DATA_TYPE`,internal_dd_char_length(`prm`.`data_type`,`prm`.`char_length`,`col`.`name`,0) AS `CHARACTER_MAXIMUM_LENGTH`,internal_dd_char_length(`prm`.`data_type`,`prm`.`char_length`,`col`.`name`,1) AS `CHARACTER_OCTET_LENGTH`,`prm`.`numeric_precision` AS `NUMERIC_PRECISION`,if((`prm`.`numeric_precision` is null),NULL,ifnull(`prm`.`numeric_scale`,0)) AS `NUMERIC_SCALE`,`prm`.`datetime_precision` AS `DATETIME_PRECISION`,(case `prm`.`data_type` when 'MYSQL_TYPE_STRING' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_SET' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) else NULL end) AS `CHARACTER_SET_NAME`,(case `prm`.`data_type` when 'MYSQL_TYPE_STRING' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_SET' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) else NULL end) AS `COLLATION_NAME`,`prm`.`data_type_utf8` AS `DTD_IDENTIFIER`,`rtn`.`type` AS `ROUTINE_TYPE` from (((((`mysql`.`parameters` `prm` join `mysql`.`routines` `rtn` on((`prm`.`routine_id` = `rtn`.`id`))) join `mysql`.`schemata` `sch` on((`rtn`.`schema_id` = `sch`.`id`))) join `mysql`.`catalogs` `cat` on((`cat`.`id` = `sch`.`catalog_id`))) join `mysql`.`collations` `col` on((`prm`.`collation_id` = `col`.`id`))) join `mysql`.`character_sets` `cs` on((`col`.`character_set_id` = `cs`.`id`))) where (0 <> can_access_routine(`sch`.`name`,`rtn`.`name`,`rtn`.`type`,`rtn`.`definer`,false)) utf8mb3 utf8mb3_general_ci +PARAMETERS CREATE ALGORITHM=UNDEFINED DEFINER=`mysql.infoschema`@`localhost` SQL SECURITY DEFINER VIEW `PARAMETERS` AS select `cat`.`name` AS `SPECIFIC_CATALOG`,`sch`.`name` AS `SPECIFIC_SCHEMA`,`rtn`.`name` AS `SPECIFIC_NAME`,if((`rtn`.`type` = 'FUNCTION'),(`prm`.`ordinal_position` - 1),`prm`.`ordinal_position`) AS `ORDINAL_POSITION`,if(((`rtn`.`type` = 'FUNCTION') and (`prm`.`ordinal_position` = 1)),NULL,`prm`.`mode`) AS `PARAMETER_MODE`,if(((`rtn`.`type` = 'FUNCTION') and (`prm`.`ordinal_position` = 1)),NULL,`prm`.`name`) AS `PARAMETER_NAME`,substring_index(substring_index(substring_index(`prm`.`data_type_utf8`,'*/ ',-(1)),'(',1),' ',1) AS `DATA_TYPE`,internal_dd_char_length(`prm`.`data_type`,`prm`.`char_length`,`col`.`name`,0) AS `CHARACTER_MAXIMUM_LENGTH`,internal_dd_char_length(`prm`.`data_type`,`prm`.`char_length`,`col`.`name`,1) AS `CHARACTER_OCTET_LENGTH`,`prm`.`numeric_precision` AS `NUMERIC_PRECISION`,if((`prm`.`numeric_precision` is null),NULL,ifnull(`prm`.`numeric_scale`,0)) AS `NUMERIC_SCALE`,`prm`.`datetime_precision` AS `DATETIME_PRECISION`,(case `prm`.`data_type` when 'MYSQL_TYPE_STRING' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_SET' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) else NULL end) AS `CHARACTER_SET_NAME`,(case `prm`.`data_type` when 'MYSQL_TYPE_STRING' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_SET' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) else NULL end) AS `COLLATION_NAME`,`prm`.`data_type_utf8` AS `DTD_IDENTIFIER`,`rtn`.`type` AS `ROUTINE_TYPE` from (((((`mysql`.`parameters` `prm` join `mysql`.`routines` `rtn` on((`prm`.`routine_id` = `rtn`.`id`))) join `mysql`.`schemata` `sch` on((`rtn`.`schema_id` = `sch`.`id`))) join `mysql`.`catalogs` `cat` on((`cat`.`id` = `sch`.`catalog_id`))) join `mysql`.`collations` `col` on((`prm`.`collation_id` = `col`.`id`))) join `mysql`.`character_sets` `cs` on((`col`.`character_set_id` = `cs`.`id`))) where (0 <> can_access_routine(`sch`.`name`,`rtn`.`name`,`rtn`.`type`,`rtn`.`definer`,false)) utf8mb3 utf8mb3_general_ci SELECT * FROM information_schema.columns WHERE table_schema = 'information_schema' AND table_name = 'PARAMETERS' diff --git a/mysql-test/r/information_schema_routines.result b/mysql-test/r/information_schema_routines.result index 58f573230e2..fad3559ca55 100644 --- a/mysql-test/r/information_schema_routines.result +++ b/mysql-test/r/information_schema_routines.result @@ -2,7 +2,7 @@ USE INFORMATION_SCHEMA; SHOW CREATE TABLE INFORMATION_SCHEMA.ROUTINES; View Create View character_set_client collation_connection -ROUTINES CREATE ALGORITHM=UNDEFINED DEFINER=`mysql.infoschema`@`localhost` SQL SECURITY DEFINER VIEW `ROUTINES` AS select `rtn`.`name` AS `SPECIFIC_NAME`,`cat`.`name` AS `ROUTINE_CATALOG`,`sch`.`name` AS `ROUTINE_SCHEMA`,`rtn`.`name` AS `ROUTINE_NAME`,`rtn`.`type` AS `ROUTINE_TYPE`,if((`rtn`.`type` = 'PROCEDURE'),'',substring_index(substring_index(`rtn`.`result_data_type_utf8`,'(',1),' ',1)) AS `DATA_TYPE`,internal_dd_char_length(`rtn`.`result_data_type`,`rtn`.`result_char_length`,`coll_result`.`name`,0) AS `CHARACTER_MAXIMUM_LENGTH`,internal_dd_char_length(`rtn`.`result_data_type`,`rtn`.`result_char_length`,`coll_result`.`name`,1) AS `CHARACTER_OCTET_LENGTH`,`rtn`.`result_numeric_precision` AS `NUMERIC_PRECISION`,`rtn`.`result_numeric_scale` AS `NUMERIC_SCALE`,`rtn`.`result_datetime_precision` AS `DATETIME_PRECISION`,(case `rtn`.`result_data_type` when 'MYSQL_TYPE_STRING' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_SET' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) else NULL end) AS `CHARACTER_SET_NAME`,(case `rtn`.`result_data_type` when 'MYSQL_TYPE_STRING' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_SET' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) else NULL end) AS `COLLATION_NAME`,if((`rtn`.`type` = 'PROCEDURE'),NULL,`rtn`.`result_data_type_utf8`) AS `DTD_IDENTIFIER`,'SQL' AS `ROUTINE_BODY`,if(can_access_routine(`sch`.`name`,`rtn`.`name`,`rtn`.`type`,`rtn`.`definer`,true),`rtn`.`definition_utf8`,NULL) AS `ROUTINE_DEFINITION`,NULL AS `EXTERNAL_NAME`,`rtn`.`external_language` AS `EXTERNAL_LANGUAGE`,'SQL' AS `PARAMETER_STYLE`,if((`rtn`.`is_deterministic` = 0),'NO','YES') AS `IS_DETERMINISTIC`,`rtn`.`sql_data_access` AS `SQL_DATA_ACCESS`,NULL AS `SQL_PATH`,`rtn`.`security_type` AS `SECURITY_TYPE`,`rtn`.`created` AS `CREATED`,`rtn`.`last_altered` AS `LAST_ALTERED`,`rtn`.`sql_mode` AS `SQL_MODE`,`rtn`.`comment` AS `ROUTINE_COMMENT`,`rtn`.`definer` AS `DEFINER`,`cs_client`.`name` AS `CHARACTER_SET_CLIENT`,`coll_conn`.`name` AS `COLLATION_CONNECTION`,`coll_db`.`name` AS `DATABASE_COLLATION` from ((((((((`mysql`.`routines` `rtn` join `mysql`.`schemata` `sch` on((`rtn`.`schema_id` = `sch`.`id`))) join `mysql`.`catalogs` `cat` on((`cat`.`id` = `sch`.`catalog_id`))) join `mysql`.`collations` `coll_client` on((`coll_client`.`id` = `rtn`.`client_collation_id`))) join `mysql`.`character_sets` `cs_client` on((`cs_client`.`id` = `coll_client`.`character_set_id`))) join `mysql`.`collations` `coll_conn` on((`coll_conn`.`id` = `rtn`.`connection_collation_id`))) join `mysql`.`collations` `coll_db` on((`coll_db`.`id` = `rtn`.`schema_collation_id`))) left join `mysql`.`collations` `coll_result` on((`coll_result`.`id` = `rtn`.`result_collation_id`))) left join `mysql`.`character_sets` `cs_result` on((`cs_result`.`id` = `coll_result`.`character_set_id`))) where (0 <> can_access_routine(`sch`.`name`,`rtn`.`name`,`rtn`.`type`,`rtn`.`definer`,false)) utf8mb3 utf8mb3_general_ci +ROUTINES CREATE ALGORITHM=UNDEFINED DEFINER=`mysql.infoschema`@`localhost` SQL SECURITY DEFINER VIEW `ROUTINES` AS select `rtn`.`name` AS `SPECIFIC_NAME`,`cat`.`name` AS `ROUTINE_CATALOG`,`sch`.`name` AS `ROUTINE_SCHEMA`,`rtn`.`name` AS `ROUTINE_NAME`,`rtn`.`type` AS `ROUTINE_TYPE`,if((`rtn`.`type` = 'PROCEDURE'),'',substring_index(substring_index(substring_index(`rtn`.`result_data_type_utf8`,'*/ ',-(1)),'(',1),' ',1)) AS `DATA_TYPE`,internal_dd_char_length(`rtn`.`result_data_type`,`rtn`.`result_char_length`,`coll_result`.`name`,0) AS `CHARACTER_MAXIMUM_LENGTH`,internal_dd_char_length(`rtn`.`result_data_type`,`rtn`.`result_char_length`,`coll_result`.`name`,1) AS `CHARACTER_OCTET_LENGTH`,`rtn`.`result_numeric_precision` AS `NUMERIC_PRECISION`,`rtn`.`result_numeric_scale` AS `NUMERIC_SCALE`,`rtn`.`result_datetime_precision` AS `DATETIME_PRECISION`,(case `rtn`.`result_data_type` when 'MYSQL_TYPE_STRING' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) when 'MYSQL_TYPE_SET' then if((`cs_result`.`name` = 'binary'),NULL,`cs_result`.`name`) else NULL end) AS `CHARACTER_SET_NAME`,(case `rtn`.`result_data_type` when 'MYSQL_TYPE_STRING' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) when 'MYSQL_TYPE_SET' then if((`cs_result`.`name` = 'binary'),NULL,`coll_result`.`name`) else NULL end) AS `COLLATION_NAME`,if((`rtn`.`type` = 'PROCEDURE'),NULL,`rtn`.`result_data_type_utf8`) AS `DTD_IDENTIFIER`,'SQL' AS `ROUTINE_BODY`,if(can_access_routine(`sch`.`name`,`rtn`.`name`,`rtn`.`type`,`rtn`.`definer`,true),`rtn`.`definition_utf8`,NULL) AS `ROUTINE_DEFINITION`,NULL AS `EXTERNAL_NAME`,`rtn`.`external_language` AS `EXTERNAL_LANGUAGE`,'SQL' AS `PARAMETER_STYLE`,if((`rtn`.`is_deterministic` = 0),'NO','YES') AS `IS_DETERMINISTIC`,`rtn`.`sql_data_access` AS `SQL_DATA_ACCESS`,NULL AS `SQL_PATH`,`rtn`.`security_type` AS `SECURITY_TYPE`,`rtn`.`created` AS `CREATED`,`rtn`.`last_altered` AS `LAST_ALTERED`,`rtn`.`sql_mode` AS `SQL_MODE`,`rtn`.`comment` AS `ROUTINE_COMMENT`,`rtn`.`definer` AS `DEFINER`,`cs_client`.`name` AS `CHARACTER_SET_CLIENT`,`coll_conn`.`name` AS `COLLATION_CONNECTION`,`coll_db`.`name` AS `DATABASE_COLLATION` from ((((((((`mysql`.`routines` `rtn` join `mysql`.`schemata` `sch` on((`rtn`.`schema_id` = `sch`.`id`))) join `mysql`.`catalogs` `cat` on((`cat`.`id` = `sch`.`catalog_id`))) join `mysql`.`collations` `coll_client` on((`coll_client`.`id` = `rtn`.`client_collation_id`))) join `mysql`.`character_sets` `cs_client` on((`cs_client`.`id` = `coll_client`.`character_set_id`))) join `mysql`.`collations` `coll_conn` on((`coll_conn`.`id` = `rtn`.`connection_collation_id`))) join `mysql`.`collations` `coll_db` on((`coll_db`.`id` = `rtn`.`schema_collation_id`))) left join `mysql`.`collations` `coll_result` on((`coll_result`.`id` = `rtn`.`result_collation_id`))) left join `mysql`.`character_sets` `cs_result` on((`cs_result`.`id` = `coll_result`.`character_set_id`))) where (0 <> can_access_routine(`sch`.`name`,`rtn`.`name`,`rtn`.`type`,`rtn`.`definer`,false)) utf8mb3 utf8mb3_general_ci SELECT * FROM information_schema.columns WHERE table_schema = 'information_schema' AND table_name = 'ROUTINES' diff --git a/mysql-test/r/no_binlog_gtid_empty_statement.result b/mysql-test/r/no_binlog_gtid_empty_statement.result index fe73164143f..cba023da268 100644 --- a/mysql-test/r/no_binlog_gtid_empty_statement.result +++ b/mysql-test/r/no_binlog_gtid_empty_statement.result @@ -1,10 +1,10 @@ USE test; SET GTID_NEXT= '#'; CREATE TABLE t1 (c1 INT); -/*!99999 SET @@SESSION.non_supported_session_variable = 1*/; +/*!99998 SET @@SESSION.non_supported_session_variable = 1*/; SET GTID_NEXT= '#'; INSERT INTO t1 VALUES (1); -/*!99999 SET @@SESSION.non_supported_session_variable = 1*/; +/*!99998 SET @@SESSION.non_supported_session_variable = 1*/; SET GTID_NEXT= '#'; DROP TABLE t1; include/rpl_set_gtid_mode.inc [ON on servers 1] @@ -12,10 +12,10 @@ SET GTID_NEXT='AUTOMATIC'; RESET MASTER; SET GTID_NEXT= '#'; CREATE TABLE t2 (c1 INT); -/*!99999 SET @@SESSION.non_supported_session_variable = 1*/; +/*!99998 SET @@SESSION.non_supported_session_variable = 1*/; SET GTID_NEXT= '#'; INSERT INTO t2 VALUES (1); -/*!99999 SET @@SESSION.non_supported_session_variable = 1*/; +/*!99998 SET @@SESSION.non_supported_session_variable = 1*/; SET GTID_NEXT= '#'; DROP TABLE t2; SET GTID_NEXT='AUTOMATIC'; diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index a7e413ef53c..6efc0e98ab6 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -6241,14 +6241,14 @@ select 1; /*! select 2; */ select 3; /*!00000 select 4; */ -/*!99999 select 5; */ +/*!99998 select 5; */ end $$ create procedure proc_25411_b( /* real comment */ /*! p1 int, */ /*!00000 p2 int */ -/*!99999 ,p3 int */ +/*!99998 ,p3 int */ ) begin select p1, p2; @@ -6256,11 +6256,11 @@ end $$ create procedure proc_25411_c() begin -select 1/*!,2*//*!00000,3*//*!99999,4*/; -select 1/*! ,2*//*!00000 ,3*//*!99999 ,4*/; -select 1/*!,2 *//*!00000,3 *//*!99999,4 */; -select 1/*! ,2 *//*!00000 ,3 *//*!99999 ,4 */; -select 1 /*!,2*/ /*!00000,3*/ /*!99999,4*/ ; +select 1/*!,2*//*!00000,3*//*!99998,4*/; +select 1/*! ,2*//*!00000 ,3*//*!99998 ,4*/; +select 1/*!,2 *//*!00000,3 *//*!99998,4 */; +select 1/*! ,2 *//*!00000 ,3 *//*!99998 ,4 */; +select 1 /*!,2*/ /*!00000,3*/ /*!99998,4*/ ; end $$ Warnings: diff --git a/mysql-test/r/sql_safe_updates_cmdline.result b/mysql-test/r/sql_safe_updates_cmdline.result new file mode 100644 index 00000000000..6bcfe791e1e --- /dev/null +++ b/mysql-test/r/sql_safe_updates_cmdline.result @@ -0,0 +1,83 @@ +# +# Test 1: Verify global and session default values are ON +# when server is started with --sql-safe-updates=1 +# +SELECT @@global.sql_safe_updates AS global_value; +global_value +1 +SELECT @@session.sql_safe_updates AS session_value; +session_value +1 +# +# Test 2: Verify that sql_safe_updates is enforced +# (UPDATE without WHERE using KEY should fail) +# +CREATE TABLE t1 (id INT PRIMARY KEY, name VARCHAR(50)); +INSERT INTO t1 VALUES (1, 'test1'), (2, 'test2'), (3, 'test3'); +# This should fail with sql_safe_updates enabled +UPDATE t1 SET name = 'updated'; +ERROR HY000: You are using safe update mode and you tried to update a table without a WHERE that uses a KEY column. +# This should fail with sql_safe_updates enabled +DELETE FROM t1; +ERROR HY000: You are using safe update mode and you tried to update a table without a WHERE that uses a KEY column. +# UPDATE with WHERE using KEY column should succeed +UPDATE t1 SET name = 'updated1' WHERE id = 1; +SELECT * FROM t1 WHERE id = 1; +id name +1 updated1 +# +# Test 3: Verify session can override the global value +# +SET SESSION sql_safe_updates = OFF; +SELECT @@session.sql_safe_updates AS session_after_off; +session_after_off +0 +SELECT @@global.sql_safe_updates AS global_unchanged; +global_unchanged +1 +# Now UPDATE without WHERE should succeed +UPDATE t1 SET name = 'all_updated'; +SELECT * FROM t1 ORDER BY id; +id name +1 all_updated +2 all_updated +3 all_updated +# +# Test 4: Verify new session inherits global value +# +# Reconnect to get a new session +SELECT @@session.sql_safe_updates AS new_session_value; +new_session_value +1 +SELECT @@global.sql_safe_updates AS global_value; +global_value +1 +# UPDATE without WHERE should fail in new session +UPDATE t1 SET name = 'fail'; +ERROR HY000: You are using safe update mode and you tried to update a table without a WHERE that uses a KEY column. +# +# Test 5: Verify SET GLOBAL works +# +SET GLOBAL sql_safe_updates = OFF; +SELECT @@global.sql_safe_updates AS global_after_set_off; +global_after_set_off +0 +# Reconnect to verify new session gets the new global value +SELECT @@session.sql_safe_updates AS new_session_inherits_global; +new_session_inherits_global +0 +# UPDATE without WHERE should now succeed in new session +UPDATE t1 SET name = 'success'; +SELECT * FROM t1 ORDER BY id; +id name +1 success +2 success +3 success +# +# Cleanup +# +SET GLOBAL sql_safe_updates = OFF; +DROP TABLE t1; +# +# Test completed successfully +# diff --git a/mysql-test/suite/binlog/r/binlog_rewrite.result b/mysql-test/suite/binlog/r/binlog_rewrite.result index 612c2a4d3ea..e526d1721ea 100644 --- a/mysql-test/suite/binlog/r/binlog_rewrite.result +++ b/mysql-test/suite/binlog/r/binlog_rewrite.result @@ -19,7 +19,7 @@ GRANT EXECUTE ON PROCEDURE p1 /*before to*/TO/*after to*/ user1@localhost, user2 GRANT EXECUTE ON PROCEDURE p1 TO user1@localhost, user2@localhost; REVOKE EXECUTE ON PROCEDURE p1 FROM user1@localhost, user2@localhost; CREATE FUNCTION f1() RETURNS INT RETURN 123; -GRANT EXECUTE ON FUNCTION f1 /*before to*/TO/*after to*/ user1@localhost/*!10000 , user2@localhost*/ /*!99999 THIS_WOULD_BREAK */; +GRANT EXECUTE ON FUNCTION f1 /*before to*/TO/*after to*/ user1@localhost/*!10000 , user2@localhost*/ /*!99998 THIS_WOULD_BREAK */; REVOKE EXECUTE ON FUNCTION f1 FROM user1@localhost, user2@localhost; DROP USER user1@localhost, user2@localhost; DROP FUNCTION f1; diff --git a/mysql-test/suite/binlog/t/binlog_rewrite.test b/mysql-test/suite/binlog/t/binlog_rewrite.test index 62534fbd1ef..cc6739e647a 100644 --- a/mysql-test/suite/binlog/t/binlog_rewrite.test +++ b/mysql-test/suite/binlog/t/binlog_rewrite.test @@ -40,7 +40,7 @@ GRANT EXECUTE ON PROCEDURE p1 TO user1@localhost, user2@localhost; REVOKE EXECUTE ON PROCEDURE p1 FROM user1@localhost, user2@localhost; CREATE FUNCTION f1() RETURNS INT RETURN 123; -GRANT EXECUTE ON FUNCTION f1 /*before to*/TO/*after to*/ user1@localhost/*!10000 , user2@localhost*/ /*!99999 THIS_WOULD_BREAK */; +GRANT EXECUTE ON FUNCTION f1 /*before to*/TO/*after to*/ user1@localhost/*!10000 , user2@localhost*/ /*!99998 THIS_WOULD_BREAK */; REVOKE EXECUTE ON FUNCTION f1 FROM user1@localhost, user2@localhost; DROP USER user1@localhost, user2@localhost; diff --git a/mysql-test/suite/component_keyring_file/r/database.result b/mysql-test/suite/component_keyring_file/r/database.result index 544ccfeba68..1f119c11d3b 100644 --- a/mysql-test/suite/component_keyring_file/r/database.result +++ b/mysql-test/suite/component_keyring_file/r/database.result @@ -449,7 +449,7 @@ DROP DATABASE db1; CREATE DATABASE `db1` /*!99999 DEFAULT ENCRYPTION='Y' */; SHOW CREATE DATABASE db1; Database Create Database -db1 CREATE DATABASE `db1` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */ +db1 CREATE DATABASE `db1` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='Y' */ DROP DATABASE db1; ````````````````````````````````````````````````````````` # ALTER DATABASE withDEFAULT ENCRYPTION clause 'y/n' diff --git a/mysql-test/suite/duckdb/r/duckdb_invalid_datetime.result b/mysql-test/suite/duckdb/r/duckdb_invalid_datetime.result new file mode 100644 index 00000000000..e9a338871b7 --- /dev/null +++ b/mysql-test/suite/duckdb/r/duckdb_invalid_datetime.result @@ -0,0 +1,157 @@ +# +# Setup +# +# ========================================================= +# Scenario 1: INSERT with invalid integer (57399) +# This is the original crash case from GitHub issue#131. +# ========================================================= +DROP TABLE IF EXISTS t_duckdb, t_innodb; +CREATE TABLE t_duckdb ( +id INT NOT NULL AUTO_INCREMENT, +col1 DATETIME NULL DEFAULT '2026-01-01 00:00:00', +PRIMARY KEY (id) +) ENGINE=DuckDB; +CREATE TABLE t_innodb ( +id INT NOT NULL AUTO_INCREMENT, +col1 DATETIME NULL DEFAULT '2026-01-01 00:00:00', +PRIMARY KEY (id) +) ENGINE=InnoDB; +INSERT IGNORE INTO t_duckdb (id, col1) VALUES (1, 57399); +Warnings: +Warning 1265 Data truncated for column 'col1' at row 1 +INSERT IGNORE INTO t_innodb (id, col1) VALUES (1, 57399); +Warnings: +Warning 1265 Data truncated for column 'col1' at row 1 +INSERT INTO t_duckdb (id, col1) VALUES (2, 57399); +ERROR 22007: Incorrect datetime value: '57399' for column 'col1' at row 1 +INSERT INTO t_innodb (id, col1) VALUES (2, 57399); +ERROR 22007: Incorrect datetime value: '57399' for column 'col1' at row 1 +SET @saved_sql_mode = @@SESSION.sql_mode; +SET sql_mode = ''; +INSERT INTO t_duckdb (id, col1) VALUES (2, 57399); +Warnings: +Warning 1265 Data truncated for column 'col1' at row 1 +INSERT INTO t_innodb (id, col1) VALUES (2, 57399); +Warnings: +Warning 1265 Data truncated for column 'col1' at row 1 +SET sql_mode = @saved_sql_mode; +SELECT * FROM t_duckdb ORDER BY id; +id col1 +1 0000-00-00 00:00:00 +2 0000-00-00 00:00:00 +SELECT * FROM t_innodb ORDER BY id; +id col1 +1 0000-00-00 00:00:00 +2 0000-00-00 00:00:00 +include/assert.inc [Checksum is the same] +# Scenario 1 PASSED: checksums match +# ================================================== +# Scenario 2: INSERT with various invalid integers +# ================================================== +DROP TABLE IF EXISTS t_duckdb, t_innodb; +CREATE TABLE t_duckdb ( +id INT NOT NULL AUTO_INCREMENT, +col1 DATETIME NULL DEFAULT '2026-01-01 00:00:00', +PRIMARY KEY (id) +) ENGINE=DuckDB; +CREATE TABLE t_innodb ( +id INT NOT NULL AUTO_INCREMENT, +col1 DATETIME NULL DEFAULT '2026-01-01 00:00:00', +PRIMARY KEY (id) +) ENGINE=InnoDB; +INSERT IGNORE INTO t_duckdb (id, col1) VALUES +(1, 0), (2, 99), (3, 101), (4, 57399), (5, 13320199), (6, -1); +Warnings: +Warning 1264 Out of range value for column 'col1' at row 1 +Warning 1265 Data truncated for column 'col1' at row 2 +Warning 1265 Data truncated for column 'col1' at row 4 +Warning 1265 Data truncated for column 'col1' at row 5 +Warning 1264 Out of range value for column 'col1' at row 6 +INSERT IGNORE INTO t_innodb (id, col1) VALUES +(1, 0), (2, 99), (3, 101), (4, 57399), (5, 13320199), (6, -1); +Warnings: +Warning 1264 Out of range value for column 'col1' at row 1 +Warning 1265 Data truncated for column 'col1' at row 2 +Warning 1265 Data truncated for column 'col1' at row 4 +Warning 1265 Data truncated for column 'col1' at row 5 +Warning 1264 Out of range value for column 'col1' at row 6 +SELECT * FROM t_duckdb ORDER BY id; +id col1 +1 0000-00-00 00:00:00 +2 0000-00-00 00:00:00 +3 2000-01-01 00:00:00 +4 0000-00-00 00:00:00 +5 0000-00-00 00:00:00 +6 0000-00-00 00:00:00 +SELECT * FROM t_innodb ORDER BY id; +id col1 +1 0000-00-00 00:00:00 +2 0000-00-00 00:00:00 +3 2000-01-01 00:00:00 +4 0000-00-00 00:00:00 +5 0000-00-00 00:00:00 +6 0000-00-00 00:00:00 +include/assert.inc [Checksum is the same] +# Scenario 2 PASSED: checksums match +# ========================================================= +# Scenario 3: INSERT with mixing valid and invalid values +# ========================================================= +DROP TABLE IF EXISTS t_duckdb, t_innodb; +CREATE TABLE t_duckdb ( +id INT NOT NULL AUTO_INCREMENT, +col1 DATETIME NULL DEFAULT '2026-01-01 00:00:00', +PRIMARY KEY (id) +) ENGINE=DuckDB; +CREATE TABLE t_innodb ( +id INT NOT NULL AUTO_INCREMENT, +col1 DATETIME NULL DEFAULT '2026-01-01 00:00:00', +PRIMARY KEY (id) +) ENGINE=InnoDB; +INSERT IGNORE INTO t_duckdb (id, col1) VALUES +(1, '2026-06-15 10:30:00'), +(2, 57399), +(3, '2020-01-01 00:00:00'), +(4, 0), +(5, 20240615103000), +(6, NULL), +(7, '0000-00-00 00:00:00'); +Warnings: +Warning 1265 Data truncated for column 'col1' at row 2 +Warning 1264 Out of range value for column 'col1' at row 4 +Warning 1264 Out of range value for column 'col1' at row 7 +INSERT IGNORE INTO t_innodb (id, col1) VALUES +(1, '2026-06-15 10:30:00'), +(2, 57399), +(3, '2020-01-01 00:00:00'), +(4, 0), +(5, 20240615103000), +(6, NULL), +(7, '0000-00-00 00:00:00'); +Warnings: +Warning 1265 Data truncated for column 'col1' at row 2 +Warning 1264 Out of range value for column 'col1' at row 4 +Warning 1264 Out of range value for column 'col1' at row 7 +SELECT * FROM t_duckdb ORDER BY id; +id col1 +1 2026-06-15 10:30:00 +2 0000-00-00 00:00:00 +3 2020-01-01 00:00:00 +4 0000-00-00 00:00:00 +5 2024-06-15 10:30:00 +6 NULL +7 0000-00-00 00:00:00 +SELECT * FROM t_innodb ORDER BY id; +id col1 +1 2026-06-15 10:30:00 +2 0000-00-00 00:00:00 +3 2020-01-01 00:00:00 +4 0000-00-00 00:00:00 +5 2024-06-15 10:30:00 +6 NULL +7 0000-00-00 00:00:00 +include/assert.inc [Checksum is the same] +# Scenario 3 PASSED: checksums match +# +# Cleanup +# +DROP TABLE t_duckdb, t_innodb; diff --git a/mysql-test/suite/duckdb/t/duckdb_invalid_datetime.test b/mysql-test/suite/duckdb/t/duckdb_invalid_datetime.test new file mode 100644 index 00000000000..ab2d67a11ba --- /dev/null +++ b/mysql-test/suite/duckdb/t/duckdb_invalid_datetime.test @@ -0,0 +1,124 @@ +# =========================================================== +# Test: duckdb_datetime_zero_date +# Description: +# Regression test for GitHub issue#131. +# Inserting an invalid integer (e.g. 57399) into a DATETIME +# column on a DuckDB engine table used to crash the server +# due to an assertion failure in sec_since_epoch() when +# month=0 (zero date). +# +# This test verifies that DuckDB and InnoDB produce +# identical results for various invalid/zero-date INSERT +# scenarios. +# +# Tested feature: DuckDB DATETIME zero-date handling +# Related source: storage/duckdb/delta_appender.cc +# =========================================================== + +--echo # +--echo # Setup +--echo # +--write_file $MYSQL_TMP_DIR/init_datetime_test.inc + --disable_warnings + DROP TABLE IF EXISTS t_duckdb, t_innodb; + --enable_warnings + CREATE TABLE t_duckdb ( + id INT NOT NULL AUTO_INCREMENT, + col1 DATETIME NULL DEFAULT '2026-01-01 00:00:00', + PRIMARY KEY (id) + ) ENGINE=DuckDB; + + CREATE TABLE t_innodb ( + id INT NOT NULL AUTO_INCREMENT, + col1 DATETIME NULL DEFAULT '2026-01-01 00:00:00', + PRIMARY KEY (id) + ) ENGINE=InnoDB; +EOF + +--write_file $MYSQL_TMP_DIR/check_duckdb_datetime.inc + SELECT * FROM t_duckdb ORDER BY id; + SELECT * FROM t_innodb ORDER BY id; + + --let $checksum_t_innodb = query_get_value(CHECKSUM TABLE t_innodb, Checksum, 1) + --let $checksum_t_duckdb = query_get_value(CHECKSUM TABLE t_duckdb, Checksum, 1) + --let $assert_cond= $checksum_t_innodb = $checksum_t_duckdb + --let $assert_text= Checksum is the same + --source include/assert.inc +EOF + + +--echo # ========================================================= +--echo # Scenario 1: INSERT with invalid integer (57399) +--echo # This is the original crash case from GitHub issue#131. +--echo # ========================================================= +--source $MYSQL_TMP_DIR/init_datetime_test.inc + +INSERT IGNORE INTO t_duckdb (id, col1) VALUES (1, 57399); +INSERT IGNORE INTO t_innodb (id, col1) VALUES (1, 57399); + +--error ER_TRUNCATED_WRONG_VALUE +INSERT INTO t_duckdb (id, col1) VALUES (2, 57399); +--error ER_TRUNCATED_WRONG_VALUE +INSERT INTO t_innodb (id, col1) VALUES (2, 57399); + +SET @saved_sql_mode = @@SESSION.sql_mode; +SET sql_mode = ''; +INSERT INTO t_duckdb (id, col1) VALUES (2, 57399); +INSERT INTO t_innodb (id, col1) VALUES (2, 57399); +SET sql_mode = @saved_sql_mode; + +--source $MYSQL_TMP_DIR/check_duckdb_datetime.inc +--echo # Scenario 1 PASSED: checksums match + + +--echo # ================================================== +--echo # Scenario 2: INSERT with various invalid integers +--echo # ================================================== +--source $MYSQL_TMP_DIR/init_datetime_test.inc + +# 0: zero value +# 101: boundary (min valid YYMMDD-like is 101 -> 2000-01-01) +# 99: below minimum valid +# 57399: invalid month=73 +# 13320199: invalid day=99 +# -1: negative value +INSERT IGNORE INTO t_duckdb (id, col1) VALUES + (1, 0), (2, 99), (3, 101), (4, 57399), (5, 13320199), (6, -1); +INSERT IGNORE INTO t_innodb (id, col1) VALUES + (1, 0), (2, 99), (3, 101), (4, 57399), (5, 13320199), (6, -1); + +--source $MYSQL_TMP_DIR/check_duckdb_datetime.inc +--echo # Scenario 2 PASSED: checksums match + + +--echo # ========================================================= +--echo # Scenario 3: INSERT with mixing valid and invalid values +--echo # ========================================================= +--source $MYSQL_TMP_DIR/init_datetime_test.inc + +INSERT IGNORE INTO t_duckdb (id, col1) VALUES + (1, '2026-06-15 10:30:00'), + (2, 57399), + (3, '2020-01-01 00:00:00'), + (4, 0), + (5, 20240615103000), + (6, NULL), + (7, '0000-00-00 00:00:00'); +INSERT IGNORE INTO t_innodb (id, col1) VALUES + (1, '2026-06-15 10:30:00'), + (2, 57399), + (3, '2020-01-01 00:00:00'), + (4, 0), + (5, 20240615103000), + (6, NULL), + (7, '0000-00-00 00:00:00'); + +--source $MYSQL_TMP_DIR/check_duckdb_datetime.inc +--echo # Scenario 3 PASSED: checksums match + +--echo # +--echo # Cleanup +--echo # +DROP TABLE t_duckdb, t_innodb; +--remove_file $MYSQL_TMP_DIR/init_datetime_test.inc +--remove_file $MYSQL_TMP_DIR/check_duckdb_datetime.inc diff --git a/mysql-test/suite/encryption/r/database.result b/mysql-test/suite/encryption/r/database.result index fa51b1b8c30..64cc2ac14ce 100644 --- a/mysql-test/suite/encryption/r/database.result +++ b/mysql-test/suite/encryption/r/database.result @@ -440,7 +440,7 @@ db1 CREATE DATABASE `db1` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4 DROP DATABASE db1; ````````````````````````````````````````````````````````` # See that we ignore the clause with invalid mysql version. -CREATE DATABASE `db1` /*!99999 DEFAULT ENCRYPTION='Y' */; +CREATE DATABASE `db1` /*!99998 DEFAULT ENCRYPTION='Y' */; SHOW CREATE DATABASE db1; Database Create Database db1 CREATE DATABASE `db1` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */ diff --git a/mysql-test/suite/encryption/t/database.test b/mysql-test/suite/encryption/t/database.test index 462fac399a4..3efaa2bec5f 100644 --- a/mysql-test/suite/encryption/t/database.test +++ b/mysql-test/suite/encryption/t/database.test @@ -137,7 +137,7 @@ SHOW CREATE DATABASE db1; DROP DATABASE db1; --echo ````````````````````````````````````````````````````````` --echo # See that we ignore the clause with invalid mysql version. -CREATE DATABASE `db1` /*!99999 DEFAULT ENCRYPTION='Y' */; +CREATE DATABASE `db1` /*!99998 DEFAULT ENCRYPTION='Y' */; SHOW CREATE DATABASE db1; DROP DATABASE db1; diff --git a/mysql-test/suite/information_schema/r/i_s_schema_definition_debug.result b/mysql-test/suite/information_schema/r/i_s_schema_definition_debug.result index deea8ee5e32..35ac283e4a2 100644 --- a/mysql-test/suite/information_schema/r/i_s_schema_definition_debug.result +++ b/mysql-test/suite/information_schema/r/i_s_schema_definition_debug.result @@ -209,7 +209,7 @@ CREATE OR REPLACE DEFINER=`mysql.infoschema`@`localhost` VIEW information_schema col.ordinal_position AS ORDINAL_POSITION, col.default_value_utf8 AS COLUMN_DEFAULT, IF (col.is_nullable = 1, 'YES','NO') AS IS_NULLABLE, - SUBSTRING_INDEX(SUBSTRING_INDEX(col.column_type_utf8, '(', 1),' ', 1) AS DATA_TYPE, + SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(col.column_type_utf8, '*/ ', -1), '(', 1),' ', 1) AS DATA_TYPE, INTERNAL_DD_CHAR_LENGTH(col.type, col.char_length, coll.name, 0) AS CHARACTER_MAXIMUM_LENGTH, INTERNAL_DD_CHAR_LENGTH(col.type, col.char_length, coll.name, 1) AS CHARACTER_OCTET_LENGTH, IF (col.numeric_precision = 0, NULL, col.numeric_precision) AS NUMERIC_PRECISION, @@ -242,7 +242,7 @@ INSERT INTO I_S_check_table(t) VALUES ("CREATE OR REPLACE DEFINER=`mysql.infosch col.ordinal_position AS ORDINAL_POSITION, col.default_value_utf8 AS COLUMN_DEFAULT, IF (col.is_nullable = 1, 'YES','NO') AS IS_NULLABLE, - SUBSTRING_INDEX(SUBSTRING_INDEX(col.column_type_utf8, '(', 1),' ', 1) AS DATA_TYPE, + SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(col.column_type_utf8, '*/ ', -1), '(', 1),' ', 1) AS DATA_TYPE, INTERNAL_DD_CHAR_LENGTH(col.type, col.char_length, coll.name, 0) AS CHARACTER_MAXIMUM_LENGTH, INTERNAL_DD_CHAR_LENGTH(col.type, col.char_length, coll.name, 1) AS CHARACTER_OCTET_LENGTH, IF (col.numeric_precision = 0, NULL, col.numeric_precision) AS NUMERIC_PRECISION, @@ -660,7 +660,7 @@ CREATE OR REPLACE DEFINER=`mysql.infoschema`@`localhost` VIEW information_schema IF (rtn.type = 'FUNCTION', prm.ordinal_position-1, prm.ordinal_position) AS ORDINAL_POSITION, IF (rtn.type = 'FUNCTION' AND prm.ordinal_position = 1, NULL, prm.mode) AS PARAMETER_MODE, IF (rtn.type = 'FUNCTION' AND prm.ordinal_position = 1, NULL, prm.name) AS PARAMETER_NAME, - SUBSTRING_INDEX(SUBSTRING_INDEX(prm.data_type_utf8, '(', 1), ' ', 1) AS DATA_TYPE, + SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(prm.data_type_utf8, '*/ ', -1), '(', 1), ' ', 1) AS DATA_TYPE, INTERNAL_DD_CHAR_LENGTH(prm.data_type, prm.char_length, col.name, 0) AS CHARACTER_MAXIMUM_LENGTH, INTERNAL_DD_CHAR_LENGTH(prm.data_type, prm.char_length, col.name, 1) AS CHARACTER_OCTET_LENGTH, prm.numeric_precision AS NUMERIC_PRECISION, @@ -684,7 +684,7 @@ INSERT INTO I_S_check_table(t) VALUES ("CREATE OR REPLACE DEFINER=`mysql.infosch IF (rtn.type = 'FUNCTION', prm.ordinal_position-1, prm.ordinal_position) AS ORDINAL_POSITION, IF (rtn.type = 'FUNCTION' AND prm.ordinal_position = 1, NULL, prm.mode) AS PARAMETER_MODE, IF (rtn.type = 'FUNCTION' AND prm.ordinal_position = 1, NULL, prm.name) AS PARAMETER_NAME, - SUBSTRING_INDEX(SUBSTRING_INDEX(prm.data_type_utf8, '(', 1), ' ', 1) AS DATA_TYPE, + SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(prm.data_type_utf8, '*/ ', -1), '(', 1), ' ', 1) AS DATA_TYPE, INTERNAL_DD_CHAR_LENGTH(prm.data_type, prm.char_length, col.name, 0) AS CHARACTER_MAXIMUM_LENGTH, INTERNAL_DD_CHAR_LENGTH(prm.data_type, prm.char_length, col.name, 1) AS CHARACTER_OCTET_LENGTH, prm.numeric_precision AS NUMERIC_PRECISION, @@ -837,7 +837,7 @@ CREATE OR REPLACE DEFINER=`mysql.infoschema`@`localhost` VIEW information_schema sch.name AS ROUTINE_SCHEMA, rtn.name AS ROUTINE_NAME, rtn.type AS ROUTINE_TYPE, - IF(rtn.type = 'PROCEDURE', '', SUBSTRING_INDEX(SUBSTRING_INDEX( rtn.result_data_type_utf8, '(', 1), ' ', 1)) AS DATA_TYPE, + IF(rtn.type = 'PROCEDURE', '', SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX( rtn.result_data_type_utf8, '*/ ', -1), '(', 1), ' ', 1)) AS DATA_TYPE, INTERNAL_DD_CHAR_LENGTH(rtn.result_data_type, rtn.result_char_length, coll_result.name, 0) AS CHARACTER_MAXIMUM_LENGTH, INTERNAL_DD_CHAR_LENGTH(rtn.result_data_type, rtn.result_char_length, coll_result.name, 1) AS CHARACTER_OCTET_LENGTH, rtn.result_numeric_precision AS NUMERIC_PRECISION, @@ -880,7 +880,7 @@ INSERT INTO I_S_check_table(t) VALUES ("CREATE OR REPLACE DEFINER=`mysql.infosch sch.name AS ROUTINE_SCHEMA, rtn.name AS ROUTINE_NAME, rtn.type AS ROUTINE_TYPE, - IF(rtn.type = 'PROCEDURE', '', SUBSTRING_INDEX(SUBSTRING_INDEX( rtn.result_data_type_utf8, '(', 1), ' ', 1)) AS DATA_TYPE, + IF(rtn.type = 'PROCEDURE', '', SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX( rtn.result_data_type_utf8, '*/ ', -1), '(', 1), ' ', 1)) AS DATA_TYPE, INTERNAL_DD_CHAR_LENGTH(rtn.result_data_type, rtn.result_char_length, coll_result.name, 0) AS CHARACTER_MAXIMUM_LENGTH, INTERNAL_DD_CHAR_LENGTH(rtn.result_data_type, rtn.result_char_length, coll_result.name, 1) AS CHARACTER_OCTET_LENGTH, rtn.result_numeric_precision AS NUMERIC_PRECISION, diff --git a/mysql-test/suite/information_schema/t/i_s_schema_definition_debug.test b/mysql-test/suite/information_schema/t/i_s_schema_definition_debug.test index c2090783942..ffde71094a3 100644 --- a/mysql-test/suite/information_schema/t/i_s_schema_definition_debug.test +++ b/mysql-test/suite/information_schema/t/i_s_schema_definition_debug.test @@ -116,9 +116,9 @@ eval CREATE TABLE $I_S_view_names (name VARCHAR(64) PRIMARY KEY); --echo # Print the actual I_S version stored on disk. let $current_i_s_version = `SELECT SUBSTRING_INDEX( - SUBSTRING_INDEX(SUBSTRING(properties, - LOCATE('IS_VERSION', properties), 30), ';', 1) - , '=', -1) AS I_S_VERSION + SUBSTRING_INDEX( + CONCAT(';', properties), ';IS_VERSION=', -1) + , ';', 1) AS I_S_VERSION FROM mysql.dd_properties`; --echo Current I_S_VERSION=$current_i_s_version @@ -304,7 +304,7 @@ INSERT INTO I_S_published_schema '06bdff30a209ee9b41bf1b654048010e0ca67399ccca594600659816fb3ee036'); INSERT INTO I_S_published_schema VALUES ('80030', '80030', 0, - 'f2f9eac78a7fe69de079bd55831bb4451236c492a76f9195e43051db44e84510'); + '55420c6267947962b2198fad2aa9b4aa9cbac51c49a046a360bf76032fe0e992'); INSERT INTO I_S_published_schema VALUES ('80030', '80030', 1, '4399948df993b3a2084c4a6dd80d76e38b2ff874e68b7e33a16b907eeb090ff0'); diff --git a/mysql-test/suite/rds/r/vidx_column.result b/mysql-test/suite/rds/r/vidx_column.result new file mode 100644 index 00000000000..d822cb1fced --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_column.result @@ -0,0 +1,188 @@ +# 1. KEYWORDs +# Word vector is not able to be the name, while words m, distance, +# euclidean, cosine is still able. +CREATE TABLE t1 ( +vector int +); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'int +)' at line 2 +CREATE TABLE t1 ( +m int, +distance int, +euclidean int, +cosine int +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `m` int DEFAULT NULL, + `distance` int DEFAULT NULL, + `euclidean` int DEFAULT NULL, + `cosine` int DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +DROP TABLE t1; + +# 2. NONSUPPORT about VECTOR COLUMN +# The variable vidx_disabled will forbid to create VECTOR column +SET GLOBAL vidx_disabled = ON; +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(16382) +); +ERROR HY000: Creating vector columns or indexes is disabled. +SET GLOBAL vidx_disabled = OFF; +# The range of the dimension of VECTOR columns is limited by row size +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(16383) +); +ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 65535. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs +# The dimension of VECTOR column must be set explicitly +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR +); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ')' at line 4 +# The dimension of VECTOR column must be set by number not string +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR("kkk") +); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '"kkk") +)' at line 3 + +# 3. SUPPORT about VECTOR COLUMN +# The range of the dimension of VECTOR columns is limited by row size +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(16382) +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(16382) */ varbinary(65528) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +DROP TABLE t1; +# The dimension of VECTOR column can be 0 +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(0) +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(0) */ varbinary(0) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +DROP TABLE t1; +# Set Decimal as the VECTOR dimension will be corrected as the integer +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(10.1) +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(10) */ varbinary(40) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +DROP TABLE t1; +# The VECTOR field can be used in routines. +CREATE PROCEDURE vector_distance(v1 VECTOR(5), v2 VECTOR(5)) +BEGIN +SELECT VEC_DISTANCE_EUCLIDEAN(v1, v2); +END$$ +CREATE FUNCTION str2vector(in_str TEXT) RETURNS VECTOR(5) +BEGIN +RETURN VEC_FROMTEXT(in_str); +END$$ +CALL vector_distance(VEC_FROMTEXT('[1,2,3,4,5]'), VEC_FROMTEXT('[1,2,3,4,5]')); +VEC_DISTANCE_EUCLIDEAN(v1, v2) +0 +SELECT VEC_TOTEXT(str2vector('[1,2,3,4,5]')); +VEC_TOTEXT(str2vector('[1,2,3,4,5]')) +[1,2,3,4,5] +DROP PROCEDURE vector_distance; +DROP FUNCTION str2vector; + +# 4. DML about VECTOR COLUMN +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5) +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +SET transaction_isolation = 'READ-COMMITTED'; +# NULL is allowed if column is nullable +INSERT INTO t1(v) VALUES (NULL); +# other types except string is not allowed +INSERT INTO t1(v) VALUES (1); +ERROR HY000: Value of type 'longlong, size: 8' cannot be converted to 'vector(5)' type. +INSERT INTO t1(v) VALUES (1.1); +# Vector with different dimension is not allowed, including empty vector +INSERT INTO t1(v) VALUES (""); +ERROR HY000: Incorrect vector value: '' for column 'v' at row 1 +INSERT INTO t1(v) VALUES (x'e360d63ebe554f3fcdbc523f4522193f523608'); +ERROR HY000: Value of type 'string, size: 19' cannot be converted to 'vector(5)' type. +INSERT INTO t1(v) VALUES (x'e360d63ebe554f3fcdbc523f4522193f5236083d3d3d'); +ERROR HY000: Value of type 'string, size: 22' cannot be converted to 'vector(5)' type. +# string is allowed, but warning if charset is not binary and sql_mode=strict +SET sql_mode="STRICT_TRANS_TABLES"; +Warnings: +Warning 3135 'NO_ZERO_DATE', 'NO_ZERO_IN_DATE' and 'ERROR_FOR_DIVISION_BY_ZERO' sql modes should be used with strict mode. They will be merged with strict mode in a future release. +INSERT INTO t1(v) VALUES ("===================="); +ERROR HY000: Incorrect vector value: '====================' for column 'v' at row 1 +SET sql_mode=""; +INSERT INTO t1(v) VALUES ("===================="); +Warnings: +Warning 1366 Incorrect vector value: '====================' for column 'v' at row 1 +# HEX string is allowed +INSERT INTO t1(v) VALUES (x'e360d63ebe554f3fcdbc523f4522193f5236083d'); +SELECT VEC_TOTEXT(v) FROM t1; +VEC_TOTEXT(v) +NULL +[0.046201,0.046201,0.046201,0.046201,0.046201] +[0.418708,0.809902,0.823193,0.598179,0.0332549] +# Vector column can be the part of normal index +CREATE INDEX i1 ON t1(v, id); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `i1` (`v`,`id`) +) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +SELECT VEC_TOTEXT(v) FROM t1 FORCE INDEX(i1); +VEC_TOTEXT(v) +NULL +[0.046201,0.046201,0.046201,0.046201,0.046201] +[0.418708,0.809902,0.823193,0.598179,0.0332549] +# Vector column can build a prefix index. Number means the prefix +# dimension rather than the prefix length. +CREATE INDEX i2 ON t1(v(2)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `i1` (`v`,`id`), + KEY `i2` (`v`(2)) +) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +SELECT VEC_TOTEXT(v) FROM t1 FORCE INDEX(i2); +VEC_TOTEXT(v) +NULL +[0.046201,0.046201,0.046201,0.046201,0.046201] +[0.418708,0.809902,0.823193,0.598179,0.0332549] +DROP TABLE t1; +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/r/vidx_ddl.result b/mysql-test/suite/rds/r/vidx_ddl.result new file mode 100644 index 00000000000..5d5efe71a1c --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_ddl.result @@ -0,0 +1,310 @@ +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET transaction_isolation = 'READ-COMMITTED'; +SET GLOBAL vidx_disabled = OFF; +# 1. NONSUPPORT about VECTOR INDEX +# The variable vidx_disabled will forbid to create VECTOR index +SET GLOBAL vidx_disabled = ON; +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) +); +ERROR HY000: Creating vector columns or indexes is disabled. +SET GLOBAL vidx_disabled = OFF; +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5) +); +SET GLOBAL vidx_disabled = ON; +ALTER TABLE t1 ADD VECTOR INDEX vi(v); +ERROR HY000: Creating vector columns or indexes is disabled. +DROP TABLE t1; +SET GLOBAL vidx_disabled = OFF; +# Vector index is not supported for partitioned or temp tables. +CREATE TEMPORARY TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) +); +ERROR 42000: This version of MySQL doesn't yet support 'the VECTOR index in partitioned or temp tables' +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +c1 INT, +v VECTOR(5), +VECTOR INDEX vi(v) +)PARTITION BY HASH (c1) PARTITIONS 3; +ERROR 42000: This version of MySQL doesn't yet support 'the VECTOR index in partitioned or temp tables' +# Vector index can only be created on one VECTOR column not null +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +t VECTOR(5), +VECTOR INDEX vi(v,t) +); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ',t) +)' at line 5 +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v INT NOT NULL, +VECTOR INDEX vi(v) +); +ERROR HY000: Incorrect usage of vector index: Only support one visible VECTOR column +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) +); +ALTER TABLE t1 MODIFY COLUMN v VECTOR(5) INVISIBLE; +ERROR HY000: Incorrect usage of vector index: Only support one visible VECTOR column +DROP TABLE t1; +# VECTOR INDEX can't be created on generate column +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v BLOB NOT NULL, +VECTOR INDEX vi(VEC_FROMTEXT(v)) +); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '(v)) +)' at line 4 +# VECTOR INDEX‘s options should be set in their range. m should be +# in [3,200]. distance should be in {euclidean, cosine} +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) m=2 +); +ERROR HY000: Incorrect usage of vector index: Invalid options. +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) m=201 +); +ERROR HY000: Incorrect usage of vector index: Invalid options. +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) distance=wrong +); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'wrong +)' at line 4 +# VECTOR keys can not be prefix +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(768), +VECTOR INDEX vi(v(1)) +); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '(1)) +)' at line 4 +# VECTOR key's dimension must be greater than 0 +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(0), +VECTOR INDEX vi(v) +); +ERROR 42000: The used storage engine can't index column 'v' +# The range of the dimension of VECTOR keys is not limited by key size +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VARCHAR(3076), +INDEX vi(v) +); +ERROR 42000: Specified key was too long; max key length is 3072 bytes +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(769), +VECTOR INDEX vi(v) +); +DROP TABLE t1; +# Can't be invisible +CREATE TABLE t10 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) INVISIBLE +); +ERROR HY000: Incorrect usage of vector index: Must be visible. +# Don't support multi VECTOR indexes yet +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v), +VECTOR INDEX vi2(v) +); +ERROR 42000: This version of MySQL doesn't yet support 'multiple VECTOR indexes' +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) +); +CREATE VECTOR INDEX vi2 ON t1(v); +ERROR 42000: This version of MySQL doesn't yet support 'multiple VECTOR indexes' +ALTER TABLE t1 ADD VECTOR INDEX vi2(v); +ERROR 42000: This version of MySQL doesn't yet support 'multiple VECTOR indexes' +DROP TABLE t1; + +# 2. Don't support inplace ddl yet +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5) +); +ALTER TABLE t1 ADD VECTOR INDEX vi(v), algorithm=instant; +ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=COPY/INPLACE. +ALTER TABLE t1 ADD VECTOR INDEX vi(v), algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY. +ALTER TABLE t1 ADD VECTOR INDEX vi(v); +ALTER TABLE t1 DROP INDEX vi, algorithm=instant; +ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=COPY/INPLACE. +ALTER TABLE t1 DROP INDEX vi, algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY. +ALTER TABLE t1 DROP INDEX vi; +DROP TABLE t1; + +# 3. DDL about VECTOR INDEX +# CREATE TABLE +CREATE DATABASE IF NOT EXISTS test2; +USE test2; +# VECTOR INDEX should be at the end of key list +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +text LONGTEXT NOT NULL, +meta JSON NOT NULL, +v VECTOR(5), +VECTOR INDEX vi(v) m=3 distance=euclidean +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `text` longtext NOT NULL, + `meta` json NOT NULL, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`)/*!99999 , + VECTOR KEY `vi` (`v`) M=3 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +ALTER TABLE t1 ADD INDEX i2(v); +CREATE INDEX idx_meta ON `t1` ((CAST(JSON_UNQUOTE(JSON_EXTRACT(meta, '$.document_id')) AS CHAR(36)))); +CREATE FULLTEXT INDEX idx_full_text ON `t1` (text); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `text` longtext NOT NULL, + `meta` json NOT NULL, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `i2` (`v`), + KEY `idx_meta` ((cast(json_unquote(json_extract(`meta`,_utf8mb4'$.document_id')) as char(36) charset utf8mb4))), + FULLTEXT KEY `idx_full_text` (`text`)/*!99999 , + VECTOR KEY `vi` (`v`) M=3 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +DROP TABLE t1; +# DROP VECTOR INDEX +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) m=3 distance=euclidean +); +INSERT INTO t1(v) SELECT VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']')); +DROP INDEX vi ON t1; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +# CREATE VECTOR INDEX +CREATE VECTOR INDEX vi ON t1(v) m=200 distance=cosine; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`)/*!99999 , + VECTOR KEY `vi` (`v`) M=200 DISTANCE=COSINE */ +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +# ALTER TABLE DROP VECTOR INDEX +ALTER TABLE t1 DROP INDEX `vi`; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +# ALTER TABLE ADD VECTOR INDEX +ALTER TABLE t1 ADD VECTOR INDEX vi(v) m=200 distance=cosine; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`)/*!99999 , + VECTOR KEY `vi` (`v`) M=200 DISTANCE=COSINE */ +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +# ALTER TABLE RENAME VECTOR INDEX +ALTER TABLE t1 RENAME INDEX `vi` TO `vi2`; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`)/*!99999 , + VECTOR KEY `vi2` (`v`) M=200 DISTANCE=COSINE */ +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +# OPTIMIZE TABLE +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test2.t1 optimize note Table does not support optimize, doing recreate + analyze instead +test2.t1 optimize status OK +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`)/*!99999 , + VECTOR KEY `vi2` (`v`) M=200 DISTANCE=COSINE */ +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +# TRUNCATE TABLE +TRUNCATE TABLE t1; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`)/*!99999 , + VECTOR KEY `vi2` (`v`) M=200 DISTANCE=COSINE */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +# RENAME TABLE +RENAME TABLE t1 TO t2; +SHOW CREATE TABLE t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`)/*!99999 , + VECTOR KEY `vi2` (`v`) M=200 DISTANCE=COSINE */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +CREATE TABLE t1 LIKE t2; +# DROP TABLE +DROP TABLE t2; +# DROP DATABASE +DROP DATABASE test2; +USE test; + +# 4. Vars about VECTOR INDEX +SET SESSION vidx_default_distance = 'cosine'; +SET SESSION vidx_hnsw_default_m = 155; +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`)/*!99999 , + VECTOR KEY `vi` (`v`) M=155 DISTANCE=COSINE */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +DROP TABLE t1; +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/r/vidx_debug.result b/mysql-test/suite/rds/r/vidx_debug.result new file mode 100644 index 00000000000..4060f6716eb --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_debug.result @@ -0,0 +1,218 @@ +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +CALL mtr.add_suppression("\\[ERROR\\].*Scanned file '.*' for tablespace .* cannot be opened because it is not in a sub-directory named for the schema"); +CALL mtr.add_suppression("\\[Warning\\] .*MY-\\d+.* Tablespace .*, name '.*', file '*.*' is missing!"); + +# 1. Failed or crash during CREATE +# 1.1 whole DDL is failed because DDL of vidx table is failed +SET DEBUG = "d,failed_before_vidx_ddl"; +CREATE TABLE t1 (v VECTOR(5), VECTOR INDEX vi(v)); +ERROR HY000: Create vector index `vi` in `test`.`t1` (aux_tab: vidx_000000000000042d_00) failed: debug failed before vidx ddl. +# expect that both base tablespace and vidx tablespace are not created +SHOW CREATE TABLE t1; +ERROR 42S02: Table 'test.t1' doesn't exist +# 1.2 crash before create vidx table +SET DEBUG = "d,crash_before_vidx_ddl"; +CREATE TABLE t1 (v VECTOR(5), VECTOR INDEX vi(v)); +# expect that base tablespace has been created but vidx tablespace is not created +t1.ibd +# expect that creation is rollback after crash recovery +SHOW CREATE TABLE t1; +ERROR 42S02: Table 'test.t1' doesn't exist +# 1.3 crash before post ddl +SET DEBUG = "d,ddl_log_before_post_ddl"; +CREATE TABLE t1 (v VECTOR(5), VECTOR INDEX vi(v)); +# expect that both base tablespace and vidx tablespace are created +t1.ibd +vidx_#_00.ibd +# expect that creation is committed after crash recovery +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +t1.ibd +vidx_#_00.ibd +SET DEBUG = '+d,skip_dd_table_access_check'; +SELECT count(*) AS `Expected as 0` FROM mysql.innodb_ddl_log; +Expected as 0 +0 + +# 2. Failed during DML +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,5]")); +SET DEBUG = "d,failed_before_vidx_dml"; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,6]")); +ERROR HY000: Incorrect usage of vector index: debug failed before vidx dml. +SELECT VEC_TOTEXT(v) FROM t1; +VEC_TOTEXT(v) +[1,2,3,4,5] +UPDATE t1 SET v = VEC_FROMTEXT("[1,2,3,4,6]"); +ERROR HY000: Incorrect usage of vector index: debug failed before vidx dml. +SELECT VEC_TOTEXT(v) FROM t1; +VEC_TOTEXT(v) +[1,2,3,4,5] +DELETE FROM t1; +ERROR HY000: Incorrect usage of vector index: debug failed before vidx dml. +SELECT VEC_TOTEXT(v) FROM t1; +VEC_TOTEXT(v) +[1,2,3,4,5] + +# 3. Failed or crash during RENAME +# 3.1 whole DDL is failed because DDL of vidx table is failed +CREATE DATABASE test2; +SET DEBUG = "d,failed_before_vidx_ddl"; +RENAME TABLE t1 TO test2.t2; +ERROR HY000: Rename vector index `vi` in `test2`.`t2` (aux_tab: vidx_000000000000042f_00) failed: debug failed before vidx ddl. +# expect that both base tablespace and vidx tablespace are not renamed +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +SHOW CREATE TABLE test2.t2; +ERROR 42S02: Table 'test2.t2' doesn't exist +files in test/ +t1.ibd +vidx_#_00.ibd +files in test2/ +# 3.2 crash before rename vidx table +SET DEBUG = "d,crash_before_vidx_ddl"; +RENAME TABLE t1 TO test2.t2; +# expect that base tablespace has been renamed but vidx tablespace is not renamed +files in test/ +vidx_#_00.ibd +files in test2/ +t2.ibd +# expect that renaming is rollback after crash recovery +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +SHOW CREATE TABLE test2.t2; +ERROR 42S02: Table 'test2.t2' doesn't exist +files in test/ +t1.ibd +vidx_#_00.ibd +files in test2/ +# 3.3 crash before post ddl +SET DEBUG = "d,ddl_log_before_post_ddl"; +RENAME TABLE t1 TO test2.t2; +# expect that both base tablespace and vidx tablespace are not renamed +files in test/ +files in test2/ +t2.ibd +vidx_#_00.ibd +# expect that rename is committed after crash recovery +SHOW CREATE TABLE t1; +ERROR 42S02: Table 'test.t1' doesn't exist +SHOW CREATE TABLE test2.t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +files in test/ +files in test2/ +t2.ibd +vidx_#_00.ibd +SET DEBUG = '+d,skip_dd_table_access_check'; +SELECT count(*) AS `Expected as 0` FROM mysql.innodb_ddl_log; +Expected as 0 +0 +RENAME TABLE test2.t2 TO t1; +DROP DATABASE test2; + +# 4. Failed or crash during TRUNCATE +# 4.1 whole DDL is failed because DDL of vidx table is failed +SET DEBUG = "d,failed_before_vidx_ddl"; +TRUNCATE TABLE t1; +ERROR HY000: Truncate vector index `vi` in `test`.`t1` (aux_tab: vidx_0000000000000431_00) failed: debug failed before vidx ddl. +# expect that both base tablespace and vidx tablespace are not truncated +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +t1.ibd +vidx_#_00.ibd +# 4.2 crash before rename vidx table +SET DEBUG = "d,crash_before_vidx_ddl"; +TRUNCATE TABLE t1; +# expect that both base tablespace and vidx tablespace are not truncated +t1.ibd +vidx_#_00.ibd +# expect that truncation is rollback after crash recovery +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +t1.ibd +vidx_#_00.ibd +# 4.3 crash before post ddl +SET DEBUG = "d,ddl_log_before_post_ddl"; +TRUNCATE TABLE t1; +# expect that both base tablespace and vidx tablespace are not truncated +t1.ibd +vidx_#_00.ibd +# expect that truncation is committed after crash recovery +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +t1.ibd +vidx_#_00.ibd +SET DEBUG = '+d,skip_dd_table_access_check'; +SELECT count(*) AS `Expected as 0` FROM mysql.innodb_ddl_log; +Expected as 0 +0 + +# 5. Failed or crash during DROP +# 5.1 whole DDL is failed because DDL of vidx table is failed +SET DEBUG = "d,failed_before_vidx_ddl"; +DROP TABLE t1; +ERROR HY000: Drop vector index `vi` in `test`.`t1` (aux_tab: vidx_0000000000000433_00) failed: debug failed before vidx ddl. +# expect that both base tablespace and vidx tablespace are not dropped +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +t1.ibd +vidx_#_00.ibd +# 5.2 crash before rename vidx table +SET DEBUG = "d,crash_before_vidx_ddl"; +DROP TABLE t1; +# expect that both base tablespace and vidx tablespace are not dropped +t1.ibd +vidx_#_00.ibd +# expect that drop is rollback after crash recovery +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +t1.ibd +vidx_#_00.ibd +# 5.3 crash before post ddl +SET DEBUG = "d,ddl_log_before_post_ddl"; +DROP TABLE t1; +# expect that both base tablespace and vidx tablespace are not dropped +t1.ibd +vidx_#_00.ibd +# expect that drop is committed after crash recovery +SHOW CREATE TABLE t1; +ERROR 42S02: Table 'test.t1' doesn't exist +SET DEBUG = '+d,skip_dd_table_access_check'; +SELECT count(*) AS `Expected as 0` FROM mysql.innodb_ddl_log; +Expected as 0 +0 diff --git a/mysql-test/suite/rds/r/vidx_dml.result b/mysql-test/suite/rds/r/vidx_dml.result new file mode 100644 index 00000000000..f53bd211827 --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_dml.result @@ -0,0 +1,115 @@ +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; +# 1. Prepare +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +a INT NOT NULL, +v VECTOR(5), +INDEX (a), +VECTOR INDEX vi(v) +) ENGINE = InnoDB; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `a` int NOT NULL, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `a` (`a`)/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci + +# 2. Insert 100 rows +INSERT INTO t1(a, v) SELECT 1, VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']')); + +# 3. Delete +DELETE FROM t1 WHERE id % 3 = 0; + +# 4. Update +UPDATE t1 SET v = (SELECT VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']'))) WHERE id % 3 = 1; + +# 5. Select +SELECT COUNT(*) FROM t1; +COUNT(*) +67 +# expect to use vector index scan +EXPLAIN SELECT * FROM t1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 16; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL index NULL vi 23 NULL 67 100.00 NULL +EXPLAIN SELECT * FROM t1 WHERE id>1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 16; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL index PRIMARY vi 23 NULL 67 100.00 Using where +EXPLAIN SELECT * FROM t1 WHERE a=1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 16; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL index a vi 23 NULL 67 100.00 Using where +EXPLAIN SELECT VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) AS distance FROM t1 ORDER BY distance limit 16; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL index NULL vi 23 NULL 67 100.00 NULL +# expect to use JT_ALL +EXPLAIN SELECT * FROM t1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 17; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL ALL NULL NULL NULL NULL 67 100.00 Using filesort +EXPLAIN SELECT * FROM t1 WHERE id>1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 17; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL ALL PRIMARY NULL NULL NULL 67 100.00 Using where; Using filesort +EXPLAIN SELECT * FROM t1 WHERE a=1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 17; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL ALL a NULL NULL NULL 67 100.00 Using where; Using filesort +EXPLAIN SELECT VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) AS distance FROM t1 ORDER BY distance limit 17; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL ALL NULL NULL NULL NULL 67 100.00 Using filesort +# expect to use SK range +EXPLAIN SELECT * FROM t1 WHERE a>1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 1; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL range a a 4 NULL 1 100.00 Using index condition; Using filesort +# FORCE INDEX +EXPLAIN SELECT * FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 16; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL ALL NULL NULL NULL NULL 67 100.00 Using filesort +EXPLAIN SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")); +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL index NULL vi 23 NULL 67 100.00 NULL +EXPLAIN SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 100; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL index NULL vi 23 NULL 67 100.00 NULL +# select using VIEW +CREATE VIEW `view1` AS SELECT * FROM t1; +SELECT id, VEC_TOTEXT(v) FROM `view1` ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 1; +SELECT id, VEC_TOTEXT(v) FROM `view1` ORDER BY VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), v) limit 1; +DROP VIEW `view1`; + +# 6. Select using prepare stmt +PREPARE stmt FROM 'SELECT id FROM t1 ORDER BY vec_distance_euclidean(v, ?) LIMIT ?'; +SET @vec = VEC_FROMTEXT("[1,2,3,4,5]"); +SET @limit = 10; +EXECUTE stmt USING @vec, @limit; +SET @vec = VEC_FROMTEXT("[1,2,3,4]"); +EXECUTE stmt USING @vec, @limit; +ERROR HY000: Incorrect arguments to VEC_DISTANCE_EUCLIDEAN + +# 7. Select using distance with different objects +# numeric +SELECT * FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, 1) limit 1; +ERROR HY000: Incorrect arguments to VEC_DISTANCE_EUCLIDEAN +SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, 1) limit 1; +ERROR HY000: Incorrect arguments to VEC_DISTANCE_EUCLIDEAN +SELECT * FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, 1.1) limit 1; +ERROR HY000: Incorrect arguments to VEC_DISTANCE_EUCLIDEAN +SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, 1.1) limit 1; +ERROR HY000: Incorrect arguments to VEC_DISTANCE_EUCLIDEAN +# string with different length +SELECT * FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, "1.1") limit 1; +ERROR HY000: Incorrect arguments to VEC_DISTANCE_EUCLIDEAN +SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, "1.1") limit 1; +ERROR HY000: Incorrect arguments to VEC_DISTANCE_EUCLIDEAN +# string with same length +SELECT * FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0") limit 1; +SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0") limit 1; + +# 8. Delete all +DELETE FROM t1; + +# 9. Clear +DROP TABLE t1; +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/r/vidx_fix_71283799.result b/mysql-test/suite/rds/r/vidx_fix_71283799.result new file mode 100644 index 00000000000..78746e9c56f --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_fix_71283799.result @@ -0,0 +1,24 @@ +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; +# Prepare a vector table in other database. +CREATE DATABASE test2; +CREATE TABLE test2.t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) +) ENGINE = InnoDB; +INSERT INTO test2.t1 SELECT 1, VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']')); +# Don't do purge before restart. +SET GLOBAL debug="d,do_not_meta_lock_in_background"; +# Delete data. +DELETE FROM test2.t1 where id = 1; +# Restart and wait until all purged. +# Before this patch, purge thread holds MDL SR locks on auxiliary tables. +SELECT COUNT(1) FROM performance_schema.metadata_locks WHERE OBJECT_SCHEMA='test2'; +COUNT(1) +0 +# Before this patch, DDL about test2.t1 will hang. +DROP TABLE test2.t1; +DROP DATABASE test2; +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/r/vidx_fix_71294192.result b/mysql-test/suite/rds/r/vidx_fix_71294192.result new file mode 100644 index 00000000000..c110f13b104 --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_fix_71294192.result @@ -0,0 +1,16 @@ +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; +# Prepare a vector table in other database. +CREATE DATABASE test2; +CREATE TABLE test2.t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) +) ENGINE = InnoDB; +INSERT INTO test2.t1(id, v) SELECT 1, VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']')); +# Restart mysqld to ensure the auxiliary table is not opened before dropped. +# Before this patch, the DROP will cause a crash. +DROP TABLE test2.t1; +DROP DATABASE test2; +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/r/vidx_fix_MDEV_37068.result b/mysql-test/suite/rds/r/vidx_fix_MDEV_37068.result new file mode 100644 index 00000000000..b5d195afc41 --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_fix_MDEV_37068.result @@ -0,0 +1,15 @@ +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; +# +# MDEV-37068 Can't find record in 't1' on INSERT to Vector table +# +SET sql_mode=''; +create table t1 (v vector (1) not null,vector vec (v),unique vu (v)) engine=innodb; +start transaction; +insert t1 values (VEC_FROMTEXT("[1]")), (VEC_FROMTEXT("[1]")); +ERROR 23000: Duplicate entry '\x00\x00\x80?' for key 't1.vu' +insert t1 values (VEC_FROMTEXT("[1]")); +drop table t1; +set sql_mode=default; +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/r/vidx_func.result b/mysql-test/suite/rds/r/vidx_func.result new file mode 100644 index 00000000000..73d15f4f25c --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_func.result @@ -0,0 +1,137 @@ +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; +# 1. SUPPORT about VECTOR functions +SELECT VECTOR_DIM(VEC_FROMTEXT("[1,2,3,4,5]")); +VECTOR_DIM(VEC_FROMTEXT("[1,2,3,4,5]")) +5 +SELECT VEC_TOTEXT(x'e360d63ebe554f3fcdbc523f4522193f5236083d'); +VEC_TOTEXT(x'e360d63ebe554f3fcdbc523f4522193f5236083d') +[0.418708,0.809902,0.823193,0.598179,0.0332549] +SELECT VEC_TOTEXT("abcd"); +VEC_TOTEXT("abcd") +[1.6778e22] +SELECT VEC_TOTEXT(VEC_FROMTEXT("[1,2,3,4,5]")); +VEC_TOTEXT(VEC_FROMTEXT("[1,2,3,4,5]")) +[1,2,3,4,5] +SELECT VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5]")); +VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5]")) +0 +SELECT VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5.1]")); +VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5.1]")) +0.09999990463256836 +SELECT VEC_DISTANCE_COSINE(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5]")); +VEC_DISTANCE_COSINE(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5]")) +0 +SELECT VEC_DISTANCE_COSINE(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5.1]")); +VEC_DISTANCE_COSINE(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5.1]")) +0.00004867880623027343 +# NULL, empty vector is allowed in VEC_TOTEXT and VECTOR_DIM +SELECT VEC_TOTEXT(x''); +VEC_TOTEXT(x'') + +SELECT VEC_TOTEXT(NULL); +VEC_TOTEXT(NULL) +NULL +SELECT VECTOR_DIM(x''); +VECTOR_DIM(x'') +0 +SELECT VECTOR_DIM(NULL); +VECTOR_DIM(NULL) +NULL +# NULL vector is allowed in VEC_FROMTEXT +SELECT VEC_FROMTEXT(NULL); +VEC_FROMTEXT(NULL) +NULL + +# 3. NONSUPPORT about VECTOR functions +# The length of arguments should be able to converted to a vector dimension +SELECT VEC_TOTEXT("abc"); +ERROR HY000: Invalid binary vector format. Must use IEEE standard float representation in little-endian format. Use VEC_FromText() to generate it. +# Invalid binary format +SELECT VEC_FROMTEXT("1,2,3,4,5]"); +ERROR HY000: Data cannot be converted to a valid vector: '1,2,3,4,5]' +SELECT VEC_FROMTEXT(""); +ERROR HY000: Data cannot be converted to a valid vector: '' +# The number of arguments is not correct +SELECT VEC_TOTEXT(); +ERROR 42000: Incorrect parameter count in the call to native function 'VEC_TOTEXT' +SELECT VECTOR_DIM(); +ERROR 42000: Incorrect parameter count in the call to native function 'VECTOR_DIM' +SELECT VECTOR_DIM(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5]")); +ERROR 42000: Incorrect parameter count in the call to native function 'VECTOR_DIM' +# The distance tween two vectors having different dimensions should be NULL +SELECT VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4]")); +VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4]")) +NULL +SELECT VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), NULL); +VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), NULL) +NULL +SELECT VEC_DISTANCE_COSINE(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5,6]")); +VEC_DISTANCE_COSINE(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5,6]")) +NULL +SELECT VEC_DISTANCE_COSINE(NULL, VEC_FROMTEXT("[1,2,3,4,5,6]")); +VEC_DISTANCE_COSINE(NULL, VEC_FROMTEXT("[1,2,3,4,5,6]")) +NULL +SELECT VEC_DISTANCE_COSINE(NULL, NULL); +VEC_DISTANCE_COSINE(NULL, NULL) +NULL + +# 4. DML using vector functions +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5) +); +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,5]")); +INSERT INTO t1(v) VALUES (TO_VECTOR("[1.1,2.1,3.1,4.1,5.1]")); +INSERT INTO t1(v) VALUES (STRING_TO_VECTOR("[1.2,2.2,3.2,4.2,5.2]")); +INSERT INTO t1(v) VALUES (x'e360d63ebe554f3fcdbc523f4522193f5236083d'); +SELECT id, VEC_TOTEXT(v) FROM t1; +id VEC_TOTEXT(v) +1 [1,2,3,4,5] +2 [1.1,2.1,3.1,4.1,5.1] +3 [1.2,2.2,3.2,4.2,5.2] +4 [0.418708,0.809902,0.823193,0.598179,0.0332549] +SELECT VECTOR_DIM(v) FROM t1; +VECTOR_DIM(v) +5 +5 +5 +5 +SELECT VEC_TOTEXT(v) FROM t1; +VEC_TOTEXT(v) +[1,2,3,4,5] +[1.1,2.1,3.1,4.1,5.1] +[1.2,2.2,3.2,4.2,5.2] +[0.418708,0.809902,0.823193,0.598179,0.0332549] +SELECT FROM_VECTOR(v) FROM t1; +FROM_VECTOR(v) +[1,2,3,4,5] +[1.1,2.1,3.1,4.1,5.1] +[1.2,2.2,3.2,4.2,5.2] +[0.418708,0.809902,0.823193,0.598179,0.0332549] +SELECT VECTOR_TO_STRING(v) FROM t1; +VECTOR_TO_STRING(v) +[1,2,3,4,5] +[1.1,2.1,3.1,4.1,5.1] +[1.2,2.2,3.2,4.2,5.2] +[0.418708,0.809902,0.823193,0.598179,0.0332549] +SELECT id, VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) FROM t1; +id VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) +1 0 +2 0.2236066378139594 +3 0.4472134888760042 +4 6.537098835577236 +SELECT id, VEC_DISTANCE_COSINE(v, VEC_FROMTEXT("[1,2,3,4,5]")) FROM t1; +id VEC_DISTANCE_COSINE(v, VEC_FROMTEXT("[1,2,3,4,5]")) +1 0 +2 0.00007830157192822984 +3 0.00029710017677209155 +4 0.3027471560934656 +SELECT id FROM t1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")); +id +1 +2 +3 +4 +DROP TABLE t1; +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/r/vidx_i_s_metadata.result b/mysql-test/suite/rds/r/vidx_i_s_metadata.result new file mode 100644 index 00000000000..e40a5c328e2 --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_i_s_metadata.result @@ -0,0 +1,86 @@ +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; +######################################################################## +# Setup +######################################################################## +CREATE DATABASE IF NOT EXISTS vidx_is_test; +USE vidx_is_test; +CREATE TABLE t_vec ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(64) +); +CREATE PROCEDURE p_vec_param(IN v1 VECTOR(64), IN v2 VECTOR(64)) +BEGIN +SELECT VEC_DISTANCE_EUCLIDEAN(v1, v2); +END$$ +CREATE FUNCTION f_vec_return(in_str TEXT) RETURNS VECTOR(64) +DETERMINISTIC +BEGIN +RETURN VEC_FROMTEXT(in_str); +END$$ +######################################################################## +# PART 1. INFORMATION_SCHEMA.COLUMNS - DATA_TYPE should be 'varbinary' +######################################################################## +SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, COLUMN_TYPE +FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA = 'vidx_is_test' AND TABLE_NAME = 't_vec' + AND COLUMN_NAME = 'v'; +TABLE_NAME t_vec +COLUMN_NAME v +DATA_TYPE varbinary +COLUMN_TYPE /*!99999 vector(64) */ varbinary(256) +######################################################################## +# PART 2. INFORMATION_SCHEMA.PARAMETERS - DATA_TYPE should be 'varbinary' +######################################################################## +# Procedure parameters +SELECT SPECIFIC_NAME, PARAMETER_NAME, DATA_TYPE, DTD_IDENTIFIER +FROM INFORMATION_SCHEMA.PARAMETERS +WHERE SPECIFIC_SCHEMA = 'vidx_is_test' AND SPECIFIC_NAME = 'p_vec_param' + ORDER BY ORDINAL_POSITION; +SPECIFIC_NAME p_vec_param +PARAMETER_NAME v1 +DATA_TYPE varbinary +DTD_IDENTIFIER /*!99999 vector(64) */ varbinary(256) +SPECIFIC_NAME p_vec_param +PARAMETER_NAME v2 +DATA_TYPE varbinary +DTD_IDENTIFIER /*!99999 vector(64) */ varbinary(256) +# Function return value + parameters +SELECT SPECIFIC_NAME, PARAMETER_NAME, DATA_TYPE, DTD_IDENTIFIER +FROM INFORMATION_SCHEMA.PARAMETERS +WHERE SPECIFIC_SCHEMA = 'vidx_is_test' AND SPECIFIC_NAME = 'f_vec_return' + ORDER BY ORDINAL_POSITION; +SPECIFIC_NAME f_vec_return +PARAMETER_NAME NULL +DATA_TYPE varbinary +DTD_IDENTIFIER /*!99999 vector(64) */ varbinary(256) +SPECIFIC_NAME f_vec_return +PARAMETER_NAME in_str +DATA_TYPE text +DTD_IDENTIFIER text +######################################################################## +# PART 3. INFORMATION_SCHEMA.ROUTINES - DATA_TYPE should be 'varbinary' +######################################################################## +# Function return type +SELECT ROUTINE_NAME, ROUTINE_TYPE, DATA_TYPE, DTD_IDENTIFIER +FROM INFORMATION_SCHEMA.ROUTINES +WHERE ROUTINE_SCHEMA = 'vidx_is_test' AND ROUTINE_NAME = 'f_vec_return'; +ROUTINE_NAME f_vec_return +ROUTINE_TYPE FUNCTION +DATA_TYPE varbinary +DTD_IDENTIFIER /*!99999 vector(64) */ varbinary(256) +# Procedure has empty DATA_TYPE (standard behavior) +SELECT ROUTINE_NAME, ROUTINE_TYPE, DATA_TYPE +FROM INFORMATION_SCHEMA.ROUTINES +WHERE ROUTINE_SCHEMA = 'vidx_is_test' AND ROUTINE_NAME = 'p_vec_param'; +ROUTINE_NAME p_vec_param +ROUTINE_TYPE PROCEDURE +DATA_TYPE +######################################################################## +# Cleanup +######################################################################## +DROP TABLE t_vec; +DROP PROCEDURE p_vec_param; +DROP FUNCTION f_vec_return; +DROP DATABASE vidx_is_test; +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/r/vidx_rpl.result b/mysql-test/suite/rds/r/vidx_rpl.result new file mode 100644 index 00000000000..f98c780a551 --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_rpl.result @@ -0,0 +1,160 @@ +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +include/master-slave.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. +[connection master] +[connection slave] +SET GLOBAL vidx_disabled = OFF; +[connection master] +SET GLOBAL vidx_disabled = OFF; + +# 1. Test ddl log event +# a. Show query binlog records vector column as /*!99999 vector(X) */ varbinary(4*X) +# vector index as index not existing +# b. Check operation of vector index can't be in one query with other operations +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) +) ENGINE = InnoDB; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Query # # use `test`; CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`)/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB +ALTER TABLE t1 ADD COLUMN `v2` vector(10); +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN `v2` /*!99999 vector(10) */ varbinary(40) +ALTER TABLE t1 RENAME INDEX `vi` TO `vi2`, DROP COLUMN `v2`; +ERROR 42000: This version of MySQL doesn't yet support 'perform other operations while alter a vector index' +ALTER TABLE t1 RENAME INDEX `vi` TO `vi2`; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Query # # use `test`; /*!99999 ALTER TABLE t1 RENAME INDEX `vi` TO `vi2` */ +ALTER TABLE t1 DROP INDEX `vi2`, DROP COLUMN `v2`; +ERROR 42000: This version of MySQL doesn't yet support 'perform other operations while alter a vector index' +ALTER TABLE t1 DROP INDEX `vi2`; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Query # # use `test`; /*!99999 ALTER TABLE t1 DROP INDEX `vi2` */ +CREATE VECTOR INDEX `vi` ON t1(v); +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Query # # use `test`; /*!99999 CREATE VECTOR INDEX `vi` ON t1(v) */ +DROP INDEX `vi` ON t1; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Query # # use `test`; /*!99999 DROP INDEX `vi` ON t1 */ +include/sync_slave_sql_with_master.inc +[connection slave] +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + `v2` /*!99999 vector(10) */ varbinary(40) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci + +# 2. Test dml in transactions +# Prepare +[connection master] +SET transaction_isolation = 'READ-COMMITTED'; +DROP TABLE IF EXISTS t1; +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) +) ENGINE = InnoDB; +# Insert data using transactions +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,10]")); +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,9]")); +BEGIN; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,8]")); +SAVEPOINT sp1; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,7]")); +SAVEPOINT sp2; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,6]")); +ROLLBACK TO SAVEPOINT sp2; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,5]")); +COMMIT; +BEGIN; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,4]")); +ROLLBACK; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,3]")); +include/rpl_sync.inc + +# Validate data in mater +SELECT id, VEC_TOTEXT(v) FROM t1; +id VEC_TOTEXT(v) +1 [1,2,3,4,10] +2 [1,2,3,4,9] +3 [1,2,3,4,8] +4 [1,2,3,4,7] +6 [1,2,3,4,5] +8 [1,2,3,4,3] +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL ALL NULL NULL NULL NULL 8 100.00 Using filesort +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id VEC_TOTEXT(v) +6 [1,2,3,4,5] +4 [1,2,3,4,7] +8 [1,2,3,4,3] +3 [1,2,3,4,8] +2 [1,2,3,4,9] +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL index NULL vi 23 NULL 8 100.00 NULL +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id VEC_TOTEXT(v) +6 [1,2,3,4,5] +4 [1,2,3,4,7] +8 [1,2,3,4,3] +3 [1,2,3,4,8] +2 [1,2,3,4,9] + +# Validate data in slave +[connection slave] +SET transaction_isolation = 'READ-COMMITTED'; +SELECT id, VEC_TOTEXT(v) FROM t1; +id VEC_TOTEXT(v) +1 [1,2,3,4,10] +2 [1,2,3,4,9] +3 [1,2,3,4,8] +4 [1,2,3,4,7] +6 [1,2,3,4,5] +8 [1,2,3,4,3] +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL ALL NULL NULL NULL NULL 6 100.00 Using filesort +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id VEC_TOTEXT(v) +6 [1,2,3,4,5] +4 [1,2,3,4,7] +8 [1,2,3,4,3] +3 [1,2,3,4,8] +2 [1,2,3,4,9] +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL index NULL vi 23 NULL 6 100.00 NULL +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id VEC_TOTEXT(v) +6 [1,2,3,4,5] +4 [1,2,3,4,7] +8 [1,2,3,4,3] +3 [1,2,3,4,8] +2 [1,2,3,4,9] + +# 3. Clear +SET GLOBAL vidx_disabled = ON; +[connection master] +DROP TABLE t1; +SET GLOBAL vidx_disabled = ON; +include/rpl_sync.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rds/r/vidx_support_nullable.result b/mysql-test/suite/rds/r/vidx_support_nullable.result new file mode 100644 index 00000000000..b347bcfdf8a --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_support_nullable.result @@ -0,0 +1,43 @@ +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +a INT, +v VECTOR(5), +VECTOR INDEX vi(v) +) ENGINE = InnoDB; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int NOT NULL AUTO_INCREMENT, + `a` int DEFAULT NULL, + `v` /*!99999 vector(5) */ varbinary(20) DEFAULT NULL, + PRIMARY KEY (`id`)/*!99999 , + VECTOR KEY `vi` (`v`) M=6 DISTANCE=EUCLIDEAN */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +INSERT INTO t1 VALUES(1, 1, NULL); +INSERT INTO t1 VALUES(2, 2, VEC_FROMTEXT("[1,2,3,4,5]")); +INSERT INTO t1 VALUES(3, 3, NULL); +INSERT INTO t1 VALUES(4, 4, NULL); +DELETE FROM t1 WHERE id = 1; +UPDATE t1 SET v = NULL WHERE id = 2; +UPDATE t1 SET v = VEC_FROMTEXT("[1,2,3,4,5]") WHERE id = 3; +UPDATE t1 SET a=5 WHERE id = 4; +SELECT id, a, VEC_TOTEXT(v) FROM t1; +id a VEC_TOTEXT(v) +2 2 NULL +3 3 [1,2,3,4,5] +4 5 NULL +# Rows with NULL vector columns cannot be found using the vector index. +SELECT id, a, VEC_TOTEXT(v), VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) as dis FROM t1 FORCE INDEX(`vi`) ORDER BY dis; +id a VEC_TOTEXT(v) dis +3 3 [1,2,3,4,5] 0 +# Rows with NULL distances to target will be placed at the end of the result set. +SELECT id, a, VEC_TOTEXT(v), VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) as dis FROM t1 FORCE INDEX(PRIMARY) ORDER BY dis; +id a VEC_TOTEXT(v) dis +3 3 [1,2,3,4,5] 0 +2 2 NULL NULL +4 5 NULL NULL +DROP TABLE t1; +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/r/vidx_trx.result b/mysql-test/suite/rds/r/vidx_trx.result new file mode 100644 index 00000000000..7b17ef2546e --- /dev/null +++ b/mysql-test/suite/rds/r/vidx_trx.result @@ -0,0 +1,60 @@ +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; +# 1. Prepare +CREATE TABLE t1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +v VECTOR(5), +VECTOR INDEX vi(v) +) ENGINE = InnoDB; + +# 2. Insert data using transactions +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,10]")); +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,9]")); +BEGIN; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,8]")); +SAVEPOINT sp1; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,7]")); +SAVEPOINT sp2; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,6]")); +ROLLBACK TO SAVEPOINT sp2; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,5]")); +COMMIT; +BEGIN; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,4]")); +ROLLBACK; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,3]")); + +# 3. Validate data +SELECT id, VEC_TOTEXT(v) FROM t1; +id VEC_TOTEXT(v) +1 [1,2,3,4,10] +2 [1,2,3,4,9] +3 [1,2,3,4,8] +4 [1,2,3,4,7] +6 [1,2,3,4,5] +8 [1,2,3,4,3] +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL ALL NULL NULL NULL NULL 8 100.00 Using filesort +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id VEC_TOTEXT(v) +6 [1,2,3,4,5] +4 [1,2,3,4,7] +8 [1,2,3,4,3] +3 [1,2,3,4,8] +2 [1,2,3,4,9] +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id select_type table partitions type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 NULL index NULL vi 23 NULL 8 100.00 NULL +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +id VEC_TOTEXT(v) +6 [1,2,3,4,5] +4 [1,2,3,4,7] +8 [1,2,3,4,3] +3 [1,2,3,4,8] +2 [1,2,3,4,9] + +# 4. Clear +DROP TABLE t1; +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/t/vidx_column.test b/mysql-test/suite/rds/t/vidx_column.test new file mode 100644 index 00000000000..1e10478e342 --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_column.test @@ -0,0 +1,157 @@ +########################################### +# The syntax and rules for VECTOR COLUMN # +########################################### +--source include/have_binlog_format_mixed_or_row.inc + +--echo # 1. KEYWORDs +--echo # Word vector is not able to be the name, while words m, distance, +--echo # euclidean, cosine is still able. +--error ER_PARSE_ERROR +CREATE TABLE t1 ( + vector int + ); + +CREATE TABLE t1 ( + m int, + distance int, + euclidean int, + cosine int + ); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +--echo +--echo # 2. NONSUPPORT about VECTOR COLUMN +--echo # The variable vidx_disabled will forbid to create VECTOR column +SET GLOBAL vidx_disabled = ON; + +--error ER_VECTOR_DISABLED +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(16382) + ); + +SET GLOBAL vidx_disabled = OFF; + +--echo # The range of the dimension of VECTOR columns is limited by row size +--error ER_TOO_BIG_ROWSIZE +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(16383) + ); + +--echo # The dimension of VECTOR column must be set explicitly +--error ER_PARSE_ERROR +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR + ); + +--echo # The dimension of VECTOR column must be set by number not string +--error ER_PARSE_ERROR +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR("kkk") + ); + +--echo +--echo # 3. SUPPORT about VECTOR COLUMN +--echo # The range of the dimension of VECTOR columns is limited by row size +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(16382) + ); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +--echo # The dimension of VECTOR column can be 0 +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(0) + ); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +--echo # Set Decimal as the VECTOR dimension will be corrected as the integer +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(10.1) + ); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +--echo # The VECTOR field can be used in routines. +DELIMITER $$; +CREATE PROCEDURE vector_distance(v1 VECTOR(5), v2 VECTOR(5)) +BEGIN + SELECT VEC_DISTANCE_EUCLIDEAN(v1, v2); +END$$ +CREATE FUNCTION str2vector(in_str TEXT) RETURNS VECTOR(5) +BEGIN + RETURN VEC_FROMTEXT(in_str); +END$$ +DELIMITER ;$$ + +CALL vector_distance(VEC_FROMTEXT('[1,2,3,4,5]'), VEC_FROMTEXT('[1,2,3,4,5]')); +SELECT VEC_TOTEXT(str2vector('[1,2,3,4,5]')); + +DROP PROCEDURE vector_distance; +DROP FUNCTION str2vector; + +--echo +--echo # 4. DML about VECTOR COLUMN +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5) + ); +SHOW CREATE TABLE t1; + +SET transaction_isolation = 'READ-COMMITTED'; + +--echo # NULL is allowed if column is nullable +INSERT INTO t1(v) VALUES (NULL); + +--echo # other types except string is not allowed +--error ER_DATA_INCOMPATIBLE_WITH_VECTOR +INSERT INTO t1(v) VALUES (1); +# the field size is different between the debug mode and the release mode. +--disable_result_log +--error ER_DATA_INCOMPATIBLE_WITH_VECTOR +INSERT INTO t1(v) VALUES (1.1); +--enable_result_log + +--echo # Vector with different dimension is not allowed, including empty vector +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +INSERT INTO t1(v) VALUES (""); +--error ER_DATA_INCOMPATIBLE_WITH_VECTOR +INSERT INTO t1(v) VALUES (x'e360d63ebe554f3fcdbc523f4522193f523608'); +--error ER_DATA_INCOMPATIBLE_WITH_VECTOR +INSERT INTO t1(v) VALUES (x'e360d63ebe554f3fcdbc523f4522193f5236083d3d3d'); + +--echo # string is allowed, but warning if charset is not binary and sql_mode=strict +SET sql_mode="STRICT_TRANS_TABLES"; +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +INSERT INTO t1(v) VALUES ("===================="); + +SET sql_mode=""; +INSERT INTO t1(v) VALUES ("===================="); + +--echo # HEX string is allowed +INSERT INTO t1(v) VALUES (x'e360d63ebe554f3fcdbc523f4522193f5236083d'); + +SELECT VEC_TOTEXT(v) FROM t1; + +--echo # Vector column can be the part of normal index +CREATE INDEX i1 ON t1(v, id); +SHOW CREATE TABLE t1; +SELECT VEC_TOTEXT(v) FROM t1 FORCE INDEX(i1); + +--echo # Vector column can build a prefix index. Number means the prefix +--echo # dimension rather than the prefix length. +CREATE INDEX i2 ON t1(v(2)); +SHOW CREATE TABLE t1; +SELECT VEC_TOTEXT(v) FROM t1 FORCE INDEX(i2); + +DROP TABLE t1; + +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/t/vidx_ddl.test b/mysql-test/suite/rds/t/vidx_ddl.test new file mode 100644 index 00000000000..c33aadae603 --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_ddl.test @@ -0,0 +1,276 @@ +########################################## +# The DDL for VECTOR Index # +########################################## +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +--source include/have_binlog_format_mixed_or_row.inc + +SET transaction_isolation = 'READ-COMMITTED'; +SET GLOBAL vidx_disabled = OFF; +--let $MYSQLD_DATADIR= `select @@datadir` + +--echo # 1. NONSUPPORT about VECTOR INDEX +--echo # The variable vidx_disabled will forbid to create VECTOR index +SET GLOBAL vidx_disabled = ON; + +--error ER_VECTOR_DISABLED +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) + ); + +SET GLOBAL vidx_disabled = OFF; + +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5) + ); + +SET GLOBAL vidx_disabled = ON; + +--error ER_VECTOR_DISABLED +ALTER TABLE t1 ADD VECTOR INDEX vi(v); +DROP TABLE t1; + +SET GLOBAL vidx_disabled = OFF; + +--echo # Vector index is not supported for partitioned or temp tables. +--error ER_NOT_SUPPORTED_YET +CREATE TEMPORARY TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) + ); + +--error ER_NOT_SUPPORTED_YET +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + c1 INT, + v VECTOR(5), + VECTOR INDEX vi(v) + )PARTITION BY HASH (c1) PARTITIONS 3; + +--echo # Vector index can only be created on one VECTOR column not null +--error ER_PARSE_ERROR +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + t VECTOR(5), + VECTOR INDEX vi(v,t) + ); + +--error ER_VECTOR_INDEX_USAGE +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v INT NOT NULL, + VECTOR INDEX vi(v) + ); + +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) + ); + +--error ER_VECTOR_INDEX_USAGE +ALTER TABLE t1 MODIFY COLUMN v VECTOR(5) INVISIBLE; + +DROP TABLE t1; + +--echo # VECTOR INDEX can't be created on generate column +--error ER_PARSE_ERROR +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v BLOB NOT NULL, + VECTOR INDEX vi(VEC_FROMTEXT(v)) + ); + +--echo # VECTOR INDEX‘s options should be set in their range. m should be +--echo # in [3,200]. distance should be in {euclidean, cosine} +--error ER_VECTOR_INDEX_USAGE +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) m=2 + ); + +--error ER_VECTOR_INDEX_USAGE +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) m=201 + ); + +--error ER_PARSE_ERROR +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) distance=wrong + ); + +--echo # VECTOR keys can not be prefix +--error ER_PARSE_ERROR +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(768), + VECTOR INDEX vi(v(1)) + ); + +--echo # VECTOR key's dimension must be greater than 0 +--error ER_WRONG_KEY_COLUMN +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(0), + VECTOR INDEX vi(v) + ); + +--echo # The range of the dimension of VECTOR keys is not limited by key size +--error ER_TOO_LONG_KEY +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VARCHAR(3076), + INDEX vi(v) + ); + +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(769), + VECTOR INDEX vi(v) + ); +DROP TABLE t1; + +--echo # Can't be invisible +--error ER_VECTOR_INDEX_USAGE +CREATE TABLE t10 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) INVISIBLE + ); + +--echo # Don't support multi VECTOR indexes yet +--error ER_NOT_SUPPORTED_YET +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v), + VECTOR INDEX vi2(v) + ); + +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) + ); +--error ER_NOT_SUPPORTED_YET +CREATE VECTOR INDEX vi2 ON t1(v); +--error ER_NOT_SUPPORTED_YET +ALTER TABLE t1 ADD VECTOR INDEX vi2(v); +DROP TABLE t1; + +--echo +--echo # 2. Don't support inplace ddl yet +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5) + ); +--error ER_ALTER_OPERATION_NOT_SUPPORTED +ALTER TABLE t1 ADD VECTOR INDEX vi(v), algorithm=instant; +--error ER_ALTER_OPERATION_NOT_SUPPORTED +ALTER TABLE t1 ADD VECTOR INDEX vi(v), algorithm=inplace; + +ALTER TABLE t1 ADD VECTOR INDEX vi(v); + +--error ER_ALTER_OPERATION_NOT_SUPPORTED +ALTER TABLE t1 DROP INDEX vi, algorithm=instant; +--error ER_ALTER_OPERATION_NOT_SUPPORTED +ALTER TABLE t1 DROP INDEX vi, algorithm=inplace; + +ALTER TABLE t1 DROP INDEX vi; +DROP TABLE t1; + +--echo +--echo # 3. DDL about VECTOR INDEX +--echo # CREATE TABLE +CREATE DATABASE IF NOT EXISTS test2; +USE test2; + +--echo # VECTOR INDEX should be at the end of key list +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + text LONGTEXT NOT NULL, + meta JSON NOT NULL, + v VECTOR(5), + VECTOR INDEX vi(v) m=3 distance=euclidean + ); +SHOW CREATE TABLE t1; + +ALTER TABLE t1 ADD INDEX i2(v); +CREATE INDEX idx_meta ON `t1` ((CAST(JSON_UNQUOTE(JSON_EXTRACT(meta, '$.document_id')) AS CHAR(36)))); +CREATE FULLTEXT INDEX idx_full_text ON `t1` (text); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +--echo # DROP VECTOR INDEX +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) m=3 distance=euclidean + ); + +--disable_warnings +INSERT INTO t1(v) SELECT VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']')); +--enable_warnings + +DROP INDEX vi ON t1; +SHOW CREATE TABLE t1; + +--echo # CREATE VECTOR INDEX +CREATE VECTOR INDEX vi ON t1(v) m=200 distance=cosine; +SHOW CREATE TABLE t1; + +--echo # ALTER TABLE DROP VECTOR INDEX +ALTER TABLE t1 DROP INDEX `vi`; +SHOW CREATE TABLE t1; + +--echo # ALTER TABLE ADD VECTOR INDEX +ALTER TABLE t1 ADD VECTOR INDEX vi(v) m=200 distance=cosine; +SHOW CREATE TABLE t1; + +--echo # ALTER TABLE RENAME VECTOR INDEX +ALTER TABLE t1 RENAME INDEX `vi` TO `vi2`; +SHOW CREATE TABLE t1; + +--echo # OPTIMIZE TABLE +OPTIMIZE TABLE t1; +SHOW CREATE TABLE t1; + +--echo # TRUNCATE TABLE +TRUNCATE TABLE t1; +SHOW CREATE TABLE t1; + +--echo # RENAME TABLE +RENAME TABLE t1 TO t2; +SHOW CREATE TABLE t2; +CREATE TABLE t1 LIKE t2; + +--echo # DROP TABLE +DROP TABLE t2; + +--echo # DROP DATABASE +DROP DATABASE test2; +USE test; + +--echo +--echo # 4. Vars about VECTOR INDEX +SET SESSION vidx_default_distance = 'cosine'; +SET SESSION vidx_hnsw_default_m = 155; + +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) + ); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +SET GLOBAL vidx_disabled = ON; \ No newline at end of file diff --git a/mysql-test/suite/rds/t/vidx_debug-master.opt b/mysql-test/suite/rds/t/vidx_debug-master.opt new file mode 100644 index 00000000000..c03d1638375 --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_debug-master.opt @@ -0,0 +1,2 @@ +--vidx_disabled=OFF +--transaction_isolation=READ-COMMITTED \ No newline at end of file diff --git a/mysql-test/suite/rds/t/vidx_debug.test b/mysql-test/suite/rds/t/vidx_debug.test new file mode 100644 index 00000000000..c093dec3337 --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_debug.test @@ -0,0 +1,272 @@ +########################################## +# The error situation for VECTOR Index # +########################################## +--source include/have_debug.inc +--source include/have_binlog_format_mixed_or_row.inc + +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +CALL mtr.add_suppression("\\[ERROR\\].*Scanned file '.*' for tablespace .* cannot be opened because it is not in a sub-directory named for the schema"); +CALL mtr.add_suppression("\\[Warning\\] .*MY-\\d+.* Tablespace .*, name '.*', file '*.*' is missing!"); + +--let $MYSQLD_DATADIR= `select @@datadir` + +--echo +--echo # 1. Failed or crash during CREATE +--echo # 1.1 whole DDL is failed because DDL of vidx table is failed +SET DEBUG = "d,failed_before_vidx_ddl"; + +--error ER_VECTOR_INDEX_FAILED +CREATE TABLE t1 (v VECTOR(5), VECTOR INDEX vi(v)); + +--echo # expect that both base tablespace and vidx tablespace are not created +--error ER_NO_SUCH_TABLE +SHOW CREATE TABLE t1; +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +--echo # 1.2 crash before create vidx table +SET DEBUG = "d,crash_before_vidx_ddl"; + +--source include/expect_crash.inc +--error 0,CR_SERVER_LOST,ER_INTERNAL_ERROR +CREATE TABLE t1 (v VECTOR(5), VECTOR INDEX vi(v)); + +--echo # expect that base tablespace has been created but vidx tablespace is not created +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +--echo # expect that creation is rollback after crash recovery +--source include/start_mysqld_no_echo.inc + +--error ER_NO_SUCH_TABLE +SHOW CREATE TABLE t1; +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +--echo # 1.3 crash before post ddl +SET DEBUG = "d,ddl_log_before_post_ddl"; + +--source include/expect_crash.inc +--error 0,CR_SERVER_LOST,ER_INTERNAL_ERROR +CREATE TABLE t1 (v VECTOR(5), VECTOR INDEX vi(v)); + +--echo # expect that both base tablespace and vidx tablespace are created +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +--echo # expect that creation is committed after crash recovery +--source include/start_mysqld_no_echo.inc + +SHOW CREATE TABLE t1; +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +SET DEBUG = '+d,skip_dd_table_access_check'; +SELECT count(*) AS `Expected as 0` FROM mysql.innodb_ddl_log; + +--echo +--echo # 2. Failed during DML +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,5]")); + +SET DEBUG = "d,failed_before_vidx_dml"; +--error ER_VECTOR_INDEX_USAGE +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,6]")); + +SELECT VEC_TOTEXT(v) FROM t1; + +--error ER_VECTOR_INDEX_USAGE +UPDATE t1 SET v = VEC_FROMTEXT("[1,2,3,4,6]"); + +SELECT VEC_TOTEXT(v) FROM t1; + +--error ER_VECTOR_INDEX_USAGE +DELETE FROM t1; + +SELECT VEC_TOTEXT(v) FROM t1; + +--echo +--echo # 3. Failed or crash during RENAME +--echo # 3.1 whole DDL is failed because DDL of vidx table is failed +CREATE DATABASE test2; +SET DEBUG = "d,failed_before_vidx_ddl"; + +--error ER_VECTOR_INDEX_FAILED +RENAME TABLE t1 TO test2.t2; + +--echo # expect that both base tablespace and vidx tablespace are not renamed +SHOW CREATE TABLE t1; +--error ER_NO_SUCH_TABLE +SHOW CREATE TABLE test2.t2; +--echo files in test/ +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ +--echo files in test2/ +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test2/ + +--echo # 3.2 crash before rename vidx table +SET DEBUG = "d,crash_before_vidx_ddl"; + +--source include/expect_crash.inc +--error 0,CR_SERVER_LOST,ER_INTERNAL_ERROR +RENAME TABLE t1 TO test2.t2; + +--echo # expect that base tablespace has been renamed but vidx tablespace is not renamed +--echo files in test/ +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ +--echo files in test2/ +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test2/ + +--echo # expect that renaming is rollback after crash recovery +--source include/start_mysqld_no_echo.inc + +SHOW CREATE TABLE t1; +--error ER_NO_SUCH_TABLE +SHOW CREATE TABLE test2.t2; +--echo files in test/ +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ +--echo files in test2/ +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test2/ + +--echo # 3.3 crash before post ddl +SET DEBUG = "d,ddl_log_before_post_ddl"; + +--source include/expect_crash.inc +--error 0,CR_SERVER_LOST,ER_INTERNAL_ERROR +RENAME TABLE t1 TO test2.t2; + +--echo # expect that both base tablespace and vidx tablespace are not renamed +--echo files in test/ +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ +--echo files in test2/ +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test2/ + +--echo # expect that rename is committed after crash recovery +--source include/start_mysqld_no_echo.inc + +--error ER_NO_SUCH_TABLE +SHOW CREATE TABLE t1; +SHOW CREATE TABLE test2.t2; +--echo files in test/ +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ +--echo files in test2/ +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test2/ + +SET DEBUG = '+d,skip_dd_table_access_check'; +SELECT count(*) AS `Expected as 0` FROM mysql.innodb_ddl_log; + +RENAME TABLE test2.t2 TO t1; +DROP DATABASE test2; + +--echo +--echo # 4. Failed or crash during TRUNCATE +--echo # 4.1 whole DDL is failed because DDL of vidx table is failed +SET DEBUG = "d,failed_before_vidx_ddl"; + +--error ER_VECTOR_INDEX_FAILED +TRUNCATE TABLE t1; + +--echo # expect that both base tablespace and vidx tablespace are not truncated +SHOW CREATE TABLE t1; +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +--echo # 4.2 crash before rename vidx table +SET DEBUG = "d,crash_before_vidx_ddl"; + +--source include/expect_crash.inc +--error 0,CR_SERVER_LOST,ER_INTERNAL_ERROR +TRUNCATE TABLE t1; + +--echo # expect that both base tablespace and vidx tablespace are not truncated +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ | grep -v '^#' + +--echo # expect that truncation is rollback after crash recovery +--source include/start_mysqld_no_echo.inc + +SHOW CREATE TABLE t1; +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +--echo # 4.3 crash before post ddl +SET DEBUG = "d,ddl_log_before_post_ddl"; + +--source include/expect_crash.inc +--error 0,CR_SERVER_LOST,ER_INTERNAL_ERROR +TRUNCATE TABLE t1; + +--echo # expect that both base tablespace and vidx tablespace are not truncated +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ | grep -v '^#' + +--echo # expect that truncation is committed after crash recovery +--source include/start_mysqld_no_echo.inc + +SHOW CREATE TABLE t1; +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +SET DEBUG = '+d,skip_dd_table_access_check'; +SELECT count(*) AS `Expected as 0` FROM mysql.innodb_ddl_log; + +--echo +--echo # 5. Failed or crash during DROP +--echo # 5.1 whole DDL is failed because DDL of vidx table is failed +SET DEBUG = "d,failed_before_vidx_ddl"; + +--error ER_VECTOR_INDEX_FAILED +DROP TABLE t1; + +--echo # expect that both base tablespace and vidx tablespace are not dropped +SHOW CREATE TABLE t1; +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +--echo # 5.2 crash before rename vidx table +SET DEBUG = "d,crash_before_vidx_ddl"; + +--source include/expect_crash.inc +--error 0,CR_SERVER_LOST,ER_INTERNAL_ERROR +DROP TABLE t1; + +--echo # expect that both base tablespace and vidx tablespace are not dropped +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +--echo # expect that drop is rollback after crash recovery +--source include/start_mysqld_no_echo.inc + +SHOW CREATE TABLE t1; +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +--echo # 5.3 crash before post ddl +SET DEBUG = "d,ddl_log_before_post_ddl"; + +--source include/expect_crash.inc +--error 0,CR_SERVER_LOST,ER_INTERNAL_ERROR +DROP TABLE t1; + +--echo # expect that both base tablespace and vidx tablespace are not dropped +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +--echo # expect that drop is committed after crash recovery +--source include/start_mysqld_no_echo.inc + +--error ER_NO_SUCH_TABLE +SHOW CREATE TABLE t1; +--replace_regex /vidx_.*_00\.ibd/vidx_#_00\.ibd/ +--exec ls $MYSQLD_DATADIR/test/ + +SET DEBUG = '+d,skip_dd_table_access_check'; +SELECT count(*) AS `Expected as 0` FROM mysql.innodb_ddl_log; diff --git a/mysql-test/suite/rds/t/vidx_dml.test b/mysql-test/suite/rds/t/vidx_dml.test new file mode 100644 index 00000000000..42ddba4054f --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_dml.test @@ -0,0 +1,122 @@ +########################################## +# The DML for VECTOR INDEX # +########################################## +--source include/have_binlog_format_mixed_or_row.inc + +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; + +--echo # 1. Prepare +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + a INT NOT NULL, + v VECTOR(5), + INDEX (a), + VECTOR INDEX vi(v) + ) ENGINE = InnoDB; +SHOW CREATE TABLE t1; + +--echo +--echo # 2. Insert 100 rows +--disable_warnings +INSERT INTO t1(a, v) SELECT 1, VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']')); + +--disable_query_log +let $i=99; +while ($i) +{ + INSERT INTO t1(a, v) SELECT 1, VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']')); + dec $i; +} +--enable_query_log + +--echo +--echo # 3. Delete +DELETE FROM t1 WHERE id % 3 = 0; + +--echo +--echo # 4. Update +UPDATE t1 SET v = (SELECT VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']'))) WHERE id % 3 = 1; + +--echo +--echo # 5. Select +SELECT COUNT(*) FROM t1; + +--echo # expect to use vector index scan +EXPLAIN SELECT * FROM t1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 16; +EXPLAIN SELECT * FROM t1 WHERE id>1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 16; +EXPLAIN SELECT * FROM t1 WHERE a=1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 16; +EXPLAIN SELECT VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) AS distance FROM t1 ORDER BY distance limit 16; + +--echo # expect to use JT_ALL +EXPLAIN SELECT * FROM t1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 17; +EXPLAIN SELECT * FROM t1 WHERE id>1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 17; +EXPLAIN SELECT * FROM t1 WHERE a=1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 17; +EXPLAIN SELECT VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) AS distance FROM t1 ORDER BY distance limit 17; + +--echo # expect to use SK range +EXPLAIN SELECT * FROM t1 WHERE a>1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 1; + +--echo # FORCE INDEX +EXPLAIN SELECT * FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 16; +EXPLAIN SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")); +EXPLAIN SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 100; +--enable_warnings + +--echo # select using VIEW +# MDEV-35922 +CREATE VIEW `view1` AS SELECT * FROM t1; +--disable_result_log +SELECT id, VEC_TOTEXT(v) FROM `view1` ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) limit 1; +SELECT id, VEC_TOTEXT(v) FROM `view1` ORDER BY VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), v) limit 1; +--enable_result_log +DROP VIEW `view1`; + +--echo +--echo # 6. Select using prepare stmt +PREPARE stmt FROM 'SELECT id FROM t1 ORDER BY vec_distance_euclidean(v, ?) LIMIT ?'; +SET @vec = VEC_FROMTEXT("[1,2,3,4,5]"); +SET @limit = 10; +--disable_result_log +EXECUTE stmt USING @vec, @limit; +--enable_result_log + +SET @vec = VEC_FROMTEXT("[1,2,3,4]"); +--error ER_WRONG_ARGUMENTS +EXECUTE stmt USING @vec, @limit; + +--echo +--echo # 7. Select using distance with different objects +--echo # numeric +--error ER_WRONG_ARGUMENTS +SELECT * FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, 1) limit 1; +--error ER_WRONG_ARGUMENTS +SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, 1) limit 1; + +--error ER_WRONG_ARGUMENTS +SELECT * FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, 1.1) limit 1; +--error ER_WRONG_ARGUMENTS +SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, 1.1) limit 1; + +--echo # string with different length +--error ER_WRONG_ARGUMENTS +SELECT * FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, "1.1") limit 1; +--error ER_WRONG_ARGUMENTS +SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, "1.1") limit 1; + +--echo # string with same length +--disable_result_log +SELECT * FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0") limit 1; +SELECT * FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0") limit 1; +--enable_result_log + +--echo +--echo # 8. Delete all +DELETE FROM t1; + +--echo +--echo # 9. Clear +DROP TABLE t1; + +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/t/vidx_fix_71283799.test b/mysql-test/suite/rds/t/vidx_fix_71283799.test new file mode 100644 index 00000000000..aeb50c943d0 --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_fix_71283799.test @@ -0,0 +1,41 @@ +################################################################################ +# Bug: aone#71283799 srv_purge_thread holds SHARED_READ MDL locks on vector # +# index auxiliary tables, blocking DDL operations. # +# Fix: Release the MDL lock immediately after accessing auxiliary table # +# metadata in build_hlindex_key. # +################################################################################ +--source include/have_debug.inc +--source include/have_binlog_format_mixed_or_row.inc + +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; + +--echo # Prepare a vector table in other database. +CREATE DATABASE test2; +CREATE TABLE test2.t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) + ) ENGINE = InnoDB; + +INSERT INTO test2.t1 SELECT 1, VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']')); + +--echo # Don't do purge before restart. +SET GLOBAL debug="d,do_not_meta_lock_in_background"; + +--echo # Delete data. +DELETE FROM test2.t1 where id = 1; + +--echo # Restart and wait until all purged. +--source include/restart_mysqld_no_echo.inc +--source include/wait_innodb_all_purged.inc + +--echo # Before this patch, purge thread holds MDL SR locks on auxiliary tables. +SELECT COUNT(1) FROM performance_schema.metadata_locks WHERE OBJECT_SCHEMA='test2'; + +--echo # Before this patch, DDL about test2.t1 will hang. +DROP TABLE test2.t1; +DROP DATABASE test2; + +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/t/vidx_fix_71294192.test b/mysql-test/suite/rds/t/vidx_fix_71294192.test new file mode 100644 index 00000000000..a1828dce570 --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_fix_71294192.test @@ -0,0 +1,31 @@ +################################################################################ +# Bug: aone#71294192 Can't find auxiliary table in dd if DROP vector table # +# before its dict_table_t is opened. # +# Fix: 1. Use correct db name to find auxiliary table. # +# 2. Remove '__hlindexes__' after the auxiliary table is dropped before # +# the table is dropped. # +################################################################################ +--source include/have_binlog_format_mixed_or_row.inc + +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; + +--echo # Prepare a vector table in other database. +CREATE DATABASE test2; +CREATE TABLE test2.t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) + ) ENGINE = InnoDB; + +INSERT INTO test2.t1(id, v) SELECT 1, VEC_FROMTEXT(CONCAT('[', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ',', RAND(), ']')); + +--echo # Restart mysqld to ensure the auxiliary table is not opened before dropped. +--source include/restart_mysqld_no_echo.inc + +--echo # Before this patch, the DROP will cause a crash. +DROP TABLE test2.t1; +DROP DATABASE test2; + +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/t/vidx_fix_MDEV_37068.test b/mysql-test/suite/rds/t/vidx_fix_MDEV_37068.test new file mode 100644 index 00000000000..cd9cd3aa11d --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_fix_MDEV_37068.test @@ -0,0 +1,19 @@ +--source include/have_binlog_format_mixed_or_row.inc + +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; + +--echo # +--echo # MDEV-37068 Can't find record in 't1' on INSERT to Vector table +--echo # +SET sql_mode=''; +create table t1 (v vector (1) not null,vector vec (v),unique vu (v)) engine=innodb; +start transaction; +--error ER_DUP_ENTRY +insert t1 values (VEC_FROMTEXT("[1]")), (VEC_FROMTEXT("[1]")); +insert t1 values (VEC_FROMTEXT("[1]")); +drop table t1; +set sql_mode=default; + +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/t/vidx_func.test b/mysql-test/suite/rds/t/vidx_func.test new file mode 100644 index 00000000000..dc1536da5e9 --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_func.test @@ -0,0 +1,82 @@ +############################################# +# The syntax and rules for VECTOR FUNCTION # +############################################# +--source include/have_binlog_format_mixed_or_row.inc + +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; + +--echo # 1. SUPPORT about VECTOR functions +SELECT VECTOR_DIM(VEC_FROMTEXT("[1,2,3,4,5]")); +SELECT VEC_TOTEXT(x'e360d63ebe554f3fcdbc523f4522193f5236083d'); +SELECT VEC_TOTEXT("abcd"); +SELECT VEC_TOTEXT(VEC_FROMTEXT("[1,2,3,4,5]")); +SELECT VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5]")); +SELECT VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5.1]")); +SELECT VEC_DISTANCE_COSINE(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5]")); +SELECT VEC_DISTANCE_COSINE(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5.1]")); + +--echo # NULL, empty vector is allowed in VEC_TOTEXT and VECTOR_DIM +SELECT VEC_TOTEXT(x''); +SELECT VEC_TOTEXT(NULL); + +SELECT VECTOR_DIM(x''); +SELECT VECTOR_DIM(NULL); + +--echo # NULL vector is allowed in VEC_FROMTEXT +SELECT VEC_FROMTEXT(NULL); + +--echo +--echo # 3. NONSUPPORT about VECTOR functions +--echo # The length of arguments should be able to converted to a vector dimension +--error ER_VECTOR_BINARY_FORMAT_INVALID +SELECT VEC_TOTEXT("abc"); + +--echo # Invalid binary format +--error ER_TO_VECTOR_CONVERSION +SELECT VEC_FROMTEXT("1,2,3,4,5]"); +--error ER_TO_VECTOR_CONVERSION +SELECT VEC_FROMTEXT(""); + +--echo # The number of arguments is not correct +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT VEC_TOTEXT(); + +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT VECTOR_DIM(); +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT VECTOR_DIM(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5]")); + +--echo # The distance tween two vectors having different dimensions should be NULL +SELECT VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4]")); +SELECT VEC_DISTANCE_EUCLIDEAN(VEC_FROMTEXT("[1,2,3,4,5]"), NULL); +SELECT VEC_DISTANCE_COSINE(VEC_FROMTEXT("[1,2,3,4,5]"), VEC_FROMTEXT("[1,2,3,4,5,6]")); +SELECT VEC_DISTANCE_COSINE(NULL, VEC_FROMTEXT("[1,2,3,4,5,6]")); +SELECT VEC_DISTANCE_COSINE(NULL, NULL); + +--echo +--echo # 4. DML using vector functions +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5) + ); + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,5]")); +INSERT INTO t1(v) VALUES (TO_VECTOR("[1.1,2.1,3.1,4.1,5.1]")); +INSERT INTO t1(v) VALUES (STRING_TO_VECTOR("[1.2,2.2,3.2,4.2,5.2]")); +INSERT INTO t1(v) VALUES (x'e360d63ebe554f3fcdbc523f4522193f5236083d'); + +SELECT id, VEC_TOTEXT(v) FROM t1; +SELECT VECTOR_DIM(v) FROM t1; +SELECT VEC_TOTEXT(v) FROM t1; +SELECT FROM_VECTOR(v) FROM t1; +SELECT VECTOR_TO_STRING(v) FROM t1; + +SELECT id, VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) FROM t1; +SELECT id, VEC_DISTANCE_COSINE(v, VEC_FROMTEXT("[1,2,3,4,5]")) FROM t1; + +SELECT id FROM t1 ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")); + +DROP TABLE t1; + +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/t/vidx_i_s_metadata.test b/mysql-test/suite/rds/t/vidx_i_s_metadata.test new file mode 100644 index 00000000000..8788c5ec7fd --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_i_s_metadata.test @@ -0,0 +1,97 @@ +############################################################## +# Test VECTOR type metadata in INFORMATION_SCHEMA +# +# Background: +# VECTOR columns are rewritten to versioned-comment form +# (/*!99999 vector(N) */ varbinary(4N)) in the data dictionary +# so that downstream applications unaware of the VECTOR type +# only see a plain varbinary column. +# +# Verifies DATA_TYPE is 'varbinary' (the underlying storage +# type) across COLUMNS, PARAMETERS, and ROUTINES views. +############################################################## +--source include/have_binlog_format_mixed_or_row.inc + +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; + +--echo ######################################################################## +--echo # Setup +--echo ######################################################################## + +CREATE DATABASE IF NOT EXISTS vidx_is_test; +USE vidx_is_test; + +CREATE TABLE t_vec ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(64) +); + +DELIMITER $$; + +CREATE PROCEDURE p_vec_param(IN v1 VECTOR(64), IN v2 VECTOR(64)) +BEGIN + SELECT VEC_DISTANCE_EUCLIDEAN(v1, v2); +END$$ + +CREATE FUNCTION f_vec_return(in_str TEXT) RETURNS VECTOR(64) + DETERMINISTIC +BEGIN + RETURN VEC_FROMTEXT(in_str); +END$$ + +DELIMITER ;$$ + +--vertical_results + +--echo ######################################################################## +--echo # PART 1. INFORMATION_SCHEMA.COLUMNS - DATA_TYPE should be 'varbinary' +--echo ######################################################################## + +SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, COLUMN_TYPE + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = 'vidx_is_test' AND TABLE_NAME = 't_vec' + AND COLUMN_NAME = 'v'; + +--echo ######################################################################## +--echo # PART 2. INFORMATION_SCHEMA.PARAMETERS - DATA_TYPE should be 'varbinary' +--echo ######################################################################## + +--echo # Procedure parameters +SELECT SPECIFIC_NAME, PARAMETER_NAME, DATA_TYPE, DTD_IDENTIFIER + FROM INFORMATION_SCHEMA.PARAMETERS + WHERE SPECIFIC_SCHEMA = 'vidx_is_test' AND SPECIFIC_NAME = 'p_vec_param' + ORDER BY ORDINAL_POSITION; + +--echo # Function return value + parameters +SELECT SPECIFIC_NAME, PARAMETER_NAME, DATA_TYPE, DTD_IDENTIFIER + FROM INFORMATION_SCHEMA.PARAMETERS + WHERE SPECIFIC_SCHEMA = 'vidx_is_test' AND SPECIFIC_NAME = 'f_vec_return' + ORDER BY ORDINAL_POSITION; + +--echo ######################################################################## +--echo # PART 3. INFORMATION_SCHEMA.ROUTINES - DATA_TYPE should be 'varbinary' +--echo ######################################################################## + +--echo # Function return type +SELECT ROUTINE_NAME, ROUTINE_TYPE, DATA_TYPE, DTD_IDENTIFIER + FROM INFORMATION_SCHEMA.ROUTINES + WHERE ROUTINE_SCHEMA = 'vidx_is_test' AND ROUTINE_NAME = 'f_vec_return'; + +--echo # Procedure has empty DATA_TYPE (standard behavior) +SELECT ROUTINE_NAME, ROUTINE_TYPE, DATA_TYPE + FROM INFORMATION_SCHEMA.ROUTINES + WHERE ROUTINE_SCHEMA = 'vidx_is_test' AND ROUTINE_NAME = 'p_vec_param'; + +--horizontal_results + +--echo ######################################################################## +--echo # Cleanup +--echo ######################################################################## + +DROP TABLE t_vec; +DROP PROCEDURE p_vec_param; +DROP FUNCTION f_vec_return; +DROP DATABASE vidx_is_test; + +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/t/vidx_rpl.test b/mysql-test/suite/rds/t/vidx_rpl.test new file mode 100644 index 00000000000..1d2a63b569b --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_rpl.test @@ -0,0 +1,133 @@ +########################################## +# The transaction for VECTOR INDEX # +########################################## +--source include/have_binlog_format_mixed_or_row.inc +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); + +--source include/master-slave.inc + +--source include/rpl_connection_slave.inc +SET GLOBAL vidx_disabled = OFF; + +--source include/rpl_connection_master.inc +SET GLOBAL vidx_disabled = OFF; + +--echo +--echo # 1. Test ddl log event +--echo # a. Show query binlog records vector column as /*!99999 vector(X) */ varbinary(4*X) +--echo # vector index as index not existing +--echo # b. Check operation of vector index can't be in one query with other operations +let $binlog_start = query_get_value("SHOW MASTER STATUS", Position, 1); +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) + ) ENGINE = InnoDB; +--source include/show_binlog_events.inc + +let $binlog_start = query_get_value("SHOW MASTER STATUS", Position, 1); +ALTER TABLE t1 ADD COLUMN `v2` vector(10); +--source include/show_binlog_events.inc + +let $binlog_start = query_get_value("SHOW MASTER STATUS", Position, 1); +--error ER_NOT_SUPPORTED_YET +ALTER TABLE t1 RENAME INDEX `vi` TO `vi2`, DROP COLUMN `v2`; +ALTER TABLE t1 RENAME INDEX `vi` TO `vi2`; +--source include/show_binlog_events.inc + +let $binlog_start = query_get_value("SHOW MASTER STATUS", Position, 1); +--error ER_NOT_SUPPORTED_YET +ALTER TABLE t1 DROP INDEX `vi2`, DROP COLUMN `v2`; +ALTER TABLE t1 DROP INDEX `vi2`; +--source include/show_binlog_events.inc + +let $binlog_start = query_get_value("SHOW MASTER STATUS", Position, 1); +CREATE VECTOR INDEX `vi` ON t1(v); +--source include/show_binlog_events.inc + +let $binlog_start = query_get_value("SHOW MASTER STATUS", Position, 1); +DROP INDEX `vi` ON t1; +--source include/show_binlog_events.inc + +--source include/sync_slave_sql_with_master.inc +--source include/rpl_connection_slave.inc +SHOW CREATE TABLE t1; + +--echo +--echo # 2. Test dml in transactions +--echo # Prepare +--source include/rpl_connection_master.inc +SET transaction_isolation = 'READ-COMMITTED'; + +DROP TABLE IF EXISTS t1; +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) + ) ENGINE = InnoDB; +--disable_warnings + +--echo # Insert data using transactions +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,10]")); +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,9]")); + +BEGIN; + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,8]")); + +SAVEPOINT sp1; + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,7]")); + +SAVEPOINT sp2; + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,6]")); + +ROLLBACK TO SAVEPOINT sp2; + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,5]")); + +COMMIT; + +BEGIN; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,4]")); +ROLLBACK; + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,3]")); +--source include/rpl_sync.inc + +--echo +--echo # Validate data in mater +SELECT id, VEC_TOTEXT(v) FROM t1; + +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; + +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; + +--echo +--echo # Validate data in slave +--source include/rpl_connection_slave.inc +SET transaction_isolation = 'READ-COMMITTED'; + +SELECT id, VEC_TOTEXT(v) FROM t1; + +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; + +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; + +--echo +--echo # 3. Clear +--enable_warnings +SET GLOBAL vidx_disabled = ON; + +--source include/rpl_connection_master.inc +DROP TABLE t1; + +SET GLOBAL vidx_disabled = ON; + +--source include/rpl_sync.inc +--source include/rpl_end.inc diff --git a/mysql-test/suite/rds/t/vidx_support_nullable.test b/mysql-test/suite/rds/t/vidx_support_nullable.test new file mode 100644 index 00000000000..8696da0e8b7 --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_support_nullable.test @@ -0,0 +1,35 @@ +--source include/have_binlog_format_mixed_or_row.inc + +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; + +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + a INT, + v VECTOR(5), + VECTOR INDEX vi(v) + ) ENGINE = InnoDB; +SHOW CREATE TABLE t1; + +INSERT INTO t1 VALUES(1, 1, NULL); +INSERT INTO t1 VALUES(2, 2, VEC_FROMTEXT("[1,2,3,4,5]")); +INSERT INTO t1 VALUES(3, 3, NULL); +INSERT INTO t1 VALUES(4, 4, NULL); + +DELETE FROM t1 WHERE id = 1; +UPDATE t1 SET v = NULL WHERE id = 2; +UPDATE t1 SET v = VEC_FROMTEXT("[1,2,3,4,5]") WHERE id = 3; +UPDATE t1 SET a=5 WHERE id = 4; + +SELECT id, a, VEC_TOTEXT(v) FROM t1; + +--echo # Rows with NULL vector columns cannot be found using the vector index. +SELECT id, a, VEC_TOTEXT(v), VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) as dis FROM t1 FORCE INDEX(`vi`) ORDER BY dis; + +--echo # Rows with NULL distances to target will be placed at the end of the result set. +SELECT id, a, VEC_TOTEXT(v), VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) as dis FROM t1 FORCE INDEX(PRIMARY) ORDER BY dis; + +DROP TABLE t1; + +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rds/t/vidx_trx.test b/mysql-test/suite/rds/t/vidx_trx.test new file mode 100644 index 00000000000..0bf20c34efe --- /dev/null +++ b/mysql-test/suite/rds/t/vidx_trx.test @@ -0,0 +1,61 @@ +########################################## +# The transaction for VECTOR INDEX # +########################################## +--source include/have_binlog_format_mixed_or_row.inc +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL vidx_disabled = OFF; +SET transaction_isolation = 'READ-COMMITTED'; + +--echo # 1. Prepare +CREATE TABLE t1 ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + v VECTOR(5), + VECTOR INDEX vi(v) + ) ENGINE = InnoDB; +--disable_warnings + +--echo +--echo # 2. Insert data using transactions +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,10]")); +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,9]")); + +BEGIN; + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,8]")); + +SAVEPOINT sp1; + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,7]")); + +SAVEPOINT sp2; + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,6]")); + +ROLLBACK TO SAVEPOINT sp2; + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,5]")); + +COMMIT; + +BEGIN; +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,4]")); +ROLLBACK; + +INSERT INTO t1(v) VALUES (VEC_FROMTEXT("[1,2,3,4,3]")); + +--echo +--echo # 3. Validate data +SELECT id, VEC_TOTEXT(v) FROM t1; + +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(PRIMARY) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; + +EXPLAIN SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; +SELECT id, VEC_TOTEXT(v) FROM t1 FORCE INDEX(`vi`) ORDER BY VEC_DISTANCE_EUCLIDEAN(v, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 5; + +--echo +--echo # 4. Clear +--enable_warnings +DROP TABLE t1; + +SET GLOBAL vidx_disabled = ON; diff --git a/mysql-test/suite/rpl/r/rpl_conditional_comments.result b/mysql-test/suite/rpl/r/rpl_conditional_comments.result index ac48dbf7c90..7b88acadd6c 100644 --- a/mysql-test/suite/rpl/r/rpl_conditional_comments.result +++ b/mysql-test/suite/rpl/r/rpl_conditional_comments.result @@ -12,11 +12,11 @@ master-bin.000001 # Query # # use `test`; CREATE TABLE t1(c1 INT) # ------------------------------------------------------------------ # In a statement, some CCs are applied while others are not. The CCs # which are not applied on master will be binlogged as common comments. -/*!99999 --- */INSERT /*!INTO*/ /*!10000 t1 */ VALUES(10) /*!99999 ,(11)*/; +/*!99998 --- */INSERT /*!INTO*/ /*!10000 t1 */ VALUES(10) /*!99998 ,(11)*/; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000001 # Query # # BEGIN -master-bin.000001 # Query # # use `test`; /* 99999 --- */INSERT /*!INTO*/ /*!10000 t1 */ VALUES(10) /* 99999 ,(11)*/ +master-bin.000001 # Query # # use `test`; /* 99998 --- */INSERT /*!INTO*/ /*!10000 t1 */ VALUES(10) /* 99998 ,(11)*/ master-bin.000001 # Query # # COMMIT include/sync_slave_sql_with_master.inc include/diff_tables.inc [master:t1,slave:t1] @@ -25,7 +25,7 @@ include/diff_tables.inc [master:t1,slave:t1] # ----------------------------------------------------------------- # Verify whether it can be binlogged correctly when executing prepared # statement. -PREPARE stmt FROM 'INSERT INTO /*!99999 blabla*/ t1 VALUES(60) /*!99999 ,(61)*/'; +PREPARE stmt FROM 'INSERT INTO /*!99998 blabla*/ t1 VALUES(60) /*!99998 ,(61)*/'; EXECUTE stmt; DROP TABLE t1; CREATE TABLE t1(c1 INT); @@ -34,7 +34,7 @@ include/sync_slave_sql_with_master.inc include/diff_tables.inc [master:t1,slave:t1] SET @value=62; -PREPARE stmt FROM 'INSERT INTO /*!99999 blabla */ t1 VALUES(?) /*!99999 ,(63)*/'; +PREPARE stmt FROM 'INSERT INTO /*!99998 blabla */ t1 VALUES(?) /*!99998 ,(63)*/'; EXECUTE stmt USING @value; DROP TABLE t1; CREATE TABLE t1(c1 INT); @@ -42,20 +42,20 @@ EXECUTE stmt USING @value; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000001 # Query # # BEGIN -master-bin.000001 # Query # # use `test`; INSERT INTO /* 99999 blabla*/ t1 VALUES(60) /* 99999 ,(61)*/ +master-bin.000001 # Query # # use `test`; INSERT INTO /* 99998 blabla*/ t1 VALUES(60) /* 99998 ,(61)*/ master-bin.000001 # Query # # COMMIT master-bin.000001 # Query # # use `test`; DROP TABLE `t1` /* generated by server */ master-bin.000001 # Query # # use `test`; CREATE TABLE t1(c1 INT) master-bin.000001 # Query # # BEGIN -master-bin.000001 # Query # # use `test`; INSERT INTO /* 99999 blabla*/ t1 VALUES(60) /* 99999 ,(61)*/ +master-bin.000001 # Query # # use `test`; INSERT INTO /* 99998 blabla*/ t1 VALUES(60) /* 99998 ,(61)*/ master-bin.000001 # Query # # COMMIT master-bin.000001 # Query # # BEGIN -master-bin.000001 # Query # # use `test`; INSERT INTO /* 99999 blabla */ t1 VALUES(62) /* 99999 ,(63)*/ +master-bin.000001 # Query # # use `test`; INSERT INTO /* 99998 blabla */ t1 VALUES(62) /* 99998 ,(63)*/ master-bin.000001 # Query # # COMMIT master-bin.000001 # Query # # use `test`; DROP TABLE `t1` /* generated by server */ master-bin.000001 # Query # # use `test`; CREATE TABLE t1(c1 INT) master-bin.000001 # Query # # BEGIN -master-bin.000001 # Query # # use `test`; INSERT INTO /* 99999 blabla */ t1 VALUES(62) /* 99999 ,(63)*/ +master-bin.000001 # Query # # use `test`; INSERT INTO /* 99998 blabla */ t1 VALUES(62) /* 99998 ,(63)*/ master-bin.000001 # Query # # COMMIT include/sync_slave_sql_with_master.inc include/diff_tables.inc [master:t1,slave:t1] @@ -64,8 +64,8 @@ include/diff_tables.inc [master:t1,slave:t1] # ----------------------------------------------------------------- # Verify it can restore the '!', if the it is an uncomplete conditional # comments -SELECT c1 FROM /*!99999 t1 WHEREN; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '/*!99999 t1 WHEREN' at line 1 +SELECT c1 FROM /*!99998 t1 WHEREN; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '/*!99998 t1 WHEREN' at line 1 # # Bug#53373 Comments are not properly removed from the # pre-processed query buffer. diff --git a/mysql-test/suite/rpl/t/rpl_conditional_comments.test b/mysql-test/suite/rpl/t/rpl_conditional_comments.test index 2a4c66b1132..4755daa0339 100644 --- a/mysql-test/suite/rpl/t/rpl_conditional_comments.test +++ b/mysql-test/suite/rpl/t/rpl_conditional_comments.test @@ -4,7 +4,7 @@ # master. So they become common comments and will not be applied on slave. # # - Example: -# 'INSERT INTO t1 VALUES (1) /*!10000, (2)*/ /*!99999 ,(3)*/ +# 'INSERT INTO t1 VALUES (1) /*!10000, (2)*/ /*!99998 ,(3)*/ # will be binlogged as # 'INSERT INTO t1 VALUES (1) /*!10000, (2)*/ /* 99999 ,(3)*/'. ############################################################################### @@ -23,7 +23,7 @@ let $binlog_start= query_get_value(SHOW MASTER STATUS, Position, 1); --echo # In a statement, some CCs are applied while others are not. The CCs --echo # which are not applied on master will be binlogged as common comments. -/*!99999 --- */INSERT /*!INTO*/ /*!10000 t1 */ VALUES(10) /*!99999 ,(11)*/; +/*!99998 --- */INSERT /*!INTO*/ /*!10000 t1 */ VALUES(10) /*!99998 ,(11)*/; source include/show_binlog_events.inc; let $binlog_start= query_get_value(SHOW MASTER STATUS, Position, 1); @@ -37,7 +37,7 @@ let $binlog_start= query_get_value(SHOW MASTER STATUS, Position, 1); --echo # Verify whether it can be binlogged correctly when executing prepared --echo # statement. --connection master -PREPARE stmt FROM 'INSERT INTO /*!99999 blabla*/ t1 VALUES(60) /*!99999 ,(61)*/'; +PREPARE stmt FROM 'INSERT INTO /*!99998 blabla*/ t1 VALUES(60) /*!99998 ,(61)*/'; EXECUTE stmt; DROP TABLE t1; CREATE TABLE t1(c1 INT); @@ -50,7 +50,7 @@ EXECUTE stmt; --connection master --echo SET @value=62; -PREPARE stmt FROM 'INSERT INTO /*!99999 blabla */ t1 VALUES(?) /*!99999 ,(63)*/'; +PREPARE stmt FROM 'INSERT INTO /*!99998 blabla */ t1 VALUES(?) /*!99998 ,(63)*/'; EXECUTE stmt USING @value; DROP TABLE t1; CREATE TABLE t1(c1 INT); @@ -70,7 +70,7 @@ let $binlog_start= query_get_value(SHOW MASTER STATUS, Position, 1); --echo # comments --connection master --error 1064 -SELECT c1 FROM /*!99999 t1 WHEREN; +SELECT c1 FROM /*!99998 t1 WHEREN; --echo # diff --git a/mysql-test/suite/rpl_nogtid/r/rpl_loaddatalocal.result b/mysql-test/suite/rpl_nogtid/r/rpl_loaddatalocal.result index 8e857623c91..0096edfd7f9 100644 --- a/mysql-test/suite/rpl_nogtid/r/rpl_loaddatalocal.result +++ b/mysql-test/suite/rpl_nogtid/r/rpl_loaddatalocal.result @@ -79,7 +79,7 @@ LOAD DATA /*!10000 LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql' INTO TABLE */ t1; LOAD DATA/*!10000 LOCAL */INFILE 'MYSQLD_DATADIR/bug43746.sql'/*!10000 INTO*/TABLE t1; LOAD DATA/*!10000 LOCAL */INFILE 'MYSQLD_DATADIR/bug43746.sql'/* empty */INTO TABLE t1; LOAD DATA/*!10000 LOCAL */INFILE 'MYSQLD_DATADIR/bug43746.sql' INTO/* empty */TABLE t1; -LOAD/*!99999 special comments that do not expand */DATA/*!99999 code from the future */LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql'/*!99999 have flux capacitor */INTO/*!99999 will travel */TABLE t1; +LOAD/*!99998 special comments that do not expand */DATA/*!99998 code from the future */LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql'/*!99998 have flux capacitor */INTO/*!99998 will travel */TABLE t1; SET sql_mode='PIPES_AS_CONCAT,ANSI_QUOTES,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,TRADITIONAL'; LOAD DATA LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql' INTO TABLE t1; [slave] diff --git a/mysql-test/suite/rpl_nogtid/t/rpl_loaddatalocal.test b/mysql-test/suite/rpl_nogtid/t/rpl_loaddatalocal.test index ca69201851e..556eb3ec79d 100644 --- a/mysql-test/suite/rpl_nogtid/t/rpl_loaddatalocal.test +++ b/mysql-test/suite/rpl_nogtid/t/rpl_loaddatalocal.test @@ -143,7 +143,7 @@ eval LOAD DATA/*!10000 LOCAL */INFILE '$MYSQLD_DATADIR/bug43746.sql'/* empty */I eval LOAD DATA/*!10000 LOCAL */INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO/* empty */TABLE t1; --replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD/*!99999 special comments that do not expand */DATA/*!99999 code from the future */LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql'/*!99999 have flux capacitor */INTO/*!99999 will travel */TABLE t1; +eval LOAD/*!99998 special comments that do not expand */DATA/*!99998 code from the future */LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql'/*!99998 have flux capacitor */INTO/*!99998 will travel */TABLE t1; SET sql_mode='PIPES_AS_CONCAT,ANSI_QUOTES,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,TRADITIONAL'; diff --git a/mysql-test/suite/sys_vars/r/all_vars.result b/mysql-test/suite/sys_vars/r/all_vars.result index 982bea5f865..37b347d57f0 100644 --- a/mysql-test/suite/sys_vars/r/all_vars.result +++ b/mysql-test/suite/sys_vars/r/all_vars.result @@ -240,6 +240,16 @@ terminology_use_previous tls_ciphersuites tls_ciphersuites use_secondary_engine +vidx_default_distance +vidx_default_distance +vidx_disabled +vidx_disabled +vidx_hnsw_cache_size +vidx_hnsw_cache_size +vidx_hnsw_default_m +vidx_hnsw_default_m +vidx_hnsw_ef_search +vidx_hnsw_ef_search xa_detach_on_prepare xa_detach_on_prepare drop table t1; diff --git a/mysql-test/t/all_persisted_variables.test b/mysql-test/t/all_persisted_variables.test index 3205f019c90..e735d12caf1 100644 --- a/mysql-test/t/all_persisted_variables.test +++ b/mysql-test/t/all_persisted_variables.test @@ -41,7 +41,7 @@ call mtr.add_suppression("\\[Warning\\] .*MY-\\d+.* Changing innodb_extend_and_i call mtr.add_suppression("Failed to initialize TLS for channel: mysql_main"); let $total_global_vars=`SELECT COUNT(*) FROM performance_schema.global_variables where variable_name NOT LIKE 'ndb_%' AND variable_name NOT LIKE 'debug_%'`; -let $total_persistent_vars=476; +let $total_persistent_vars=481; --echo *************************************************************** --echo * 0. Verify that variables present in performance_schema.global diff --git a/mysql-test/t/comments.test b/mysql-test/t/comments.test index 27e558c9482..8576140ecc9 100644 --- a/mysql-test/t/comments.test +++ b/mysql-test/t/comments.test @@ -9,7 +9,7 @@ multi line comment */; --error 1065 ; select 1 /*!32301 +1 */; -select 1 /*!999999 +1 */; +select 1 /*!99998 +1 */; select 1--1; # Note that the following returns 4 while it should return 2 # This is because the mysqld server doesn't parse -- comments @@ -31,9 +31,9 @@ select 1/*!2*/; --error ER_PARSE_ERROR select 1/*!000002*/; -select 1/*!999992*/; +select 1/*!99992*/; -select 1 + /*!00000 2 */ + 3 /*!99999 noise*/ + 4; +select 1 + /*!00000 2 */ + 3 /*!99998 noise*/ + 4; # # Bug#28779 (mysql_query() allows execution of statements with unbalanced @@ -70,7 +70,7 @@ drop table table_28779; --echo # WL#12099: Deprecate nested comments in 8.0 --echo # -SELECT 1 /*!99999 /* */ */; +SELECT 1 /*!99998 /* */ */; SELECT 2 /*!12345 /* */ */; SELECT 3 /*! /* */ */; @@ -97,7 +97,8 @@ DO 1 /*!80034 +1*/; DO 1 /*!80034 +1*/; --echo # Carriage return (0x0d). Should pass without warning. -DO 1 /*!80034 +1*/; +DO 1 /*!80034 ++1*/; --echo # Space (0x20). Should pass without warning. DO 1 /*!80034 +1*/; diff --git a/mysql-test/t/no_binlog_gtid_empty_statement.test b/mysql-test/t/no_binlog_gtid_empty_statement.test index 275cd6fe1fa..32ead4662d1 100644 --- a/mysql-test/t/no_binlog_gtid_empty_statement.test +++ b/mysql-test/t/no_binlog_gtid_empty_statement.test @@ -23,11 +23,11 @@ USE test; --source include/set_gtid_next.inc CREATE TABLE t1 (c1 INT); # This statement should be considered an empty statement -/*!99999 SET @@SESSION.non_supported_session_variable = 1*/; +/*!99998 SET @@SESSION.non_supported_session_variable = 1*/; --source include/set_gtid_next.inc INSERT INTO t1 VALUES (1); # This statement should be considered an empty statement -/*!99999 SET @@SESSION.non_supported_session_variable = 1*/; +/*!99998 SET @@SESSION.non_supported_session_variable = 1*/; --source include/set_gtid_next.inc DROP TABLE t1; @@ -46,11 +46,11 @@ RESET MASTER; --source include/set_gtid_next.inc CREATE TABLE t2 (c1 INT); # This statement should be considered an empty statement -/*!99999 SET @@SESSION.non_supported_session_variable = 1*/; +/*!99998 SET @@SESSION.non_supported_session_variable = 1*/; --source include/set_gtid_next.inc INSERT INTO t2 VALUES (1); # This statement should be considered an empty statement -/*!99999 SET @@SESSION.non_supported_session_variable = 1*/; +/*!99998 SET @@SESSION.non_supported_session_variable = 1*/; --source include/set_gtid_next.inc DROP TABLE t2; diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index d6a28310294..c32ed072f57 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -7112,7 +7112,7 @@ begin /*! select 2; */ select 3; /*!00000 select 4; */ - /*!99999 select 5; */ + /*!99998 select 5; */ end $$ @@ -7120,7 +7120,7 @@ create procedure proc_25411_b( /* real comment */ /*! p1 int, */ /*!00000 p2 int */ -/*!99999 ,p3 int */ +/*!99998 ,p3 int */ ) begin select p1, p2; @@ -7129,11 +7129,11 @@ $$ create procedure proc_25411_c() begin - select 1/*!,2*//*!00000,3*//*!99999,4*/; - select 1/*! ,2*//*!00000 ,3*//*!99999 ,4*/; - select 1/*!,2 *//*!00000,3 *//*!99999,4 */; - select 1/*! ,2 *//*!00000 ,3 *//*!99999 ,4 */; - select 1 /*!,2*/ /*!00000,3*/ /*!99999,4*/ ; + select 1/*!,2*//*!00000,3*//*!99998,4*/; + select 1/*! ,2*//*!00000 ,3*//*!99998 ,4*/; + select 1/*!,2 *//*!00000,3 *//*!99998,4 */; + select 1/*! ,2 *//*!00000 ,3 *//*!99998 ,4 */; + select 1 /*!,2*/ /*!00000,3*/ /*!99998,4*/ ; end $$ diff --git a/mysql-test/t/sql_safe_updates_cmdline-master.opt b/mysql-test/t/sql_safe_updates_cmdline-master.opt new file mode 100644 index 00000000000..b2c0ba45603 --- /dev/null +++ b/mysql-test/t/sql_safe_updates_cmdline-master.opt @@ -0,0 +1 @@ +--sql-safe-updates=1 diff --git a/mysql-test/t/sql_safe_updates_cmdline.test b/mysql-test/t/sql_safe_updates_cmdline.test new file mode 100644 index 00000000000..49beed8fece --- /dev/null +++ b/mysql-test/t/sql_safe_updates_cmdline.test @@ -0,0 +1,99 @@ +############################################################################### +# Test for --sql-safe-updates command line option # +# # +# This test verifies that the --sql-safe-updates command line option works # +# correctly to set the global default value for sql_safe_updates. # +# # +# The server is started with --sql-safe-updates=1 (see .opt file) # +############################################################################### + +# Force restart after test because we modify global sql_safe_updates +--source include/force_restart.inc + +--echo # +--echo # Test 1: Verify global and session default values are ON +--echo # when server is started with --sql-safe-updates=1 +--echo # + +SELECT @@global.sql_safe_updates AS global_value; +SELECT @@session.sql_safe_updates AS session_value; + +--echo # +--echo # Test 2: Verify that sql_safe_updates is enforced +--echo # (UPDATE without WHERE using KEY should fail) +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, name VARCHAR(50)); +INSERT INTO t1 VALUES (1, 'test1'), (2, 'test2'), (3, 'test3'); + +--echo # This should fail with sql_safe_updates enabled +--error ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE +UPDATE t1 SET name = 'updated'; + +--echo # This should fail with sql_safe_updates enabled +--error ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE +DELETE FROM t1; + +--echo # UPDATE with WHERE using KEY column should succeed +UPDATE t1 SET name = 'updated1' WHERE id = 1; +SELECT * FROM t1 WHERE id = 1; + +--echo # +--echo # Test 3: Verify session can override the global value +--echo # + +SET SESSION sql_safe_updates = OFF; +SELECT @@session.sql_safe_updates AS session_after_off; +SELECT @@global.sql_safe_updates AS global_unchanged; + +--echo # Now UPDATE without WHERE should succeed +UPDATE t1 SET name = 'all_updated'; +SELECT * FROM t1 ORDER BY id; + +--echo # +--echo # Test 4: Verify new session inherits global value +--echo # + +--echo # Reconnect to get a new session +--connect (con1, localhost, root,,) + +SELECT @@session.sql_safe_updates AS new_session_value; +SELECT @@global.sql_safe_updates AS global_value; + +--echo # UPDATE without WHERE should fail in new session +--error ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE +UPDATE t1 SET name = 'fail'; + +--connection default +--disconnect con1 + +--echo # +--echo # Test 5: Verify SET GLOBAL works +--echo # + +SET GLOBAL sql_safe_updates = OFF; +SELECT @@global.sql_safe_updates AS global_after_set_off; + +--echo # Reconnect to verify new session gets the new global value +--connect (con2, localhost, root,,) + +SELECT @@session.sql_safe_updates AS new_session_inherits_global; + +--echo # UPDATE without WHERE should now succeed in new session +UPDATE t1 SET name = 'success'; +SELECT * FROM t1 ORDER BY id; + +--connection default +--disconnect con2 + +--echo # +--echo # Cleanup +--echo # + +# Reset to default OFF to avoid affecting MTR check_warnings +SET GLOBAL sql_safe_updates = OFF; +DROP TABLE t1; + +--echo # +--echo # Test completed successfully +--echo # diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index b3e1cca7215..5d36f9ba20a 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -123,6 +123,7 @@ SET(MYSYS_SOURCES my_md5.cc my_rnd.cc my_openssl_fips.cc + hash.cc ) LIST(APPEND MYSYS_SOURCES my_aes_openssl.cc) diff --git a/mysys/array.cc b/mysys/array.cc index 0c85c6e4423..8120f3852e0 100644 --- a/mysys/array.cc +++ b/mysys/array.cc @@ -185,3 +185,21 @@ void delete_dynamic(DYNAMIC_ARRAY *array) { array->elements = array->max_element = 0; } } + +/* + Pop last element from array. + + SYNOPSIS + pop_dynamic() + array + + RETURN VALUE + pointer Ok + 0 Array is empty +*/ + +void *pop_dynamic(DYNAMIC_ARRAY *array) { + if (array->elements) + return array->buffer + (--array->elements * array->size_of_element); + return 0; +} diff --git a/mysys/hash.cc b/mysys/hash.cc new file mode 100644 index 00000000000..3ac66e736ce --- /dev/null +++ b/mysys/hash.cc @@ -0,0 +1,826 @@ +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. + Copyright (c) 2011, 2020, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* The hash functions used for saveing keys */ +/* One of key_length or key_length_offset must be given */ +/* Key length of 0 isn't allowed */ + +#include "vidx/hash.h" +#include +#include +#include "my_dbug.h" +#include "mysys_priv.h" + +#define NO_RECORD ~((my_hash_value_type)0) +#define LOWFIND 1 +#define LOWUSED 2 +#define HIGHFIND 4 +#define HIGHUSED 8 + +typedef struct st_hash_info { + uint32 next; /* index to next key */ + my_hash_value_type hash_nr; + uchar *data; /* data for current entry */ +} HASH_LINK; + +static uint my_hash_mask(my_hash_value_type hashnr, size_t buffmax, + size_t maxlength); +static void movelink(HASH_LINK *array, uint pos, uint next_link, uint newlink); +static int hashcmp(const HASH *hash, HASH_LINK *pos, const uchar *key, + size_t length); + +my_hash_value_type my_hash_sort(CHARSET_INFO *cs, const uchar *key, + size_t length) { + ulong nr1 = 1, nr2 = 4; + my_ci_hash_sort(cs, (uchar *)key, length, &nr1, &nr2); + return (my_hash_value_type)nr1; +} + +/** + @brief Initialize the hash + + @details + + Initialize the hash, by defining and giving valid values for + its elements. The failure to allocate memory for the + hash->array element will not result in a fatal failure. The + dynamic array that is part of the hash will allocate memory + as required during insertion. + + @param[in] psi_key The key to register instrumented memory + @param[in,out] hash The hash that is initialized + @param[in] growth_size size incrememnt for the underlying dynarray + @param[in] charset The character set information + @param[in] size The hash size + @param[in] key_offset The key offset for the hash + @param[in] key_length The length of the key used in + the hash + @param[in] get_key get the key for the hash + @param[in] free_element pointer to the function that + does cleanup + @param[in] flags flags set in the hash + @return indicates success or failure of initialization + @retval 0 success + @retval 1 failure +*/ +my_bool my_hash_init2(PSI_memory_key psi_key, HASH *hash, size_t growth_size, + CHARSET_INFO *charset, size_t size, size_t key_offset, + size_t key_length, my_hash_get_key get_key, + my_hash_function hash_function, + void (*free_element)(void *), uint flags) { + my_bool res; + DBUG_ENTER("my_hash_init2"); + DBUG_PRINT("enter", ("hash:%p size: %u", hash, (uint)size)); + + hash->records = 0; + hash->key_offset = key_offset; + hash->key_length = key_length; + hash->blength = 1; + hash->get_key = get_key; + hash->hash_function = hash_function ? hash_function : my_hash_sort; + hash->free = free_element; + hash->flags = flags; + hash->charset = charset; + res = my_init_dynamic_array(&hash->array, psi_key, sizeof(HASH_LINK), nullptr, + size, growth_size); + DBUG_RETURN(res); +} + +/* + Call hash->free on all elements in hash. + + SYNOPSIS + my_hash_free_elements() + hash hash table + + NOTES: + Sets records to 0 +*/ + +static inline void my_hash_free_elements(HASH *hash) { + uint records = hash->records; + if (records == 0) return; + + /* + Set records to 0 early to guard against anyone looking at the structure + during the free process + */ + hash->records = 0; + + if (hash->free) { + HASH_LINK *data = dynamic_element(&hash->array, 0, HASH_LINK *); + HASH_LINK *end = data + records; + do { + (*hash->free)((data++)->data); + } while (data < end); + } +} + +/* + Free memory used by hash. + + SYNOPSIS + my_hash_free() + hash the hash to delete elements of + + NOTES: Hash can't be reused without calling my_hash_init again. +*/ + +void my_hash_free(HASH *hash) { + DBUG_ENTER("my_hash_free"); + DBUG_PRINT("enter", ("hash:%p elements: %ld", hash, hash->records)); + + my_hash_free_elements(hash); + hash->free = 0; + delete_dynamic(&hash->array); + hash->blength = 0; + DBUG_VOID_RETURN; +} + +/* + Delete all elements from the hash (the hash itself is to be reused). + + SYNOPSIS + my_hash_reset() + hash the hash to delete elements of +*/ + +void my_hash_reset(HASH *hash) { + DBUG_ENTER("my_hash_reset"); + DBUG_PRINT("enter", ("hash:%p", hash)); + + my_hash_free_elements(hash); + reset_dynamic(&hash->array); + /* Set row pointers so that the hash can be reused at once */ + hash->blength = 1; + DBUG_VOID_RETURN; +} + +/* some helper functions */ + +/* + This function is char* instead of uchar* as HPUX11 compiler can't + handle inline functions that are not defined as native types +*/ + +static inline char *my_hash_key(const HASH *hash, const uchar *record, + size_t *length, my_bool first) { + if (hash->get_key) return (char *)(*hash->get_key)(record, length, first); + *length = hash->key_length; + return (char *)record + hash->key_offset; +} + +/* Calculate pos according to keys */ + +static uint my_hash_mask(my_hash_value_type hashnr, size_t buffmax, + size_t maxlength) { + if ((hashnr & (buffmax - 1)) < maxlength) + return (uint)(hashnr & (buffmax - 1)); + return (uint)(hashnr & ((buffmax >> 1) - 1)); +} + +static inline uint my_hash_rec_mask(HASH_LINK *pos, size_t buffmax, + size_t maxlength) { + return my_hash_mask(pos->hash_nr, buffmax, maxlength); +} + +/* for compilers which can not handle inline */ +static +#if !defined(__USLC__) && !defined(__sgi) + inline +#endif + my_hash_value_type + rec_hashnr(HASH *hash, const uchar *record) { + size_t length; + uchar *key = (uchar *)my_hash_key(hash, record, &length, 0); + return hash->hash_function(hash->charset, key, length); +} + +uchar *my_hash_search(const HASH *hash, const uchar *key, size_t length) { + HASH_SEARCH_STATE state; + return my_hash_first(hash, key, length, &state); +} + +uchar *my_hash_search_using_hash_value(const HASH *hash, + my_hash_value_type hash_value, + const uchar *key, size_t length) { + HASH_SEARCH_STATE state; + return my_hash_first_from_hash_value(hash, hash_value, key, length, &state); +} + +/* + Search after a record based on a key + + NOTE + Assigns the number of the found record to HASH_SEARCH_STATE state +*/ + +uchar *my_hash_first(const HASH *hash, const uchar *key, size_t length, + HASH_SEARCH_STATE *current_record) { + uchar *res; + assert(my_hash_inited(hash)); + + res = my_hash_first_from_hash_value( + hash, + hash->hash_function(hash->charset, key, + length ? length : hash->key_length), + key, length, current_record); + return res; +} + +uchar *my_hash_first_from_hash_value(const HASH *hash, + my_hash_value_type hash_value, + const uchar *key, size_t length, + HASH_SEARCH_STATE *current_record) { + HASH_LINK *pos; + DBUG_ENTER("my_hash_first_from_hash_value"); + + if (hash->records) { + uint flag = 1; + uint idx = my_hash_mask(hash_value, hash->blength, hash->records); + if (!length) + length = hash->key_length; // length for fixed length keys or 0 + do { + pos = dynamic_element(&hash->array, idx, HASH_LINK *); + if (!hashcmp(hash, pos, key, length)) { + DBUG_PRINT("exit", ("found key at %d", idx)); + *current_record = idx; + DBUG_RETURN(pos->data); + } + if (flag) { + flag = 0; /* Reset flag */ + if (my_hash_rec_mask(pos, hash->blength, hash->records) != idx) + break; /* Wrong link */ + } + } while ((idx = pos->next) != NO_RECORD); + } + *current_record = NO_RECORD; + DBUG_RETURN(0); +} + +/* Get next record with identical key */ +/* Can only be called if previous calls was my_hash_search */ + +uchar *my_hash_next(const HASH *hash, const uchar *key, size_t length, + HASH_SEARCH_STATE *current_record) { + HASH_LINK *pos; + uint idx; + + if (*current_record != NO_RECORD) { + HASH_LINK *data = dynamic_element(&hash->array, 0, HASH_LINK *); + if (!length) + length = hash->key_length; // length for fixed length keys or 0 + for (idx = data[*current_record].next; idx != NO_RECORD; idx = pos->next) { + pos = data + idx; + if (!hashcmp(hash, pos, key, length)) { + *current_record = idx; + return pos->data; + } + } + *current_record = NO_RECORD; + } + return 0; +} + +/* Change link from pos to new_link */ + +static void movelink(HASH_LINK *array, uint find, uint next_link, + uint newlink) { + HASH_LINK *old_link; + do { + old_link = array + next_link; + } while ((next_link = old_link->next) != find); + old_link->next = newlink; + return; +} + +/* + Compare a key in a record to a whole key. Return 0 if identical + + SYNOPSIS + hashcmp() + hash hash table + pos position of hash record to use in comparison + key key for comparison + length length of key + + NOTES: + length equal 0 can mean 2 things: + 1) it is fixed key length hash (HASH::key_length != 0) and + default length should be taken in this case + 2) it is really 0 length key for variable key length hash + (HASH::key_length == 0) + + RETURN + = 0 key of record == key + != 0 key of record != key + */ + +static int hashcmp(const HASH *hash, HASH_LINK *pos, const uchar *key, + size_t length) { + size_t rec_keylength; + uchar *rec_key; + rec_key = (uchar *)my_hash_key(hash, pos->data, &rec_keylength, 1); + return my_strnncoll(hash->charset, (uchar *)rec_key, rec_keylength, + (uchar *)key, length); +} + +/** + Write a hash-key to the hash-index + + @return + @retval 0 ok + @retval 1 Duplicate key or out of memory +*/ + +my_bool my_hash_insert(HASH *info, const uchar *record) { + int flag; + size_t idx, halfbuff, first_index; + size_t length; + my_hash_value_type current_hash_nr, rec_hash_nr = 0, rec2_hash_nr = 0; + uchar *rec_data = nullptr, *rec2_data = nullptr, *key; + HASH_LINK *data, *empty, *gpos = nullptr, *gpos2 = nullptr, *pos; + + key = (uchar *)my_hash_key(info, record, &length, 1); + current_hash_nr = info->hash_function(info->charset, key, length); + + if (info->flags & HASH_UNIQUE) { + if (my_hash_search_using_hash_value(info, current_hash_nr, key, length)) + return (TRUE); /* Duplicate entry */ + } + + flag = 0; + if (!(empty = (HASH_LINK *)alloc_dynamic(&info->array))) + return (TRUE); /* No more memory */ + + data = dynamic_element(&info->array, 0, HASH_LINK *); + halfbuff = info->blength >> 1; + + idx = first_index = info->records - halfbuff; + if (idx != info->records) /* If some records */ + { + do { + my_hash_value_type hash_nr; + pos = data + idx; + hash_nr = pos->hash_nr; + if (flag == 0) /* First loop; Check if ok */ + if (my_hash_mask(hash_nr, info->blength, info->records) != first_index) + break; + if (!(hash_nr & halfbuff)) { /* Key will not move */ + if (!(flag & LOWFIND)) { + if (flag & HIGHFIND) { + flag = LOWFIND | HIGHFIND; + /* key shall be moved to the current empty position */ + gpos = empty; + rec_data = pos->data; + rec_hash_nr = pos->hash_nr; + empty = pos; /* This place is now free */ + } else { + flag = LOWFIND | LOWUSED; /* key isn't changed */ + gpos = pos; + rec_data = pos->data; + rec_hash_nr = pos->hash_nr; + } + } else { + if (!(flag & LOWUSED)) { + /* Change link of previous LOW-key */ + gpos->data = rec_data; + gpos->hash_nr = rec_hash_nr; + gpos->next = (uint)(pos - data); + flag = (flag & HIGHFIND) | (LOWFIND | LOWUSED); + } + gpos = pos; + rec_data = pos->data; + rec_hash_nr = pos->hash_nr; + } + } else { /* key will be moved */ + if (!(flag & HIGHFIND)) { + flag = (flag & LOWFIND) | HIGHFIND; + /* key shall be moved to the last (empty) position */ + gpos2 = empty; + empty = pos; + rec2_data = pos->data; + rec2_hash_nr = pos->hash_nr; + } else { + if (!(flag & HIGHUSED)) { + /* Change link of previous hash-key and save */ + gpos2->data = rec2_data; + gpos2->hash_nr = rec2_hash_nr; + gpos2->next = (uint)(pos - data); + flag = (flag & LOWFIND) | (HIGHFIND | HIGHUSED); + } + gpos2 = pos; + rec2_data = pos->data; + rec2_hash_nr = pos->hash_nr; + } + } + } while ((idx = pos->next) != NO_RECORD); + + if ((flag & (LOWFIND | LOWUSED)) == LOWFIND) { + gpos->data = rec_data; + gpos->hash_nr = rec_hash_nr; + gpos->next = NO_RECORD; + } + if ((flag & (HIGHFIND | HIGHUSED)) == HIGHFIND) { + gpos2->data = rec2_data; + gpos2->hash_nr = rec2_hash_nr; + gpos2->next = NO_RECORD; + } + } + + idx = my_hash_mask(current_hash_nr, info->blength, info->records + 1); + pos = data + idx; + /* Check if we are at the empty position */ + if (pos == empty) { + pos->next = NO_RECORD; + } else { + /* Move conflicting record to empty position (last) */ + empty[0] = pos[0]; + /* Check if the moved record was in same hash-nr family */ + gpos = data + my_hash_rec_mask(pos, info->blength, info->records + 1); + if (pos == gpos) { + /* Point to moved record */ + pos->next = (uint32)(empty - data); + } else { + pos->next = NO_RECORD; + movelink(data, (uint)(pos - data), (uint)(gpos - data), + (uint)(empty - data)); + } + } + pos->data = (uchar *)record; + pos->hash_nr = current_hash_nr; + if (++info->records == info->blength) info->blength += info->blength; + return (0); +} + +/** + Remove one record from hash-table. + + @fn hash_delete() + @param hash Hash tree + @param record Row to be deleted + + @notes + The record with the same record ptr is removed. + If there is a free-function it's called if record was found. + + hash->free() is guarantee to be called only after the row has been + deleted from the hash and the hash can be reused by other threads. + + @return + @retval 0 ok + @retval 1 Record not found +*/ + +my_bool my_hash_delete(HASH *hash, uchar *record) { + uint pos2, idx, empty_index; + my_hash_value_type pos_hashnr, lastpos_hashnr; + size_t blength; + HASH_LINK *data, *lastpos, *gpos, *pos, *pos3, *empty; + DBUG_ENTER("my_hash_delete"); + if (!hash->records) DBUG_RETURN(1); + + blength = hash->blength; + data = dynamic_element(&hash->array, 0, HASH_LINK *); + /* Search after record with key */ + pos = data + my_hash_mask(rec_hashnr(hash, record), blength, hash->records); + gpos = 0; + + while (pos->data != record) { + gpos = pos; + if (pos->next == NO_RECORD) DBUG_RETURN(1); /* Key not found */ + pos = data + pos->next; + } + + if (--(hash->records) < hash->blength >> 1) hash->blength >>= 1; + lastpos = data + hash->records; + + /* Remove link to record */ + empty = pos; + empty_index = (uint)(empty - data); + if (gpos) + gpos->next = pos->next; /* unlink current ptr */ + else if (pos->next != NO_RECORD) { + empty = data + (empty_index = pos->next); + pos[0] = empty[0]; + } + + if (empty == lastpos) /* last key at wrong pos or no next link */ + goto exit; + + /* Move the last key (lastpos) */ + lastpos_hashnr = lastpos->hash_nr; + /* pos is where lastpos should be */ + pos = data + my_hash_mask(lastpos_hashnr, hash->blength, hash->records); + if (pos == empty) /* Move to empty position. */ + { + empty[0] = lastpos[0]; + goto exit; + } + pos_hashnr = pos->hash_nr; + /* pos3 is where the pos should be */ + pos3 = data + my_hash_mask(pos_hashnr, hash->blength, hash->records); + if (pos != pos3) { /* pos is on wrong posit */ + empty[0] = pos[0]; /* Save it here */ + pos[0] = lastpos[0]; /* This should be here */ + movelink(data, (uint)(pos - data), (uint)(pos3 - data), empty_index); + goto exit; + } + pos2 = my_hash_mask(lastpos_hashnr, blength, hash->records + 1); + if (pos2 == my_hash_mask(pos_hashnr, blength, + hash->records + 1)) { /* Identical key-positions */ + if (pos2 != hash->records) { + empty[0] = lastpos[0]; + movelink(data, (uint)(lastpos - data), (uint)(pos - data), empty_index); + goto exit; + } + idx = (uint)(pos - data); /* Link pos->next after lastpos */ + } else + idx = NO_RECORD; /* Different positions merge */ + + empty[0] = lastpos[0]; + movelink(data, idx, empty_index, pos->next); + pos->next = empty_index; + +exit: + (void)pop_dynamic(&hash->array); + if (hash->free) (*hash->free)((uchar *)record); + DBUG_RETURN(0); +} + +/** + Update keys when record has changed. + This is much more efficient than using a delete & insert. +*/ + +my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key, + size_t old_key_length) { + uint new_index, new_pos_index, org_index, records, idx; + size_t length, empty, blength; + my_hash_value_type hash_nr; + HASH_LINK org_link, *data, *previous, *pos; + uchar *new_key; + DBUG_ENTER("my_hash_update"); + + new_key = (uchar *)my_hash_key(hash, record, &length, 1); + hash_nr = hash->hash_function(hash->charset, new_key, length); + + if (HASH_UNIQUE & hash->flags) { + HASH_SEARCH_STATE state; + uchar *found; + + if ((found = my_hash_first_from_hash_value(hash, hash_nr, new_key, length, + &state))) { + do { + if (found != record) DBUG_RETURN(1); /* Duplicate entry */ + } while ((found = my_hash_next(hash, new_key, length, &state))); + } + } + + data = dynamic_element(&hash->array, 0, HASH_LINK *); + blength = hash->blength; + records = hash->records; + + /* Search after record with key */ + + idx = my_hash_mask( + hash->hash_function(hash->charset, old_key, + (old_key_length ? old_key_length : hash->key_length)), + blength, records); + org_index = idx; + new_index = my_hash_mask(hash_nr, blength, records); + previous = 0; + for (;;) { + if ((pos = data + idx)->data == record) break; + previous = pos; + if ((idx = pos->next) == NO_RECORD) DBUG_RETURN(1); /* Not found in links */ + } + + if (org_index == new_index) { + data[idx].hash_nr = hash_nr; /* Hash number may have changed */ + DBUG_RETURN(0); /* Record is in right position */ + } + + org_link = *pos; + empty = idx; + + /* Relink record from current chain */ + + if (!previous) { + if (pos->next != NO_RECORD) { + empty = pos->next; + *pos = data[pos->next]; + } + } else + previous->next = pos->next; /* unlink pos */ + + /* Move data to correct position */ + if (new_index == empty) { + /* + At this point record is unlinked from the old chain, thus it holds + random position. By the chance this position is equal to position + for the first element in the new chain. That means updated record + is the only record in the new chain. + */ + if (empty != idx) { + /* + Record was moved while unlinking it from the old chain. + Copy data to a new position. + */ + data[empty] = org_link; + } + data[empty].next = NO_RECORD; + data[empty].hash_nr = hash_nr; + DBUG_RETURN(0); + } + pos = data + new_index; + new_pos_index = my_hash_rec_mask(pos, blength, records); + if (new_index != new_pos_index) { /* Other record in wrong position */ + data[empty] = *pos; + movelink(data, new_index, new_pos_index, (uint)empty); + org_link.next = NO_RECORD; + data[new_index] = org_link; + data[new_index].hash_nr = hash_nr; + } else { /* Link in chain at right position */ + org_link.next = data[new_index].next; + data[empty] = org_link; + data[empty].hash_nr = hash_nr; + data[new_index].next = (uint)empty; + } + DBUG_RETURN(0); +} + +uchar *my_hash_element(const HASH *hash, size_t idx) { + if (idx < hash->records) + return dynamic_element(&hash->array, idx, HASH_LINK *)->data; + return 0; +} + +/* + Replace old row with new row. This should only be used when key + isn't changed +*/ + +void my_hash_replace(HASH *hash, HASH_SEARCH_STATE *current_record, + uchar *new_row) { + if (*current_record != NO_RECORD) /* Safety */ + dynamic_element(&hash->array, *current_record, HASH_LINK *)->data = new_row; +} + +/** + Iterate over all elements in hash and call function with the element + + @param hash hash array + @param action function to call for each argument + @param argument second argument for call to action + + @notes + If one of functions calls returns 1 then the iteration aborts + + @retval 0 ok + @retval 1 iteration aborted becasue action returned 1 +*/ + +my_bool my_hash_iterate(HASH *hash, my_hash_walk_action action, + void *argument) { + uint records, i; + + records = hash->records; + + for (i = 0; i < records; i++) { + if ((*action)(dynamic_element(&hash->array, i, HASH_LINK *)->data, + argument)) + return 1; + } + return 0; +} + +#if !defined(DBUG_OFF) || defined(MAIN) + +my_bool my_hash_check(HASH *hash) { + int error; + uint i, rec_link, found, max_links, seek, links, idx; + uint records; + size_t blength; + HASH_LINK *data, *hash_info; + + records = hash->records; + blength = hash->blength; + data = dynamic_element(&hash->array, 0, HASH_LINK *); + error = 0; + + for (i = found = max_links = seek = 0; i < records; i++) { + size_t length; + uchar *key = (uchar *)my_hash_key(hash, data[i].data, &length, 0); + if (data[i].hash_nr != hash->hash_function(hash->charset, key, length)) { + DBUG_PRINT("error", ("record at %d has wrong hash", i)); + error = 1; + } + + if (my_hash_rec_mask(data + i, blength, records) == i) { + found++; + seek++; + links = 1; + for (idx = data[i].next; idx != NO_RECORD && found < records + 1; + idx = hash_info->next) { + if (idx >= records) { + DBUG_PRINT( + "error", + ("Found pointer outside array to %d from link starting at %d", + idx, i)); + error = 1; + } + hash_info = data + idx; + seek += ++links; + if ((rec_link = my_hash_rec_mask(hash_info, blength, records)) != i) { + DBUG_PRINT("error", ("Record in wrong link at %d: Start %d " + "Record:%p Record-link %d", + idx, i, hash_info->data, rec_link)); + error = 1; + } else + found++; + } + if (links > max_links) max_links = links; + } + } + if (found != records) { + DBUG_PRINT("error", ("Found %u of %u records", found, records)); + error = 1; + } + if (records) + DBUG_PRINT("info", + ("records: %u seeks: %d max links: %d hitrate: %.2f", + records, seek, max_links, (float)seek / (float)records)); + assert(error == 0); + return error; +} +#endif + +#ifdef MAIN + +#define RECORDS 1000 + +const uchar *test_get_key(const void *data, size_t *length, + my_bool not_used __attribute__((unused))) { + *length = 2; + return data; +} + +int main(int argc __attribute__((unused)), + char **argv __attribute__((unused))) { + uchar records[RECORDS][2], copy[2]; + HASH hash_test; + uint i; + MY_INIT(argv[0]); + DBUG_PUSH("d:t:O,/tmp/test_hash.trace"); + + printf("my_hash_init\n"); + if (my_hash_init2(PSI_INSTRUMENT_ME, &hash_test, 100, &my_charset_bin, 20, 0, + 0, test_get_key, 0, 0, HASH_UNIQUE)) { + fprintf(stderr, "hash init failed\n"); + exit(1); + } + + printf("my_hash_insert\n"); + for (i = 0; i < RECORDS; i++) { + int2store(records[i], i); + my_hash_insert(&hash_test, records[i]); + my_hash_check(&hash_test); + } + printf("my_hash_update\n"); + for (i = 0; i < RECORDS; i += 2) { + memcpy(copy, records[i], 2); + int2store(records[i], i + RECORDS); + if (my_hash_update(&hash_test, records[i], copy, 2)) { + fprintf(stderr, "hash update failed\n"); + exit(1); + } + my_hash_check(&hash_test); + } + printf("my_hash_delete\n"); + for (i = 0; i < RECORDS; i++) { + if (my_hash_delete(&hash_test, records[i])) { + fprintf(stderr, "hash delete failed\n"); + exit(1); + } + my_hash_check(&hash_test); + } + my_hash_free(&hash_test); + printf("ok\n"); + my_end(MY_CHECK_ERROR); + return (0); +} +#endif /* MAIN */ diff --git a/router/src/routing/src/sql_lexer.cc b/router/src/routing/src/sql_lexer.cc index 1f4d16d0b7d..2b17b6bdb30 100644 --- a/router/src/routing/src/sql_lexer.cc +++ b/router/src/routing/src/sql_lexer.cc @@ -1071,7 +1071,7 @@ static int lex_one_token(Lexer_yystype *yylval, THD *thd) { ulong version; version = strtol(version_str, nullptr, 10); - if (version <= MYSQL_VERSION_ID) { + if (version <= MYSQL_VERSION_ID || version >= RDS_COMMENT_VERSION) { /* Accept 'M' 'm' 'm' 'd' 'd' */ lip->yySkipn(5); /* Expand the content of the special comment as real code */ diff --git a/share/messages_to_clients.txt b/share/messages_to_clients.txt index 0c2c9b273f5..5674576a4b0 100644 --- a/share/messages_to_clients.txt +++ b/share/messages_to_clients.txt @@ -10044,6 +10044,27 @@ ER_ACCESS_DENIED_DURING_DUCKDB_CONVERT ER_DUCKDB_DATA_IMPORT_MODE eng "[DuckDB] Data import mode: %s." +ER_VECTOR_DISABLED + eng "Creating vector columns or indexes is disabled." + +ER_DATA_INCOMPATIBLE_WITH_VECTOR + eng "Value of type '%.16s, size: %zu' cannot be converted to 'vector(%zu)' type." + +ER_TO_VECTOR_CONVERSION + eng "Data cannot be converted to a valid vector: '%.*s'" + +ER_VEC_DISTANCE_TYPE + eng "Cannot determine distance type for VEC_DISTANCE, index is not found" + +ER_VECTOR_BINARY_FORMAT_INVALID + eng "Invalid binary vector format. Must use IEEE standard float representation in little-endian format. Use VEC_FromText() to generate it." + +ER_VECTOR_INDEX_USAGE + eng "Incorrect usage of vector index: %s" + +ER_VECTOR_INDEX_FAILED + eng "%s vector index `%s` in `%s`.`%s` (aux_tab: %s) failed: %s" + # # End of RDS error message. # diff --git a/sql-common/json_error_handler.cc b/sql-common/json_error_handler.cc index 5643e82dc18..829e1c04015 100644 --- a/sql-common/json_error_handler.cc +++ b/sql-common/json_error_handler.cc @@ -29,6 +29,9 @@ void JsonParseDefaultErrorHandler::operator()(const char *parse_err, size_t err_offset) const { + if (m_caller_is_duckdb) { + throw std::exception(); + } my_error(ER_INVALID_JSON_TEXT_IN_PARAM, MYF(0), m_arg_idx + 1, m_func_name, parse_err, err_offset, ""); } @@ -36,3 +39,7 @@ void JsonParseDefaultErrorHandler::operator()(const char *parse_err, void JsonDocumentDefaultDepthHandler() { my_error(ER_JSON_DOCUMENT_TOO_DEEP, MYF(0)); } + +void JsonDocumentDefaultDepthHandlerDuckDB() { + throw std::exception(); +} diff --git a/sql-common/json_error_handler.h b/sql-common/json_error_handler.h index 919809b1f4c..31dc81c63dc 100644 --- a/sql-common/json_error_handler.h +++ b/sql-common/json_error_handler.h @@ -39,6 +39,7 @@ class JsonParseDefaultErrorHandler { : m_func_name(func_name), m_arg_idx(arg_idx) {} void operator()(const char *parse_err, size_t err_offset) const; + bool m_caller_is_duckdb = false; private: const char *m_func_name; @@ -46,6 +47,7 @@ class JsonParseDefaultErrorHandler { }; void JsonDocumentDefaultDepthHandler(); +void JsonDocumentDefaultDepthHandlerDuckDB(); #endif // MYSQL_SERVER #endif // JSON_ERROR_HANDLER_INCLUDED diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 4c330d8f85f..af91b746be8 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -119,6 +119,7 @@ ADD_SUBDIRECTORY(protobuf) SET(DD_SOURCES dd/collection.cc dd/dd_event.cc + dd/dd_minor_upgrade.cc dd/dd_resource_group.cc dd/dd_routine.cc dd/dd_schema.cc @@ -819,6 +820,10 @@ SET(SQL_SOURCE srv_session.cc srv_session_info_service.cc srv_session_service.cc + vidx/vidx_func.cc + vidx/vidx_field.cc + vidx/vidx_index.cc + vidx/vidx_hnsw.cc ) IF(NOT HAVE_SETNS) @@ -953,6 +958,14 @@ IF(HAVE_NO_BUILTIN_MEMCMP) ) ENDIF() +# For vidx +IF(MY_COMPILER_IS_GNU AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7 + AND (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")) + IF(NOT HAVE_IMMINTRIN_H) + MESSAGE(FATAL_ERROR "HAVE_IMMINTRIN_H is not set to 1. SIMD optimizations may not be available.") + ENDIF() +ENDIF() + ADD_STATIC_LIBRARY(sql_main ${SQL_SOURCE} LINK_LIBRARIES ext::icu ext::xxhash) ADD_DEPENDENCIES(sql_main GenServerSource) ADD_DEPENDENCIES(sql_main GenDigestServerSource) diff --git a/sql/create_field.cc b/sql/create_field.cc index ee8d6a02135..ea3f59712cc 100644 --- a/sql/create_field.cc +++ b/sql/create_field.cc @@ -30,6 +30,7 @@ #include "sql/sql_class.h" #include "sql_string.h" #include "template_utils.h" +#include "vidx/vidx_index.h" // vidx::feature_disabled #include #include @@ -67,6 +68,7 @@ Create_field::Create_field(Field *old_field, Field *orig_field) auto_flags(old_field->auto_flags), charset(old_field->charset()), // May be NULL ptr is_explicit_collation(false), + is_vector(old_field->is_vector()), geom_type(Field::GEOM_GEOMETRY), field(old_field), is_nullable(old_field->is_nullable()), @@ -199,7 +201,7 @@ bool Create_field::init( bool has_explicit_collation, uint fld_geom_type, Value_generator *fld_gcol_info, Value_generator *fld_default_val_expr, std::optional srid, dd::Column::enum_hidden_type hidden, - bool is_array_arg) { + bool is_array_arg, bool is_vector_type) { uint sign_len, allowed_type_modifier = 0; ulong max_field_charlength = MAX_FIELD_CHARLENGTH; @@ -211,6 +213,12 @@ bool Create_field::init( field_name = fld_name; flags = fld_type_modifier; is_explicit_collation = (fld_charset != nullptr); + is_vector = (is_vector_type && fld_type == MYSQL_TYPE_VARCHAR); + + if (is_vector && vidx::feature_disabled) { + my_error(ER_VECTOR_DISABLED, MYF(0)); + return true; + } if (!has_explicit_collation && fld_charset == &my_charset_utf8mb4_0900_ai_ci) charset = thd->variables.default_collation_for_utf8mb4; diff --git a/sql/create_field.h b/sql/create_field.h index 16a6d14427c..f2595ace2d8 100644 --- a/sql/create_field.h +++ b/sql/create_field.h @@ -124,6 +124,7 @@ class Create_field { List interval_list; const CHARSET_INFO *charset; bool is_explicit_collation; // User exeplicitly provided charset ? + bool is_vector = false; Field::geometry_type geom_type; Field *field; // For alter table @@ -224,7 +225,8 @@ class Create_field { bool has_explicit_collation, uint uint_geom_type, Value_generator *gcol_info, Value_generator *default_val_expr, std::optional srid, - dd::Column::enum_hidden_type hidden, bool is_array = false); + dd::Column::enum_hidden_type hidden, bool is_array = false, + bool is_vector_type = false); ha_storage_media field_storage_type() const { return (ha_storage_media)((flags >> FIELD_FLAGS_STORAGE_MEDIA) & 3); diff --git a/sql/dd/dd_minor_upgrade.cc b/sql/dd/dd_minor_upgrade.cc new file mode 100644 index 00000000000..68c5caafc88 --- /dev/null +++ b/sql/dd/dd_minor_upgrade.cc @@ -0,0 +1,83 @@ +/***************************************************************************** + +Copyright (c) 2026, Alibaba and/or its affiliates. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*****************************************************************************/ + +#include "my_inttypes.h" +#include "sql/dd/impl/tables/dd_properties.h" +#include "sql/dd/string_type.h" +#include "sql/sql_class.h" + +#include "sql/dd/dd_minor_upgrade.h" + +namespace dd { +/** + Version for server I_S tables and System Views. + PLS increase it if update server I_S tables or System Views. + + + Historical I_S version number published: + + 1. Fixed VECTOR column DATA_TYPE in INFORMATION_SCHEMA views + --------------------------- + Strip versioned comment markers from VECTOR column DATA_TYPE in + INFORMATION_SCHEMA.COLUMNS + INFORMATION_SCHEMA.PARAMETERS + INFORMATION_SCHEMA.ROUTINES + to correctly show the underlying storage type. +*/ +static const uint EXTRA_IS_DD_VERSION = 1; + +static const String_type EXTRA_IS_DD_VERSION_STRING("EXTRA_IS_VERSION"); + +// Get the singleton instance +Minor_upgrade_ctx *Minor_upgrade_ctx::instance() { + static Minor_upgrade_ctx ctx; + return &ctx; +} + +// Get compiled extra I_S version +uint Minor_upgrade_ctx::get_target_extra_I_S_version() { + return EXTRA_IS_DD_VERSION; +} + +// Get persisted extra I_S version +uint Minor_upgrade_ctx::get_actual_extra_I_S_version(THD *thd) { + bool exists = false; + uint version = 0; + bool error MY_ATTRIBUTE((unused)) = tables::DD_properties::instance().get( + thd, EXTRA_IS_DD_VERSION_STRING.c_str(), &version, &exists); + if (error || !exists) + return UNKNOWN_EXTRA_VERSION; + else + return version; +} + +// Persist extra I_S version into dd_properties +uint Minor_upgrade_ctx::set_extra_I_S_version(THD *thd, uint version) { + return tables::DD_properties::instance().set( + thd, EXTRA_IS_DD_VERSION_STRING.c_str(), version); +} + +} // namespace dd diff --git a/sql/dd/dd_minor_upgrade.h b/sql/dd/dd_minor_upgrade.h new file mode 100644 index 00000000000..bf04d2b4765 --- /dev/null +++ b/sql/dd/dd_minor_upgrade.h @@ -0,0 +1,64 @@ +/***************************************************************************** + +Copyright (c) 2026, Alibaba and/or its affiliates. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*****************************************************************************/ + +#ifndef SQL_DD_DD_MINOR_UPGRADE_H_INCLUDED +#define SQL_DD_DD_MINOR_UPGRADE_H_INCLUDED + +#include "my_inttypes.h" + +class THD; + +/** + Minor Upgrade is designed to upgrade I_S System Views without changing + the upstream IS_DD_VERSION. + + This strategy adds EXTRA_IS_VERSION into the DD_properties table. + Increase EXTRA_IS_DD_VERSION when modifying I_S system view definitions. + + DD restart will reconstruct I_S system views if the persisted version + does not match the compiled target version. +*/ +namespace dd { + +static const uint UNKNOWN_EXTRA_VERSION = -1; + +class Minor_upgrade_ctx { + public: + Minor_upgrade_ctx() {} + ~Minor_upgrade_ctx() {} + + static Minor_upgrade_ctx *instance(); + + static uint get_target_extra_I_S_version(); + + virtual uint get_actual_extra_I_S_version(THD *thd); + + uint set_extra_I_S_version(THD *thd, uint version); +}; + +} // namespace dd + +#endif diff --git a/sql/dd/dd_routine.cc b/sql/dd/dd_routine.cc index ee19bf94c13..c2e7428b684 100644 --- a/sql/dd/dd_routine.cc +++ b/sql/dd/dd_routine.cc @@ -194,6 +194,13 @@ static void fill_parameter_info_from_field(THD *thd, Create_field *field, param_options->set("geom_type", field->geom_type); } + // Store vector type + if (field->is_vector) { + assert(field->sql_type == MYSQL_TYPE_VARCHAR); + Properties *param_options = ¶m->options(); + param_options->set("vector_type", true); + } + // Set elements of enum or set data type. if (field->interval) { assert(field->sql_type == MYSQL_TYPE_ENUM || diff --git a/sql/dd/dd_table.cc b/sql/dd/dd_table.cc index 7979b7e34f0..8ce2f58e8f9 100644 --- a/sql/dd/dd_table.cc +++ b/sql/dd/dd_table.cc @@ -107,6 +107,9 @@ #include "sql/thd_raii.h" #include "sql_string.h" #include "typelib.h" +#ifndef NDEBUG +#include "vidx/vidx_index.h" +#endif /* !NDEBUG */ namespace dd { @@ -694,6 +697,12 @@ bool fill_dd_columns_from_create_fields(THD *thd, dd::Abstract_table *tab_obj, col_options->set("geom_type", field.geom_type); } + // Store vector type + if (field.is_vector) { + assert(field.sql_type == MYSQL_TYPE_VARCHAR); + col_options->set("vector_type", true); + } + // Field storage media and column format options if (field.field_storage_type() != HA_SM_DEFAULT) col_options->set("storage", @@ -1040,6 +1049,9 @@ static void fill_dd_indexes_from_keyinfo( const KEY *primary_key_info = nullptr; for (int key_nr = 1; key != end; ++key, ++key_nr) { + if (key->flags & HA_VECTOR) { + continue; + } // // Add new DD index // @@ -2436,6 +2448,7 @@ std::unique_ptr create_tmp_table( check_cons_spec, file)) return nullptr; + assert(!vidx::dd_table_has_hlindexes(tab_obj.get())); return tab_obj; } @@ -2648,6 +2661,9 @@ bool recreate_table(THD *thd, const char *schema_name, const char *table_name) { char path[FN_REFLEN + 1]; build_table_filename(path, sizeof(path) - 1, schema_name, table_name, "", 0); + /* vector index is only supported for InnoDB tables. */ + assert(!vidx::dd_table_has_hlindexes(table_def)); + // Attempt to reconstruct the table return ha_create_table(thd, path, schema_name, table_name, &create_info, true, false, table_def); diff --git a/sql/dd/impl/cache/dictionary_client.cc b/sql/dd/impl/cache/dictionary_client.cc index c02d6d560f8..1438b362ac6 100644 --- a/sql/dd/impl/cache/dictionary_client.cc +++ b/sql/dd/impl/cache/dictionary_client.cc @@ -2070,6 +2070,9 @@ bool Dictionary_client::fetch_schema_table_names_by_engine( // specific tables. bool Dictionary_client::fetch_schema_table_names_not_hidden_by_se( const Schema *schema, std::vector *names) const { + static_assert(dd::Abstract_table::HT_HIDDEN_HLINDEX == + dd::Abstract_table::HT_HIDDEN_SE); + auto fetch_criteria = [&](Raw_record *r) -> bool { return static_cast( r->read_int(dd::tables::Tables::FIELD_HIDDEN)) != diff --git a/sql/dd/impl/system_views/columns.cc b/sql/dd/impl/system_views/columns.cc index 04f46baed22..6f81114656a 100644 --- a/sql/dd/impl/system_views/columns.cc +++ b/sql/dd/impl/system_views/columns.cc @@ -52,9 +52,17 @@ Columns::Columns() { "col.default_value_utf8"); m_target_def.add_field(FIELD_IS_NULLABLE, "IS_NULLABLE", "IF (col.is_nullable = 1, 'YES','NO')"); - m_target_def.add_field( - FIELD_DATA_TYPE, "DATA_TYPE", - "SUBSTRING_INDEX(SUBSTRING_INDEX(col.column_type_utf8, '(', 1),' ', 1)"); + /* + VECTOR columns are stored as '99999 vector(N) varbinary(4N)' in + the data dictionary for backward compatibility with downstream applications + that do not support the VECTOR type. The versioned comment is invisible to + older MySQL versions, so they only see a plain varbinary column. + Strip the comment prefix (everything up to and including '* /') to expose + the underlying storage type (e.g. 'varbinary') as DATA_TYPE. + */ + m_target_def.add_field(FIELD_DATA_TYPE, "DATA_TYPE", + "SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(col." + "column_type_utf8, '*/ ', -1), '(', 1),' ', 1)"); m_target_def.add_field( FIELD_CHARACTER_MAXIMUM_LENGTH, "CHARACTER_MAXIMUM_LENGTH", "INTERNAL_DD_CHAR_LENGTH(col.type, col.char_length, coll.name, 0)"); diff --git a/sql/dd/impl/system_views/parameters.cc b/sql/dd/impl/system_views/parameters.cc index 2cdc7ef8bea..43462f4dcd7 100644 --- a/sql/dd/impl/system_views/parameters.cc +++ b/sql/dd/impl/system_views/parameters.cc @@ -48,9 +48,17 @@ Parameters::Parameters() { m_target_def.add_field(FIELD_PARAMETER_NAME, "PARAMETER_NAME", "IF (rtn.type = 'FUNCTION' AND prm.ordinal_position = " "1, NULL, prm.name)"); + /* + VECTOR routine parameters are stored with a versioned comment prefix + (e.g. '99999 vector(N) varbinary(4N)') in the data dictionary + for backward compatibility with downstream applications that do not + support the VECTOR type. Strip the comment prefix to expose the + underlying storage type (e.g. 'varbinary') as DATA_TYPE. + */ m_target_def.add_field( FIELD_DATA_TYPE, "DATA_TYPE", - "SUBSTRING_INDEX(SUBSTRING_INDEX(prm.data_type_utf8, '(', 1), ' ', 1)"); + "SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(" + "prm.data_type_utf8, '*/ ', -1), '(', 1), ' ', 1)"); m_target_def.add_field( FIELD_CHARACTER_MAXIMUM_LENGTH, "CHARACTER_MAXIMUM_LENGTH", "INTERNAL_DD_CHAR_LENGTH(prm.data_type, prm.char_length, col.name, 0)"); diff --git a/sql/dd/impl/system_views/routines.cc b/sql/dd/impl/system_views/routines.cc index e81939e510f..7d828cac256 100644 --- a/sql/dd/impl/system_views/routines.cc +++ b/sql/dd/impl/system_views/routines.cc @@ -41,10 +41,18 @@ Routines::Routines() { "sch.name" + m_target_def.fs_name_collation()); m_target_def.add_field(FIELD_ROUTINE_NAME, "ROUTINE_NAME", "rtn.name"); m_target_def.add_field(FIELD_ROUTINE_TYPE, "ROUTINE_TYPE", "rtn.type"); + /* + VECTOR function return types are stored with a versioned comment prefix + (e.g. '99999 vector(N) varbinary(4N)') in the data dictionary + for backward compatibility with downstream applications that do not + support the VECTOR type. Strip the comment prefix to expose the + underlying storage type (e.g. 'varbinary') as DATA_TYPE. + */ m_target_def.add_field(FIELD_DATA_TYPE, "DATA_TYPE", "IF(rtn.type = 'PROCEDURE', '', " - " SUBSTRING_INDEX(SUBSTRING_INDEX(" - " rtn.result_data_type_utf8, '(', 1), ' ', 1))"); + " SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(" + " rtn.result_data_type_utf8, '*/ ', -1)," + " '(', 1), ' ', 1))"); m_target_def.add_field(FIELD_CHARACTER_MAXIMUM_LENGTH, "CHARACTER_MAXIMUM_LENGTH", "INTERNAL_DD_CHAR_LENGTH(rtn.result_data_type," diff --git a/sql/dd/impl/tables/dd_properties.cc b/sql/dd/impl/tables/dd_properties.cc index 309d5bc52ff..12d428c02b8 100644 --- a/sql/dd/impl/tables/dd_properties.cc +++ b/sql/dd/impl/tables/dd_properties.cc @@ -123,7 +123,8 @@ DD_properties::DD_properties() : m_properties() { {"MYSQLD_VERSION_UPGRADED", Property_type::UNSIGNED_INT_32}, {"MYSQL_VERSION_STABILITY", Property_type::CHARACTER_STRING}, {"SERVER_DOWNGRADE_THRESHOLD", Property_type::UNSIGNED_INT_32}, - {"SERVER_UPGRADE_THRESHOLD", Property_type::UNSIGNED_INT_32}}; + {"SERVER_UPGRADE_THRESHOLD", Property_type::UNSIGNED_INT_32}, + {"EXTRA_IS_VERSION", Property_type::UNSIGNED_INT_32}}; } // Read all properties from disk and populate the cache. diff --git a/sql/dd/impl/types/abstract_table_impl.cc b/sql/dd/impl/types/abstract_table_impl.cc index 3ef02eae691..8fbea705639 100644 --- a/sql/dd/impl/types/abstract_table_impl.cc +++ b/sql/dd/impl/types/abstract_table_impl.cc @@ -89,7 +89,11 @@ static const std::set default_valid_option_keys = { "tablespace", "timestamp", "view_valid", - "gipk"}; + "gipk", + "__hlindexes__", + "__vector_column__", + "__vector_m__", + "__vector_distance__"}; /////////////////////////////////////////////////////////////////////////// // Abstract_table_impl implementation. diff --git a/sql/dd/impl/types/column_impl.cc b/sql/dd/impl/types/column_impl.cc index 9996a83a6ea..22d8de9e746 100644 --- a/sql/dd/impl/types/column_impl.cc +++ b/sql/dd/impl/types/column_impl.cc @@ -65,9 +65,11 @@ class Sdi_rcontext; class Sdi_wcontext; static const std::set default_valid_option_keys = { - "column_format", "geom_type", "interval_count", - "not_secondary", "storage", "treat_bit_as_char", - "is_array", "gipk" /* generated implicit primary key column */}; + "column_format", "geom_type", + "interval_count", "not_secondary", + "storage", "treat_bit_as_char", + "is_array", "gipk" /* generated implicit primary key column */, + "vector_type" /* vector column */}; /////////////////////////////////////////////////////////////////////////// // Column_impl implementation. diff --git a/sql/dd/impl/types/parameter_impl.cc b/sql/dd/impl/types/parameter_impl.cc index ec2d765e939..e8374f40cba 100644 --- a/sql/dd/impl/types/parameter_impl.cc +++ b/sql/dd/impl/types/parameter_impl.cc @@ -49,7 +49,8 @@ using dd::tables::Parameters; namespace dd { -static const std::set default_valid_option_keys = {"geom_type"}; +static const std::set default_valid_option_keys = {"geom_type", + "vector_type"}; /////////////////////////////////////////////////////////////////////////// // Parameter_impl implementation. diff --git a/sql/dd/info_schema/metadata.cc b/sql/dd/info_schema/metadata.cc index 58f541916e4..e96598d1b72 100644 --- a/sql/dd/info_schema/metadata.cc +++ b/sql/dd/info_schema/metadata.cc @@ -51,6 +51,7 @@ #include "mysql/components/services/log_shared.h" #include "mysql/plugin.h" #include "sql/dd/cache/dictionary_client.h" // dd::cache::Dictionary_client +#include "sql/dd/dd_minor_upgrade.h" // dd::Minor_upgrade_ctx #include "sql/dd/dd_schema.h" // dd::Schema_MDL_locker #include "sql/dd/dd_table.h" // dd::get_sql_type_by_field_info #include "sql/dd/dd_utility.h" // check_if_server_ddse_readonly @@ -470,6 +471,12 @@ bool create_system_views(THD *thd, bool is_non_dd_based) { d->get_actual_I_S_version(thd)); error = d->set_I_S_version(thd, d->get_target_I_S_version()); dd::bootstrap::DD_bootstrap_ctx::instance().set_I_S_upgrade_done(); + + if (!error) { + dd::Minor_upgrade_ctx *upgrade_ctx = dd::Minor_upgrade_ctx::instance(); + error = upgrade_ctx->set_extra_I_S_version( + thd, upgrade_ctx->get_target_extra_I_S_version()); + } } // Restore the original character set. @@ -504,11 +511,16 @@ bool update_server_I_S_metadata(THD *thd) { // Stop if I_S version is same and no DD upgrade was done. uint actual_version = d->get_actual_I_S_version(thd); + // Extra I_S version check + dd::Minor_upgrade_ctx *upgrade_ctx = dd::Minor_upgrade_ctx::instance(); + uint extra_version = upgrade_ctx->get_actual_extra_I_S_version(thd); + // Testing to make sure we update plugins when version changes. DBUG_EXECUTE_IF("test_i_s_metadata_version", { actual_version = UNKNOWN_PLUGIN_VERSION; }); if (d->get_target_I_S_version() == actual_version && + upgrade_ctx->get_target_extra_I_S_version() == extra_version && !dd::bootstrap::DD_bootstrap_ctx::instance().dd_upgrade_done()) return false; diff --git a/sql/dd/types/abstract_table.h b/sql/dd/types/abstract_table.h index 8246fbf6461..905c3758166 100644 --- a/sql/dd/types/abstract_table.h +++ b/sql/dd/types/abstract_table.h @@ -110,6 +110,10 @@ class Abstract_table : virtual public Entity_object { For example, InnoDB's auxiliary table for FTS. */ HT_HIDDEN_SE, + /* + Set hlindex table invisible. + */ + HT_HIDDEN_HLINDEX = HT_HIDDEN_SE, /* Hidden. Temporary table created by ALTER TABLE implementation. */ diff --git a/sql/dd/upgrade_57/table.cc b/sql/dd/upgrade_57/table.cc index 31677c9290d..297a0b3df29 100644 --- a/sql/dd/upgrade_57/table.cc +++ b/sql/dd/upgrade_57/table.cc @@ -101,6 +101,9 @@ #include "sql/trigger_def.h" #include "sql_string.h" #include "thr_lock.h" +#ifndef NDEBUG +#include "vidx/vidx_index.h" +#endif /* !NDEBUG */ class Sroutine_hash_entry; @@ -1616,6 +1619,7 @@ static bool migrate_table_to_dd(THD *thd, const String_type &schema_name, key_info = share.key_info; for (i = 0; i < share.keys; i++, key_info++) { key_info->is_visible = true; + assert(!(key_info->flags & HA_VECTOR)); /* Fulltext and Spatical indexes will get fixed by mysql_prepare_create_table() @@ -1793,6 +1797,7 @@ static bool migrate_table_to_dd(THD *thd, const String_type &schema_name, thd, *sch_obj, to_table_name, &create_info, alter_info.create_list, key_info_buffer, key_count, Alter_info::ENABLE, fk_key_info_buffer, fk_number, &cc_spec_list_unused, table.file); + assert(!vidx::dd_table_has_hlindexes(table_def.get())); // Check for usage of prefix key index in PARTITION BY KEY() function. dd::warn_on_deprecated_prefix_key_partition( diff --git a/sql/dd_sp.cc b/sql/dd_sp.cc index 9a50686cdc8..f22f8248838 100644 --- a/sql/dd_sp.cc +++ b/sql/dd_sp.cc @@ -94,7 +94,8 @@ void prepare_sp_chistics_from_dd_routine(const dd::Routine *routine, } static Field *make_field(const dd::Parameter ¶m, TABLE_SHARE *share, - Field::geometry_type geom_type, TYPELIB *interval) { + Field::geometry_type geom_type, TYPELIB *interval, + bool is_vector = false) { // Decimals uint numeric_scale = 0; if (param.data_type() == dd::enum_column_types::DECIMAL || @@ -109,7 +110,8 @@ static Field *make_field(const dd::Parameter ¶m, TABLE_SHARE *share, 0, dd_get_old_field_type(param.data_type()), dd_get_mysql_charset(param.collation_id()), geom_type, Field::NONE, interval, "", false, param.is_zerofill(), - param.is_unsigned(), numeric_scale, false, 0, {}, false); + param.is_unsigned(), numeric_scale, false, 0, {}, false, + is_vector); } /** @@ -165,6 +167,13 @@ static void prepare_type_string_from_dd_param(THD *thd, geom_type = static_cast(sub_type); } + // Vector options + bool is_vector = false; + if (param->data_type() == dd::enum_column_types::VARCHAR && + param->options().exists("vector")) { + param->options().get("vector", &is_vector); + } + // Get type in string format. TABLE table; TABLE_SHARE share; @@ -172,7 +181,7 @@ static void prepare_type_string_from_dd_param(THD *thd, table.s = &share; unique_ptr_destroy_only field( - make_field(*param, table.s, geom_type, interval)); + make_field(*param, table.s, geom_type, interval, is_vector)); field->init(&table); field->sql_type(*type_str); diff --git a/sql/dd_table_share.cc b/sql/dd_table_share.cc index 062d3db9f57..eb81dbbd2ba 100644 --- a/sql/dd_table_share.cc +++ b/sql/dd_table_share.cc @@ -90,6 +90,7 @@ #include "sql/table.h" #include "sql/thd_raii.h" #include "typelib.h" +#include "vidx/vidx_index.h" namespace histograms { class Histogram; @@ -264,8 +265,8 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share, handler *handler_file = nullptr; // Mark 'system' tables (tables with one row) to help the Optimizer. - share->system = - ((share->max_rows == 1) && (share->min_rows == 1) && (share->keys == 0)); + share->system = ((share->max_rows == 1) && (share->min_rows == 1) && + (share->total_keys == 0)); bool use_extended_sk = ha_check_storage_engine_flag( share->db_type(), HTON_SUPPORTS_EXTENDED_KEYS); @@ -290,12 +291,11 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share, share->db_low_byte_first = handler_file->low_byte_first(); /* Fix key->name and key_part->field */ - if (share->keys) { + if (share->total_keys) { KEY *keyinfo; KEY_PART_INFO *key_part; - uint primary_key = (uint)(find_type(primary_key_name, &share->keynames, - FIND_TYPE_NO_PREFIX) - - 1); + uint primary_key = (uint)( + find_type(primary_key_name, &share->keynames, FIND_TYPE_NO_PREFIX) - 1); longlong ha_option = handler_file->ha_table_flags(); keyinfo = share->key_info; key_part = keyinfo->key_part; @@ -303,12 +303,14 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share, dd::Table::Index_collection::const_iterator idx_it( table_def->indexes().begin()); - for (uint key = 0; key < share->keys; key++, keyinfo++) { + for (uint key = 0; key < share->total_keys; key++, keyinfo++) { + assert(((keyinfo->flags & HA_VECTOR) > 0) == (key >= share->keys)); + /* Skip hidden dd::Index objects so idx_it is in sync with key index and keyinfo pointer. */ - while ((*idx_it)->is_hidden()) { + while (key < share->keys && (*idx_it)->is_hidden()) { ++idx_it; continue; } @@ -316,13 +318,15 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share, uint usable_parts = 0; keyinfo->name = share->keynames.type_names[key]; - /* Check that fulltext and spatial keys have correct algorithm set. */ + /* Check that vector, fulltext and spatial keys have correct algorithm + set. */ assert(!(share->key_info[key].flags & HA_FULLTEXT) || share->key_info[key].algorithm == HA_KEY_ALG_FULLTEXT); assert(!(share->key_info[key].flags & HA_SPATIAL) || share->key_info[key].algorithm == HA_KEY_ALG_RTREE); - if (primary_key >= MAX_KEY && (keyinfo->flags & HA_NOSAME)) { + if (primary_key >= MAX_KEY && (keyinfo->flags & HA_NOSAME) && + !(keyinfo->flags & HA_VECTOR)) { /* If the UNIQUE key doesn't have NULL columns and is not a part key declare this as a primary key. @@ -352,7 +356,7 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share, Skip hidden Index_element objects so idx_el_it is in sync with i and key_part pointer. */ - while ((*idx_el_it)->is_hidden()) { + while (key < share->keys && (*idx_el_it)->is_hidden()) { ++idx_el_it; continue; } @@ -429,9 +433,11 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share, Check that dd::Index_element::is_prefix() used by SEs works in the same way as code which sets HA_PART_KEY_SEG flag. */ - assert((*idx_el_it)->is_prefix() == - static_cast(key_part->key_part_flag & HA_PART_KEY_SEG)); - ++idx_el_it; + if (key < share->keys) { + assert((*idx_el_it)->is_prefix() == + static_cast(key_part->key_part_flag & HA_PART_KEY_SEG)); + ++idx_el_it; + } } /* @@ -463,8 +469,11 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share, (ha_option & HA_ANY_INDEX_MAY_BE_UNIQUE)) share->max_unique_length = std::max(share->max_unique_length, keyinfo->key_length); - - ++idx_it; + if (key < share->keys - 1) { + /* Iterate idx_it only if next key is not vector, because the vector + key has no related idx in dd. */ + ++idx_it; + } } if (primary_key < MAX_KEY && (share->keys_in_use.is_set(primary_key))) { share->primary_key = primary_key; @@ -490,7 +499,7 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share, Field *reg_field = *share->found_next_number_field; /* Check that the auto-increment column is the first column of some key. */ if ((int)(share->next_number_index = (uint)find_ref_key( - share->key_info, share->keys, share->default_values, + share->key_info, share->total_keys, share->default_values, reg_field, &share->next_number_key_offset, &share->next_number_keypart)) < 0) { my_error(ER_INVALID_DD_OBJECT, MYF(0), share->path.str, @@ -565,6 +574,8 @@ static bool fill_share_from_dd(THD *thd, TABLE_SHARE *share, const dd::Table *tab_obj) { const dd::Properties &table_options = tab_obj->options(); + share->m_se_private_id = tab_obj->se_private_id(); + // Secondary storage engine. if (table_options.exists("secondary_engine")) { table_options.get("secondary_engine", &share->secondary_engine, @@ -877,6 +888,13 @@ static Field *make_field(const dd::Column &col_obj, const CHARSET_INFO *charset, geom_type = static_cast(sub_type); } + // Vector options + bool is_vector = false; + if (field_type == MYSQL_TYPE_VARCHAR && + column_options.exists("vector_type")) { + column_options.get("vector_type", &is_vector); + } + bool treat_bit_as_char = false; if (field_type == MYSQL_TYPE_BIT) { column_options.get("treat_bit_as_char", &treat_bit_as_char); @@ -886,7 +904,7 @@ static Field *make_field(const dd::Column &col_obj, const CHARSET_INFO *charset, field_type, charset, geom_type, auto_flags, interval, name, col_obj.is_nullable(), col_obj.is_zerofill(), col_obj.is_unsigned(), decimals, treat_bit_as_char, 0, - col_obj.srs_id(), col_obj.is_array()); + col_obj.srs_id(), col_obj.is_array(), is_vector); } /** @@ -1402,6 +1420,15 @@ static bool fill_index_from_dd(THD *thd, TABLE_SHARE *share, keyinfo->flags |= HA_USES_PARSER; } + // Vector options + if (idx_options.exists("__vector_distance__")) { + assert(idx_options.exists("__vector_m__")); + idx_options.get("__vector_distance__", &keyinfo->vector_distance); + idx_options.get("__vector_m__", &keyinfo->vector_m); + assert(vidx::validate_index_option_distance(keyinfo->vector_distance)); + assert(vidx::hnsw::validate_index_option_m(keyinfo->vector_m)); + } + // Read comment dd::String_type comment = idx_obj->comment(); keyinfo->comment.length = comment.length(); @@ -1468,13 +1495,14 @@ static bool fill_indexes_from_dd(THD *thd, TABLE_SHARE *share, share->visible_indexes.init(); uint32 primary_key_parts = 0; + dd::String_type child_name; bool use_extended_sk = ha_check_storage_engine_flag( share->db_type(), HTON_SUPPORTS_EXTENDED_KEYS); // Count number of keys and total number of key parts in the table. - assert(share->keys == 0 && share->key_parts == 0); + assert(share->keys == 0 && share->total_keys == 0 && share->key_parts == 0); for (const dd::Index *idx_obj : tab_obj->indexes()) { // Skip hidden indexes @@ -1494,15 +1522,22 @@ static bool fill_indexes_from_dd(THD *thd, TABLE_SHARE *share, if (idx_obj->ordinal_position() == 1) primary_key_parts = key_parts; } + share->total_keys = share->keys; + + if (vidx::dd_table_has_hlindexes(tab_obj)) { + share->total_keys++; + share->key_parts++; + } + // Allocate and fill KEY objects. - if (share->keys) { + if (share->total_keys) { KEY_PART_INFO *key_part; ulong *rec_per_key; rec_per_key_t *rec_per_key_float; uint total_key_parts = share->key_parts; if (use_extended_sk) - total_key_parts += (primary_key_parts * (share->keys - 1)); + total_key_parts += (primary_key_parts * (share->total_keys - 1)); // // Alloc rec_per_key buffer @@ -1522,7 +1557,7 @@ static bool fill_indexes_from_dd(THD *thd, TABLE_SHARE *share, // Alloc buffers to hold keys and key_parts // - if (!(share->key_info = share->mem_root.ArrayAlloc(share->keys))) + if (!(share->key_info = share->mem_root.ArrayAlloc(share->total_keys))) return true; /* purecov: inspected */ if (!(key_part = @@ -1534,12 +1569,13 @@ static bool fill_indexes_from_dd(THD *thd, TABLE_SHARE *share, // if (!(share->keynames.type_names = (const char **)share->mem_root.Alloc( - (share->keys + 1) * sizeof(char *)))) + (share->total_keys + 1) * sizeof(char *)))) return true; /* purecov: inspected */ - memset(share->keynames.type_names, 0, ((share->keys + 1) * sizeof(char *))); + memset(share->keynames.type_names, 0, + ((share->total_keys + 1) * sizeof(char *))); - share->keynames.type_names[share->keys] = nullptr; - share->keynames.count = share->keys; + share->keynames.type_names[share->total_keys] = nullptr; + share->keynames.count = share->total_keys; // In first iteration get all the index_obj, so that we get all // user_defined_key_parts for each key. This is required to properly @@ -1565,13 +1601,20 @@ static bool fill_indexes_from_dd(THD *thd, TABLE_SHARE *share, // Update keyparts now key_nr = 0; do { + assert((key_nr < share->keys) || + (key_nr == share->keys && key_nr == share->total_keys - 1)); + // Assign the key_part_info buffer KEY *keyinfo = &share->key_info[key_nr]; keyinfo->key_part = key_part; keyinfo->set_rec_per_key_array(rec_per_key, rec_per_key_float); keyinfo->set_in_memory_estimate(IN_MEMORY_ESTIMATE_UNKNOWN); - fill_index_elements_from_dd(share, index_at_pos[key_nr], key_nr); + if (key_nr < share->keys) { + fill_index_elements_from_dd(share, index_at_pos[key_nr], key_nr); + } else if (vidx::build_hlindex_key(thd, share, tab_obj, key_nr)) { + return true; + } key_part += keyinfo->user_defined_key_parts; rec_per_key += keyinfo->user_defined_key_parts; @@ -1603,7 +1646,7 @@ static bool fill_indexes_from_dd(THD *thd, TABLE_SHARE *share, } key_nr++; - } while (key_nr < share->keys); + } while (key_nr < share->total_keys); } return (false); diff --git a/sql/duckdb/duckdb_manager.cc b/sql/duckdb/duckdb_manager.cc index 166214e7f28..94b29bf449b 100644 --- a/sql/duckdb/duckdb_manager.cc +++ b/sql/duckdb/duckdb_manager.cc @@ -35,6 +35,8 @@ this program; if not, write to the Free Software Foundation, Inc., #include "sql/duckdb/duckdb_timezone.h" #include "sql/mysqld.h" +#include + namespace myduck { DuckdbManager *DuckdbManager::m_instance = nullptr; @@ -146,4 +148,53 @@ std::shared_ptr DuckdbManager::CreateConnection() { return connection; } +bool DuckdbManager::DataFilesExist() { + char path[FN_REFLEN]; + struct stat st; + + // Check for main database file: duckdb.db + fn_format(path, DUCKDB_FILE_NAME, mysql_real_data_home, "", MYF(0)); + if (stat(path, &st) == 0 && S_ISREG(st.st_mode)) { + return true; + } + + // Check for WAL file: duckdb.db.wal + fn_format(path, DUCKDB_FILE_NAME, mysql_real_data_home, ".wal", MYF(0)); + if (stat(path, &st) == 0 && S_ISREG(st.st_mode)) { + return true; + } + + return false; +} + +bool DuckdbManager::InitializeIfNeeded() { + if (m_instance == nullptr) { + return true; + } + + // Check if we need to initialize DuckDB at startup: + // 1. DuckDB mode is ON, or + // 2. DuckDB data files exist (need crash recovery) + bool need_init = global_mode_on() || DataFilesExist(); + + if (!need_init) { + return false; + } + + if (DataFilesExist()) { + LogErr(INFORMATION_LEVEL, ER_DUCKDB, + "DuckDB data files detected, initializing for crash recovery."); + } + + // Trigger initialization (which includes crash recovery via WAL replay) + bool ret = m_instance->Initialize(); + if (ret) { + LogErr(ERROR_LEVEL, ER_DUCKDB, + "DuckdbManager::InitializeIfNeeded failed."); + return true; + } + + return false; +} + } // namespace myduck diff --git a/sql/duckdb/duckdb_manager.h b/sql/duckdb/duckdb_manager.h index 25e03ff57af..46d0bec1540 100644 --- a/sql/duckdb/duckdb_manager.h +++ b/sql/duckdb/duckdb_manager.h @@ -50,7 +50,18 @@ class DuckdbManager { static std::shared_ptr CreateConnection(); -private: + /** + * Initialize DuckDB if needed at startup. + * This should be called after CreateInstance() to trigger crash recovery + * when: + * 1. DuckDB mode is ON (global_mode == DUCKDB_ON), or + * 2. DuckDB data files exist in the data directory + * + * @return true on error, false on success + */ + static bool InitializeIfNeeded(); + + private: static DuckdbManager *m_instance; DuckdbManager(); @@ -59,6 +70,14 @@ class DuckdbManager { bool Initialize(); + /** + * Check if DuckDB data files exist in the data directory. + * Looks for duckdb.db file or WAL files. + * + * @return true if DuckDB data files exist + */ + static bool DataFilesExist(); + duckdb::DuckDB *m_database = nullptr; std::mutex m_mutex; diff --git a/sql/duckdb/duckdb_mysql_udf.cc b/sql/duckdb/duckdb_mysql_udf.cc index 5ca82cbdd0a..713a43318fd 100644 --- a/sql/duckdb/duckdb_mysql_udf.cc +++ b/sql/duckdb/duckdb_mysql_udf.cc @@ -51,6 +51,7 @@ bool mysql_json_overlaps(duckdb::string_t json1, duckdb::string_t json2) { Item_duckdb_string item_json1(json1); Item_duckdb_string item_json2(json2); Item_func_json_overlaps json_overlaps(POS(), &item_json1, &item_json2); + json_overlaps.m_caller_is_duckdb = true; json_overlaps.fixed = true; return json_overlaps.val_int(); } @@ -59,6 +60,7 @@ int64_t mysql_json_depth(duckdb::string_t json) { Item_duckdb_string item_json(json); Item_func_json_depth json_depth(POS(), &item_json); json_depth.fixed = true; + json_depth.m_caller_is_duckdb = true; return json_depth.val_int(); } @@ -80,6 +82,7 @@ void mysql_json_unquote(duckdb::DataChunk &input, Item_duckdb_string item_json(*ldata); Item_func_json_unquote json_unquote(POS(), &item_json); json_unquote.fixed = true; + json_unquote.m_caller_is_duckdb = true; String tmp; String *func_result = json_unquote.val_str(&tmp); if (json_unquote.null_value) { @@ -109,6 +112,7 @@ void mysql_json_unquote(duckdb::DataChunk &input, Item_duckdb_string item_json(data[i]); Item_func_json_unquote json_unquote(POS(), &item_json); json_unquote.fixed = true; + json_unquote.m_caller_is_duckdb = true; String tmp; String *func_result = json_unquote.val_str(&tmp); if (json_unquote.null_value) { diff --git a/sql/field.cc b/sql/field.cc index a493b5ed6e0..9c59d4859de 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -89,6 +89,7 @@ #include "sql/tztime.h" // Time_zone #include "template_utils.h" // pointer_cast #include "typelib.h" +#include "vidx/vidx_field.h" namespace dd { class Spatial_reference_system; @@ -9308,7 +9309,8 @@ Field *make_field(MEM_ROOT *mem_root, TABLE_SHARE *share, uchar *ptr, TYPELIB *interval, const char *field_name, bool is_nullable, bool is_zerofill, bool is_unsigned, uint decimals, bool treat_bit_as_char, uint pack_length_override, - std::optional srid, bool is_array) { + std::optional srid, bool is_array, + bool is_vector) { uchar *bit_ptr = nullptr; uchar bit_offset = 0; assert(mem_root); @@ -9383,9 +9385,15 @@ Field *make_field(MEM_ROOT *mem_root, TABLE_SHARE *share, uchar *ptr, return new (mem_root) Field_string(ptr, field_length, null_pos, null_bit, auto_flags, field_name, field_charset); case MYSQL_TYPE_VARCHAR: - return new (mem_root) Field_varstring( - ptr, field_length, HA_VARCHAR_PACKLENGTH(field_length), null_pos, - null_bit, auto_flags, field_name, share, field_charset); + if (is_vector) { + return new (mem_root) vidx::Field_vector( + ptr, field_length, HA_VARCHAR_PACKLENGTH(field_length), null_pos, + null_bit, auto_flags, field_name, share); + } else { + return new (mem_root) Field_varstring( + ptr, field_length, HA_VARCHAR_PACKLENGTH(field_length), null_pos, + null_bit, auto_flags, field_name, share, field_charset); + } case MYSQL_TYPE_BLOB: case MYSQL_TYPE_MEDIUM_BLOB: case MYSQL_TYPE_TINY_BLOB: @@ -9538,7 +9546,7 @@ Field *make_field(const Create_field &create_field, TABLE_SHARE *share, create_field.is_zerofill, create_field.is_unsigned, create_field.decimals, create_field.treat_bit_as_char, create_field.pack_length_override, create_field.m_srid, - create_field.is_array); + create_field.is_array, create_field.is_vector); } Field *make_field(const Create_field &create_field, TABLE_SHARE *share, @@ -10051,10 +10059,11 @@ void Field_typed_array::init(TABLE *table_arg) { field_name, is_nullable(), false, // zerofill is meaningless with JSON is_unsigned(), m_elt_decimals, - false, // treat_bit_as_char - 0, // pack_length_override - {}, // srid - false // is_array + false, // treat_bit_as_char + 0, // pack_length_override + {}, // srid + false, // is_array + is_vector() // is_vector ); if (conv_field == nullptr) return; uchar *buf = diff --git a/sql/field.h b/sql/field.h index 6d40ee0b31c..d30dbc829ee 100644 --- a/sql/field.h +++ b/sql/field.h @@ -145,6 +145,7 @@ Field (abstract) | +--Field_longstr | | +--Field_string | | +--Field_varstring +| | +--Field_vector | | +--Field_blob | | +--Field_geom | | +--Field_json @@ -634,6 +635,8 @@ class Field { return (auto_flags & (GENERATED_FROM_EXPRESSION | DEFAULT_NOW)) == 0; } + virtual bool is_vector() const { return false; } + protected: /// Holds the position to the field in record uchar *ptr; @@ -3536,7 +3539,7 @@ class Field_varstring : public Field_longstr { uint32 key_length() const final { return (uint32)field_length; } type_conversion_status store(const char *to, size_t length, const CHARSET_INFO *charset) override; - type_conversion_status store(longlong nr, bool unsigned_val) final; + type_conversion_status store(longlong nr, bool unsigned_val) override; // Inherit the store() overloads that have not been overridden. using Field_longstr::store; double val_real() const final; @@ -3551,7 +3554,7 @@ class Field_varstring : public Field_longstr { size_t make_sort_key(uchar *to, size_t length, size_t trunc_pos) const final; size_t get_key_image(uchar *buff, size_t length, imagetype type) const final; void set_key_image(const uchar *buff, size_t length) final; - void sql_type(String &str) const final; + void sql_type(String &str) const override; uchar *pack(uchar *to, const uchar *from, size_t max_length) const final; const uchar *unpack(uchar *to, const uchar *from, uint param_data) final; int cmp_binary(const uchar *a, const uchar *b, @@ -3567,16 +3570,16 @@ class Field_varstring : public Field_longstr { Field *new_field(MEM_ROOT *root, TABLE *new_table) const final; Field *new_key_field(MEM_ROOT *root, TABLE *new_table, uchar *new_ptr, uchar *new_null_ptr, uint new_null_bit) const final; - Field_varstring *clone(MEM_ROOT *mem_root) const final { + Field_varstring *clone(MEM_ROOT *mem_root) const override { assert(type() == MYSQL_TYPE_VARCHAR); assert(real_type() == MYSQL_TYPE_VARCHAR); return new (mem_root) Field_varstring(*this); } - uint is_equal(const Create_field *new_field) const final; + uint is_equal(const Create_field *new_field) const override; void hash(ulong *nr, ulong *nr2) const final; const uchar *data_ptr() const final { return ptr + length_bytes; } bool is_text_key_type() const final { return binary() ? false : true; } - uint32 get_length_bytes() const override { return length_bytes; } + uint32 get_length_bytes() const final { return length_bytes; } private: /* Store number of bytes used to store length (1 or 2) */ @@ -4589,7 +4592,8 @@ Field *make_field(MEM_ROOT *mem_root_arg, TABLE_SHARE *share, uchar *ptr, TYPELIB *interval, const char *field_name, bool is_nullable, bool is_zerofill, bool is_unsigned, uint decimals, bool treat_bit_as_char, uint pack_length_override, - std::optional srid, bool is_array); + std::optional srid, bool is_array, + bool is_vector); /** Instantiates a Field object with the given name and record buffer values. diff --git a/sql/filesort.cc b/sql/filesort.cc index 9e7367e3dcb..eccabe6aa1c 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -83,6 +83,7 @@ #include "sql/filesort_utils.h" #include "sql/handler.h" #include "sql/item.h" +#include "sql/item_strfunc.h" #include "sql/item_subselect.h" #include "sql/iterators/row_iterator.h" #include "sql/iterators/sorting_iterator.h" @@ -1433,7 +1434,8 @@ uint Sort_param::make_sortkey(Bounds_checked_array dst, bool is_null = maybe_null && *to == 0; if (maybe_null) { assert(*to == 0 || *to == 1); - if (sort_field->reverse && is_null) { + if ((vidx::check_item_func_vec_distance(item) || sort_field->reverse) && + is_null) { *to = 0xff; } ++to; diff --git a/sql/gen_lex_token.cc b/sql/gen_lex_token.cc index 2f4e23a2e8d..6300925f127 100644 --- a/sql/gen_lex_token.cc +++ b/sql/gen_lex_token.cc @@ -142,7 +142,7 @@ struct gen_lex_token_string { - mode named tokens from bison (sql_yacc.yy). See also YYMAXUTOK. */ -const int MY_MAX_TOKEN = 1301; +const int MY_MAX_TOKEN = 1305; gen_lex_token_string compiled_token_array[MY_MAX_TOKEN]; diff --git a/sql/handler.cc b/sql/handler.cc index 14eb38e64e8..07db6288a63 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -133,6 +133,7 @@ #include "sql/duckdb/duckdb_config.h" #include "sql/duckdb/duckdb_context.h" +#include "vidx/vidx_index.h" /** @def MYSQL_TABLE_IO_WAIT @@ -903,6 +904,36 @@ int ha_initialize_handlerton(st_plugin_int *plugin) { return 1; } +int setup_transaction_participant(st_plugin_int *plugin) { + /* mysql has no type transaction_participant, but it's a part of + handlerton. So using handlerton is ok. */ + handlerton *tp = (handlerton *)plugin->data; + ulong fslot; + /* Use the same way to iterate as in ha_initialize_handlerton(). */ + for (fslot = 0; fslot < se_plugin_array.size(); fslot++) { + if (!se_plugin_array[fslot]) break; + } + if (fslot < se_plugin_array.size()) + tp->slot = fslot; + else { + tp->slot = se_plugin_array.size(); + } + + if (se_plugin_array.assign_at(tp->slot, plugin) || + builtin_htons.assign_at(tp->slot, (plugin->plugin_dl == nullptr))) + return 1; + + uint tmp = tp->savepoint_offset; + tp->savepoint_offset = savepoint_alloc_size; + savepoint_alloc_size += tmp; + + if (tp->prepare) { + total_ha_2pc++; + } + + return 0; +} + int ha_init() { int error = 0; DBUG_TRACE; @@ -2553,6 +2584,9 @@ int ha_delete_table(THD *thd, handlerton *table_type, const char *path, TABLE_SHARE dummy_share; DBUG_TRACE; + error = vidx::dd_table_has_hlindexes(table_def) && + vidx::delete_table(thd, table_def, db); + dummy_table.s = &dummy_share; /* DB_TYPE_UNKNOWN is used in ALTER TABLE when renaming only .frm files */ @@ -2566,7 +2600,8 @@ int ha_delete_table(THD *thd, handlerton *table_type, const char *path, path = get_canonical_filename(file, path, tmp_path); - if ((error = file->ha_delete_table(path, table_def)) && generate_warning) { + if ((error || (error = file->ha_delete_table(path, table_def))) && + generate_warning) { /* Because file->print_error() use my_error() to generate the error message we use an internal error handler to intercept it and store the text @@ -3446,6 +3481,8 @@ int handler::ha_index_first(uchar *buf) { // Set status for the need to update generated fields m_update_generated_read_fields = table->has_gcol(); + assert(active_index < table_share->keys); + MYSQL_TABLE_IO_WAIT(PSI_TABLE_FETCH_ROW, active_index, result, { result = index_first(buf); }) if (!result && m_update_generated_read_fields) { @@ -4835,8 +4872,10 @@ int handler::ha_bulk_update_row(const uchar *old_data, uchar *new_data, int handler::ha_delete_all_rows() { assert(table_share->tmp_table != NO_TMP_TABLE || m_lock_type == F_WRLCK); mark_trx_read_write(); + int err = delete_all_rows(); + if (unlikely(!err)) err = table->hlindexes_on_delete_all(); - return delete_all_rows(); + return err; } /** @@ -5204,6 +5243,10 @@ int handler::index_next_same(uchar *buf, const uchar *key, uint keylen) { by storage engine if it supports atomic DDL. For non-temporary tables these changes will be saved to the data-dictionary by this call. + @param last_key key info of last index if table_def is created + instead of loaded from dd. Otherwise nullptr + @param recycled Whether create table came from recycling the + truncated operation @retval 0 ok @@ -5213,7 +5256,7 @@ int handler::index_next_same(uchar *buf, const uchar *key, uint keylen) { int ha_create_table(THD *thd, const char *path, const char *db, const char *table_name, HA_CREATE_INFO *create_info, bool update_create_info, bool is_temp_table, - dd::Table *table_def) { + dd::Table *table_def, KEY *last_key, bool recycled) { int error = 1; TABLE table; char name_buff[FN_REFLEN]; @@ -5224,6 +5267,7 @@ int ha_create_table(THD *thd, const char *path, const char *db, (create_info->options & HA_LEX_CREATE_TMP_TABLE) || (strstr(path, tmp_file_prefix) != nullptr); #endif + const uint64_t old_table_id = table_def->se_private_id(); DBUG_TRACE; init_tmp_table_share(thd, &share, db, 0, table_name, path, nullptr); @@ -5234,6 +5278,10 @@ int ha_create_table(THD *thd, const char *path, const char *db, share.m_psi = PSI_TABLE_CALL(get_table_share)(temp_table, &share); #endif + if (last_key == nullptr) { + last_key = share.get_vec_key(); + } + // When db_stat is 0, we can pass nullptr as dd::Table since it won't be used. destroy(&table); if (open_table_from_share(thd, &share, "", 0, (uint)READ_ALL, 0, &table, true, @@ -5257,6 +5305,10 @@ int ha_create_table(THD *thd, const char *path, const char *db, PSI_TABLE_CALL(drop_table_share) (temp_table, db, strlen(db), table_name, strlen(table_name)); #endif + } else if (vidx::key_is_vector(last_key) && + vidx::create_table(thd, last_key, table_def, &table, db, + old_table_id)) { + error = 1; } else { /* We do post-create update only for engines supporting atomic DDL @@ -7894,6 +7946,11 @@ class Binlog_log_row_cleanup { int binlog_log_row(TABLE *table, const uchar *before_record, const uchar *after_record, Log_func *log_func) { + if (table->s->is_hlindex) { + /* Don't write binlog for HLINDEX. */ + return 0; + } + bool error = false; THD *const thd = table->in_use; @@ -7966,6 +8023,11 @@ int binlog_log_row(TABLE *table, const uchar *before_record, int handler::ha_external_lock(THD *thd, int lock_type) { int error; DBUG_TRACE; + if (table->hlindex != nullptr && lock_type == F_UNLCK && + table->hlindex->file->get_lock_type() != F_UNLCK && + (error = table->hlindex->file->ha_external_lock(thd, F_UNLCK)) != 0) { + return error; + } /* Whether this is lock or unlock, this should be true, and is to verify that if get_auto_increment() was called (thus may have reserved intervals or @@ -8031,6 +8093,9 @@ int handler::ha_reset() { m_unique = nullptr; const int retval = reset(); + + table->reset_hlindexes(); + return retval; } @@ -8055,6 +8120,8 @@ int handler::ha_write_row(uchar *buf) { if (unlikely(error)) return error; + if (unlikely((error = table->hlindexes_on_insert()))) return error; + if (unlikely((error = binlog_log_row(table, nullptr, buf, log_func)))) return error; /* purecov: inspected */ @@ -8083,6 +8150,7 @@ int handler::ha_update_row(const uchar *old_data, uchar *new_data) { MYSQL_TABLE_IO_WAIT(PSI_TABLE_UPDATE_ROW, active_index, error, { error = update_row(old_data, new_data); }) + if (unlikely(!error && (error = table->hlindexes_on_update()))) return error; if (unlikely(error)) return error; if (unlikely((error = binlog_log_row(table, old_data, new_data, log_func)))) @@ -8115,6 +8183,7 @@ int handler::ha_delete_row(const uchar *buf) { { error = delete_row(buf); }) if (unlikely(error)) return error; + if (unlikely((error = table->hlindexes_on_delete(buf)))) return error; if (unlikely((error = binlog_log_row(table, buf, nullptr, log_func)))) return error; return 0; diff --git a/sql/handler.h b/sql/handler.h index dd9ba4e389f..6bd2f24181d 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -671,6 +671,7 @@ enum legacy_db_type { DB_TYPE_TEMPTABLE, DB_TYPE_FIRST_DYNAMIC = 42, DB_TYPE_DUCKDB, + DB_TYPE_HLINDEX_HELPER, DB_TYPE_DEFAULT = 127 // Must be last }; @@ -7205,6 +7206,7 @@ int ha_init(void); void ha_end(); int ha_initialize_handlerton(st_plugin_int *plugin); int ha_finalize_handlerton(st_plugin_int *plugin); +int setup_transaction_participant(st_plugin_int *plugin); TYPELIB *ha_known_exts(); int ha_panic(enum ha_panic_function flag); @@ -7227,7 +7229,8 @@ void ha_create_database(char *db); int ha_create_table(THD *thd, const char *path, const char *db, const char *table_name, HA_CREATE_INFO *create_info, bool update_create_info, bool is_temp_table, - dd::Table *table_def); + dd::Table *table_def, KEY *last_key = nullptr, + bool recycled = false); int ha_delete_table(THD *thd, handlerton *db_type, const char *path, const char *db, const char *alias, diff --git a/sql/item.h b/sql/item.h index a4fb20d9767..441785b00d5 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1588,6 +1588,16 @@ class Item : public Parse_tree_node { max_length = MAX_DATETIME_WIDTH + fsp + (fsp > 0 ? 1 : 0); } + /** + Set the data type of the Item to be VECTOR. + */ + void set_data_type_vector(uint32 max_l) { + set_data_type(MYSQL_TYPE_VARCHAR); + collation.set(&my_charset_bin, DERIVATION_IMPLICIT); + decimals = DECIMAL_NOT_SPECIFIED; + max_length = max_l; + } + /** Set the data type of the Item to be GEOMETRY. */ diff --git a/sql/item_create.cc b/sql/item_create.cc index b02274259d3..ffd0b85afd7 100644 --- a/sql/item_create.cc +++ b/sql/item_create.cc @@ -1662,6 +1662,17 @@ static const std::pair func_array[] = { {"TO_BASE64", SQL_FN(Item_func_to_base64, 1)}, {"TO_DAYS", SQL_FN(Item_func_to_days, 1)}, {"TO_SECONDS", SQL_FN(Item_func_to_seconds, 1)}, + {"VEC_DISTANCE", SQL_FN(vidx::Item_func_vec_distance, 2)}, + {"VEC_DISTANCE_EUCLIDEAN", + SQL_FN(vidx::Item_func_vec_distance_euclidean, 2)}, + {"VEC_DISTANCE_COSINE", SQL_FN(vidx::Item_func_vec_distance_cosine, 2)}, + {"VEC_FROMTEXT", SQL_FN(vidx::Item_func_vec_fromtext, 1)}, + {"TO_VECTOR", SQL_FN(vidx::Item_func_vec_fromtext, 1)}, + {"STRING_TO_VECTOR", SQL_FN(vidx::Item_func_vec_fromtext, 1)}, + {"VEC_TOTEXT", SQL_FN(vidx::Item_func_vec_totext, 1)}, + {"FROM_VECTOR", SQL_FN(vidx::Item_func_vec_totext, 1)}, + {"VECTOR_TO_STRING", SQL_FN(vidx::Item_func_vec_totext, 1)}, + {"VECTOR_DIM", SQL_FN(vidx::Item_func_vector_dim, 1)}, {"UCASE", SQL_FN(Item_func_upper, 1)}, {"UNCOMPRESS", SQL_FN(Item_func_uncompress, 1)}, {"UNCOMPRESSED_LENGTH", SQL_FN(Item_func_uncompressed_length, 1)}, diff --git a/sql/item_func.h b/sql/item_func.h index 72fefbf903a..07b6815ac7c 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -192,6 +192,7 @@ class Item_func : public Item_result_field { GE_FUNC, GT_FUNC, FT_FUNC, + VECTOR_DISTANCE_FUNC, MATCH_FUNC, LIKE_FUNC, ISNULL_FUNC, diff --git a/sql/item_json_func.cc b/sql/item_json_func.cc index 199d46c2f49..208deb68d2c 100644 --- a/sql/item_json_func.cc +++ b/sql/item_json_func.cc @@ -73,6 +73,7 @@ #include "sql/thd_raii.h" #include "sql/thr_malloc.h" #include "template_utils.h" // down_cast +#include "duckdb/common/exception.hpp" class PT_item_list; @@ -304,7 +305,8 @@ static bool check_convertible_to_json(const Item *item, int argument_number, */ static bool json_is_valid(Item **args, uint arg_idx, String *value, const char *func_name, Json_dom_ptr *dom, - bool require_str_or_json, bool *valid) { + bool require_str_or_json, bool *valid, + bool caller_is_duckdb = false) { Item *const arg_item = args[arg_idx]; enum_field_types field_type = get_normalized_field_type(arg_item); @@ -312,6 +314,9 @@ static bool json_is_valid(Item **args, uint arg_idx, String *value, if (!is_convertible_to_json(arg_item)) { if (require_str_or_json) { *valid = false; + if (caller_is_duckdb) { + throw duckdb::InvalidInputException("Invalid input in json func"); + } my_error(ER_INVALID_TYPE_FOR_JSON, MYF(0), arg_idx + 1, func_name); return true; } @@ -330,7 +335,9 @@ static bool json_is_valid(Item **args, uint arg_idx, String *value, return !*valid; } else { String *const res = arg_item->val_str(value); - if (current_thd->is_error()) return true; + if (!caller_is_duckdb) { + if (current_thd->is_error()) return true; + } if (arg_item->null_value) { *valid = true; @@ -340,8 +347,11 @@ static bool json_is_valid(Item **args, uint arg_idx, String *value, bool parse_error = false; const bool failure = parse_json( *res, dom, require_str_or_json, - [&parse_error, arg_idx, func_name](const char *parse_err, - size_t err_offset) { + [&parse_error, arg_idx, func_name, caller_is_duckdb]( + const char *parse_err, size_t err_offset) { + if (caller_is_duckdb) { + throw duckdb::InvalidInputException("Invalid input in json func"); + } my_error(ER_INVALID_JSON_TEXT_IN_PARAM, MYF(0), arg_idx + 1, func_name, parse_err, err_offset, ""); parse_error = true; @@ -1084,7 +1094,8 @@ bool json_value(Item *arg, Json_wrapper *result, bool *has_value) { } bool get_json_wrapper(Item **args, uint arg_idx, String *str, - const char *func_name, Json_wrapper *wrapper) { + const char *func_name, Json_wrapper *wrapper, + bool caller_is_duckdb) { Item *const arg = args[arg_idx]; bool has_value; @@ -1104,10 +1115,14 @@ bool get_json_wrapper(Item **args, uint arg_idx, String *str, Json_dom_ptr dom; //@< we'll receive a DOM here from a successful text parse bool valid; - if (json_is_valid(args, arg_idx, str, func_name, &dom, true, &valid)) + if (json_is_valid(args, arg_idx, str, func_name, &dom, true, &valid, + caller_is_duckdb)) return true; if (!valid) { + if (caller_is_duckdb) { + throw duckdb::InvalidInputException("Invalid input in json func"); + } my_error(ER_INVALID_TYPE_FOR_JSON, MYF(0), arg_idx + 1, func_name); return true; } @@ -1743,10 +1758,13 @@ longlong Item_func_json_depth::val_int() { Json_wrapper wrapper; try { - if (get_json_wrapper(args, 0, &m_doc_value, func_name(), &wrapper)) + if (get_json_wrapper(args, 0, &m_doc_value, func_name(), &wrapper, m_caller_is_duckdb)) return error_int(); } catch (...) { /* purecov: begin inspected */ + if (m_caller_is_duckdb) { + throw duckdb::InvalidInputException("Invalid input in json func"); + } handle_std_exception(func_name()); return error_int(); /* purecov: end */ @@ -3249,7 +3267,7 @@ String *Item_func_json_unquote::val_str(String *str) { try { if (args[0]->data_type() == MYSQL_TYPE_JSON) { Json_wrapper wr; - if (get_json_wrapper(args, 0, str, func_name(), &wr)) { + if (get_json_wrapper(args, 0, str, func_name(), &wr, m_caller_is_duckdb)) { return error_str(); } @@ -3260,8 +3278,13 @@ String *Item_func_json_unquote::val_str(String *str) { m_value.length(0); - if (wr.to_string(&m_value, false, func_name(), - JsonDocumentDefaultDepthHandler)) { + if (m_caller_is_duckdb) { + if (wr.to_string(&m_value, false, func_name(), + JsonDocumentDefaultDepthHandlerDuckDB)) { + return error_str(); + } + } else if (wr.to_string(&m_value, false, func_name(), + JsonDocumentDefaultDepthHandler)) { return error_str(); } @@ -3317,6 +3340,7 @@ String *Item_func_json_unquote::val_str(String *str) { Json_dom_ptr dom; JsonParseDefaultErrorHandler parse_handler(func_name(), 0); + parse_handler.m_caller_is_duckdb = m_caller_is_duckdb; if (parse_json(*utf8str, &dom, true, parse_handler, JsonDocumentDefaultDepthHandler)) { return error_str(); @@ -3331,6 +3355,9 @@ String *Item_func_json_unquote::val_str(String *str) { return error_str(); /* purecov: inspected */ } catch (...) { /* purecov: begin inspected */ + if (m_caller_is_duckdb) { + throw duckdb::InvalidInputException("Invalid input in json func"); + } handle_std_exception(func_name()); return error_str(); /* purecov: end */ @@ -3857,14 +3884,16 @@ longlong Item_func_json_overlaps::val_int() { Json_wrapper *doc_b = &wr_b; // arg 0 is the document 1 - if (get_json_wrapper(args, 0, &m_doc_value, func_name(), doc_a) || + if (get_json_wrapper(args, 0, &m_doc_value, func_name(), doc_a, + m_caller_is_duckdb) || args[0]->null_value) { null_value = true; return 0; } // arg 1 is the document 2 - if (get_json_wrapper(args, 1, &m_doc_value, func_name(), doc_b) || + if (get_json_wrapper(args, 1, &m_doc_value, func_name(), doc_b, + m_caller_is_duckdb) || args[1]->null_value) { null_value = true; return 0; @@ -3913,6 +3942,9 @@ longlong Item_func_json_overlaps::val_int() { } /* purecov: begin inspected */ } catch (...) { + if (m_caller_is_duckdb) { + throw duckdb::InvalidInputException("Invalid invalid in json func"); + } handle_std_exception(func_name()); return error_int(); /* purecov: end */ diff --git a/sql/item_json_func.h b/sql/item_json_func.h index 3244834aa2c..fe4b70109a1 100644 --- a/sql/item_json_func.h +++ b/sql/item_json_func.h @@ -250,10 +250,12 @@ bool json_value(Item *arg, Json_wrapper *result, bool *has_value); @param[out] str the string buffer @param[in] func_name the name of the function we are executing @param[out] wrapper the JSON value wrapper + @param[in] caller_is_duckdb the func caller is duckdb @returns false if we found a value or NULL, true if not. */ bool get_json_wrapper(Item **args, uint arg_idx, String *str, - const char *func_name, Json_wrapper *wrapper); + const char *func_name, Json_wrapper *wrapper, + bool caller_is_duckdb = false); /** Convert Json values or MySQL values to JSON. @@ -534,6 +536,7 @@ class Item_func_json_depth final : public Item_int_func { } longlong val_int() override; + bool m_caller_is_duckdb = false; }; /** @@ -915,6 +918,7 @@ class Item_func_json_unquote : public Item_str_func { } String *val_str(String *str) override; + bool m_caller_is_duckdb = false; }; /** @@ -1054,6 +1058,7 @@ class Item_func_json_overlaps : public Item_bool_func { enum_const_item_cache can_cache_json_arg(Item *arg) override { return (arg == args[0] || arg == args[1]) ? CACHE_JSON_VALUE : CACHE_NONE; } + bool m_caller_is_duckdb = false; }; class Item_func_member_of : public Item_bool_func { diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 76d8231d426..710e1cff802 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -1785,4 +1785,6 @@ class Item_func_internal_get_dd_column_extra final : public Item_str_func { String *val_str(String *) override; }; +#include "vidx/vidx_func.h" + #endif /* ITEM_STRFUNC_INCLUDED */ diff --git a/sql/iterators/basic_row_iterators.cc b/sql/iterators/basic_row_iterators.cc index 1b5a0fac8d6..b0c1e7176e8 100644 --- a/sql/iterators/basic_row_iterators.cc +++ b/sql/iterators/basic_row_iterators.cc @@ -58,13 +58,15 @@ template IndexScanIterator::IndexScanIterator(THD *thd, TABLE *table, int idx, bool use_order, double expected_rows, - ha_rows *examined_rows) + ha_rows *examined_rows, + void *vec_func) : TableRowIterator(thd, table), m_record(table->record[0]), m_idx(idx), m_use_order(use_order), m_expected_rows(expected_rows), - m_examined_rows(examined_rows) {} + m_examined_rows(examined_rows), + m_vec_func(vec_func) {} template IndexScanIterator::~IndexScanIterator() { @@ -75,12 +77,19 @@ IndexScanIterator::~IndexScanIterator() { template bool IndexScanIterator::Init() { + assert((m_idx < (int)table()->s->keys) == (m_vec_func == nullptr)); + if (!table()->file->inited) { if (table()->covering_keys.is_set(m_idx) && !table()->no_keyread) { table()->set_keyread(true); } - int error = table()->file->ha_index_init(m_idx, m_use_order); + /* m_vec_func means the vector index is used which using the auxiliary + table and later re-scanning the main table */ + int error = m_vec_func != nullptr + ? table()->file->ha_rnd_init(false) + : table()->file->ha_index_init(m_idx, m_use_order); + if (error) { PrintError(error); return true; @@ -99,12 +108,18 @@ bool IndexScanIterator::Init() { //! @cond template <> int IndexScanIterator::Read() { // Forward read. + assert((m_vec_func == nullptr) == (m_idx < (int)table()->s->keys)); + assert((m_vec_func == nullptr) == (table()->file->inited == handler::INDEX)); + int error; if (m_first) { - error = table()->file->ha_index_first(m_record); + error = (m_vec_func != nullptr) + ? table()->hlindex_read_first(m_idx, m_vec_func) + : table()->file->ha_index_first(m_record); m_first = false; } else { - error = table()->file->ha_index_next(m_record); + error = (m_vec_func != nullptr) ? table()->hlindex_read_next() + : table()->file->ha_index_next(m_record); } if (error) return HandleError(error); if (m_examined_rows != nullptr) { @@ -115,6 +130,9 @@ int IndexScanIterator::Read() { // Forward read. template <> int IndexScanIterator::Read() { // Backward read. + assert(m_idx < (int)table()->s->keys); + assert(m_vec_func == nullptr); + int error; if (m_first) { error = table()->file->ha_index_last(m_record); diff --git a/sql/iterators/basic_row_iterators.h b/sql/iterators/basic_row_iterators.h index 8a55cb837bc..8a8c0097ca9 100644 --- a/sql/iterators/basic_row_iterators.h +++ b/sql/iterators/basic_row_iterators.h @@ -113,7 +113,8 @@ class IndexScanIterator final : public TableRowIterator { // // "examined_rows", if not nullptr, is incremented for each successful Read(). IndexScanIterator(THD *thd, TABLE *table, int idx, bool use_order, - double expected_rows, ha_rows *examined_rows); + double expected_rows, ha_rows *examined_rows, + void *vec_func); ~IndexScanIterator() override; bool Init() override; @@ -125,6 +126,7 @@ class IndexScanIterator final : public TableRowIterator { const bool m_use_order; const double m_expected_rows; ha_rows *const m_examined_rows; + void *m_vec_func; bool m_first = true; }; diff --git a/sql/join_optimizer/access_path.cc b/sql/join_optimizer/access_path.cc index 6e835eebaa0..f8ad2bb624e 100644 --- a/sql/join_optimizer/access_path.cc +++ b/sql/join_optimizer/access_path.cc @@ -428,13 +428,15 @@ unique_ptr_destroy_only CreateIteratorFromAccessPath( case AccessPath::INDEX_SCAN: { const auto ¶m = path->index_scan(); if (param.reverse) { + assert(param.vec_func == nullptr); + iterator = NewIterator>( thd, mem_root, param.table, param.idx, param.use_order, - path->num_output_rows(), examined_rows); + path->num_output_rows(), examined_rows, param.vec_func); } else { iterator = NewIterator>( thd, mem_root, param.table, param.idx, param.use_order, - path->num_output_rows(), examined_rows); + path->num_output_rows(), examined_rows, param.vec_func); } break; } diff --git a/sql/join_optimizer/access_path.h b/sql/join_optimizer/access_path.h index 3d55eaa87d3..5384b3926a2 100644 --- a/sql/join_optimizer/access_path.h +++ b/sql/join_optimizer/access_path.h @@ -867,6 +867,7 @@ struct AccessPath { int idx; bool use_order; bool reverse; + void *vec_func; } index_scan; struct { TABLE *table; @@ -1247,7 +1248,8 @@ inline AccessPath *NewTableScanAccessPath(THD *thd, TABLE *table, inline AccessPath *NewIndexScanAccessPath(THD *thd, TABLE *table, int idx, bool use_order, bool reverse, - bool count_examined_rows) { + bool count_examined_rows, + void *vec_func = nullptr) { AccessPath *path = new (thd->mem_root) AccessPath; path->type = AccessPath::INDEX_SCAN; path->count_examined_rows = count_examined_rows; @@ -1255,6 +1257,7 @@ inline AccessPath *NewIndexScanAccessPath(THD *thd, TABLE *table, int idx, path->index_scan().idx = idx; path->index_scan().use_order = use_order; path->index_scan().reverse = reverse; + path->index_scan().vec_func = vec_func; return path; } diff --git a/sql/key.h b/sql/key.h index c491210b562..81da4bb8867 100644 --- a/sql/key.h +++ b/sql/key.h @@ -159,6 +159,10 @@ class KEY { */ ulong *rec_per_key{nullptr}; + /** Options of vector index */ + uint vector_distance{0}; + uint vector_m{0}; + /** @retval true if this is a functional index (at least one of the key parts is a functional key part). diff --git a/sql/key_spec.h b/sql/key_spec.h index af11926465b..5902d5c3b31 100644 --- a/sql/key_spec.h +++ b/sql/key_spec.h @@ -43,7 +43,8 @@ enum keytype { KEYTYPE_MULTIPLE, KEYTYPE_FULLTEXT, KEYTYPE_SPATIAL, - KEYTYPE_FOREIGN + KEYTYPE_FOREIGN, + KEYTYPE_VECTOR }; enum fk_option { @@ -76,6 +77,8 @@ class KEY_CREATE_INFO { LEX_CSTRING parser_name = {NullS, 0}; LEX_CSTRING comment = {NullS, 0}; bool is_visible = true; + uint vector_m = UINT_MAX; + uint vector_distance = UINT_MAX; KEY_CREATE_INFO() = default; diff --git a/sql/lex.h b/sql/lex.h index fb3402d12d5..061a3ec43af 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -824,6 +824,11 @@ static const SYMBOL symbols[] = { {SYM("ZEROFILL", ZEROFILL_SYM)}, {SYM("ZONE", ZONE_SYM)}, {SYM("||", OR_OR_SYM)}, + {SYM("VECTOR", VECTOR_SYM)}, + {SYM("M", M_SYM)}, + {SYM("DISTANCE", DISTANCE_SYM)}, + {SYM("EUCLIDEAN", EUCLIDEAN_SYM)}, + {SYM("COSINE", COSINE_SYM)}, /* Place keywords that accept optimizer hints below this comment. */ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 00f7a6b4758..3982f9e6f72 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1048,6 +1048,8 @@ const char *my_localhost = "localhost"; bool opt_large_files = sizeof(my_off_t) > 4; static bool opt_autocommit; ///< for --autocommit command-line option static get_opt_arg_source source_autocommit; +static bool opt_sql_safe_updates; ///< for --sql-safe-updates command-line option +static get_opt_arg_source source_sql_safe_updates; /* Used with --help for detailed option @@ -7918,6 +7920,11 @@ int mysqld_main(int argc, char **argv) if (myduck::DuckdbManager::CreateInstance()) unireg_abort(MYSQLD_ABORT_EXIT); + // Initialize DuckDB early if needed for crash recovery + // This ensures WAL replay happens at startup rather than on first request + if (myduck::DuckdbManager::InitializeIfNeeded()) + unireg_abort(MYSQLD_ABORT_EXIT); + if (init_server_components()) unireg_abort(MYSQLD_ABORT_EXIT); if (!server_id_supplied) @@ -9278,6 +9285,14 @@ struct my_option my_long_options[] = { "Option used by mysql-test for debugging and testing of replication.", &opt_sporadic_binlog_dump_fail, &opt_sporadic_binlog_dump_fail, nullptr, GET_BOOL, NO_ARG, 0, 0, 0, nullptr, 0, nullptr}, + /* + Because Sys_var_bit does not support command-line options, we need to + explicitly add one for --sql-safe-updates + */ + {"sql-safe-updates", 0, "Set default value for sql_safe_updates (0 or 1)", + &opt_sql_safe_updates, &opt_sql_safe_updates, nullptr, GET_BOOL, OPT_ARG, + 0, 0, 0, &source_sql_safe_updates, /* arg_source, to be copied to Sys_var */ + 0, nullptr}, {"ssl", OPT_USE_SSL, "Enable SSL for connection (automatically enabled with other flags).", &opt_use_ssl, &opt_use_ssl, nullptr, GET_BOOL, OPT_ARG, 1, 0, 0, nullptr, @@ -11265,6 +11280,12 @@ static int get_options(int *argc_ptr, char ***argv_ptr) { Sys_autocommit_ptr->set_source_name(opt->arg_source->m_path_name); Sys_autocommit_ptr->set_source(opt->arg_source->m_source); + // Synchronize @@global.sql_safe_updates value on --sql-safe-updates + if (opt_sql_safe_updates) + global_system_variables.option_bits |= OPTION_SAFE_UPDATES; + else + global_system_variables.option_bits &= ~OPTION_SAFE_UPDATES; + global_system_variables.sql_mode = expand_sql_mode(global_system_variables.sql_mode, nullptr); @@ -11919,6 +11940,7 @@ PSI_mutex_key key_mutex_slave_parallel_worker_count; PSI_mutex_key key_mutex_slave_parallel_worker; PSI_mutex_key key_structure_guard_mutex; PSI_mutex_key key_TABLE_SHARE_LOCK_ha_data; +PSI_mutex_key key_TABLE_SHARE_LOCK_share; PSI_mutex_key key_LOCK_query_plan; PSI_mutex_key key_LOCK_thd_query; PSI_mutex_key key_LOCK_cost_const; @@ -12008,6 +12030,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_mutex_slave_parallel_worker_count, "Relay_log_info::exit_count_lock", 0, 0, PSI_DOCUMENT_ME}, { &key_mutex_slave_parallel_worker, "Worker_info::jobs_lock", 0, 0, PSI_DOCUMENT_ME}, { &key_TABLE_SHARE_LOCK_ha_data, "TABLE_SHARE::LOCK_ha_data", 0, 0, PSI_DOCUMENT_ME}, + { &key_TABLE_SHARE_LOCK_share, "TABLE_SHARE::LOCK_share", 0, 0, PSI_DOCUMENT_ME}, { &key_LOCK_error_messages, "LOCK_error_messages", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, { &key_LOCK_log_throttle_qni, "LOCK_log_throttle_qni", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, { &key_gtid_ensure_index_mutex, "Gtid_state", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, diff --git a/sql/mysqld.h b/sql/mysqld.h index f86f602fbd9..153762e0fd9 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -462,6 +462,7 @@ extern PSI_mutex_key key_mutex_slave_parallel_worker; extern PSI_mutex_key key_mutex_slave_parallel_worker_count; extern PSI_mutex_key key_structure_guard_mutex; extern PSI_mutex_key key_TABLE_SHARE_LOCK_ha_data; +extern PSI_mutex_key key_TABLE_SHARE_LOCK_share; extern PSI_mutex_key key_LOCK_query_plan; extern PSI_mutex_key key_LOCK_thd_query; extern PSI_mutex_key key_LOCK_cost_const; diff --git a/sql/parse_tree_column_attrs.h b/sql/parse_tree_column_attrs.h index bd37074e556..3c06119236f 100644 --- a/sql/parse_tree_column_attrs.h +++ b/sql/parse_tree_column_attrs.h @@ -618,6 +618,7 @@ class PT_type : public Parse_tree_node { virtual uint get_uint_geom_type() const { return 0; } virtual List *get_interval_list() const { return nullptr; } virtual bool is_serial_type() const { return false; } + virtual bool is_vector() const { return false; } }; /** @@ -738,6 +739,25 @@ class PT_char_type : public PT_type { const CHARSET_INFO *get_charset() const override { return charset; } }; +class PT_vector_type : public PT_type { + char vector_length_buffer[33]{}; + + public: + PT_vector_type(THD *thd, const char *length) : PT_type(MYSQL_TYPE_VARCHAR) { +#ifndef NDEBUG + assert(length != nullptr); +#endif /* NDEBUG */ + uint vector_length = atoi(length) * sizeof(float); + sprintf(vector_length_buffer, "%u", vector_length); + + thd->m_query_has_vector_column = true; + } + + const char *get_length() const override { return vector_length_buffer; } + const CHARSET_INFO *get_charset() const override { return &my_charset_bin; } + bool is_vector() const final { return true; } +}; + enum class Blob_type { TINY = MYSQL_TYPE_TINY_BLOB, MEDIUM = MYSQL_TYPE_MEDIUM_BLOB, @@ -959,6 +979,7 @@ class PT_field_def_base : public Parse_tree_node { const char *dec; const CHARSET_INFO *charset; bool has_explicit_collation; + bool is_vector = false; uint uint_geom_type; List *interval_list; alter_info_flags_t alter_info_flags; @@ -994,6 +1015,7 @@ class PT_field_def_base : public Parse_tree_node { length = type_node->get_length(); dec = type_node->get_dec(); charset = type_node->get_charset(); + is_vector = type_node->is_vector(); uint_geom_type = type_node->get_uint_geom_type(); interval_list = type_node->get_interval_list(); check_const_spec_list = new (pc->thd->mem_root) diff --git a/sql/parse_tree_nodes.cc b/sql/parse_tree_nodes.cc index 0721d64c6b8..d3ca9e899c0 100644 --- a/sql/parse_tree_nodes.cc +++ b/sql/parse_tree_nodes.cc @@ -2205,7 +2205,8 @@ bool PT_column_def::contextualize(Table_ddl_parse_context *pc) { field_def->interval_list, field_def->charset, field_def->has_explicit_collation, field_def->uint_geom_type, field_def->gcol_info, field_def->default_val_info, opt_place, - field_def->m_srid, field_def->check_const_spec_list, field_hidden_type); + field_def->m_srid, field_def->check_const_spec_list, field_hidden_type, + field_def->is_vector); } Sql_cmd *PT_create_table_stmt::make_cmd(THD *thd) { @@ -2959,7 +2960,7 @@ bool PT_alter_table_change_column::contextualize(Table_ddl_parse_context *pc) { m_field_def->has_explicit_collation, m_field_def->uint_geom_type, m_field_def->gcol_info, m_field_def->default_val_info, m_opt_place, m_field_def->m_srid, m_field_def->check_const_spec_list, - field_hidden_type); + field_hidden_type, m_field_def->is_vector); } bool PT_alter_table_rename::contextualize(Table_ddl_parse_context *pc) { @@ -3471,7 +3472,8 @@ bool PT_json_table_column_with_path::contextualize(Parse_context *pc) { nullptr, // Gcol_info nullptr, // Default gen expression {}, // SRID - dd::Column::enum_hidden_type::HT_VISIBLE); // Hidden + dd::Column::enum_hidden_type::HT_VISIBLE, // Hidden + m_type->is_vector()); // Hidden return false; } diff --git a/sql/parse_tree_nodes.h b/sql/parse_tree_nodes.h index a849c5a8117..a95e7a65a67 100644 --- a/sql/parse_tree_nodes.h +++ b/sql/parse_tree_nodes.h @@ -2219,6 +2219,10 @@ typedef PT_index_option PT_fulltext_index_parser_name; typedef PT_index_option PT_index_visibility; +typedef PT_index_option PT_index_vector_m; +typedef PT_index_option + PT_index_vector_distance; + /** The data structure (B-tree, Hash, etc) used for an index is called 'index_type' in the manual. Internally, this is stored in diff --git a/sql/psi_memory_key.cc b/sql/psi_memory_key.cc index 4633f5afd41..6e577200847 100644 --- a/sql/psi_memory_key.cc +++ b/sql/psi_memory_key.cc @@ -145,6 +145,7 @@ PSI_memory_key key_memory_user_var_entry; PSI_memory_key key_memory_user_var_entry_value; PSI_memory_key key_memory_sp_cache; PSI_memory_key key_memory_write_set_extraction; +PSI_memory_key key_memory_vidx_mem; #ifdef HAVE_PSI_INTERFACE @@ -370,6 +371,7 @@ static PSI_memory_info all_server_memory[] = { {&key_memory_log_sink_pfs, "log_sink_pfs", PSI_FLAG_ONLY_GLOBAL_STAT, 0, PSI_DOCUMENT_ME}, {&key_memory_histograms, "histograms", 0, 0, PSI_DOCUMENT_ME}, + {&key_memory_vidx_mem, "vector_index", 0, 0, PSI_DOCUMENT_ME}, {&key_memory_hash_join, "hash_join", PSI_FLAG_MEM_COLLECT, 0, PSI_DOCUMENT_ME}, {&key_memory_rm_table_foreach_root, "rm_table::foreach_root", diff --git a/sql/psi_memory_key.h b/sql/psi_memory_key.h index fddd758d594..7f2b7cd95c0 100644 --- a/sql/psi_memory_key.h +++ b/sql/psi_memory_key.h @@ -171,5 +171,5 @@ extern PSI_memory_key key_memory_user_var_entry; extern PSI_memory_key key_memory_user_var_entry_value; extern PSI_memory_key key_memory_sp_cache; extern PSI_memory_key key_memory_write_set_extraction; - +extern PSI_memory_key key_memory_vidx_mem; #endif // PSI_MEMORY_KEY_INCLUDED diff --git a/sql/server_component/table_access_service.cc b/sql/server_component/table_access_service.cc index 683cb5d9d29..2a9c58f4521 100644 --- a/sql/server_component/table_access_service.cc +++ b/sql/server_component/table_access_service.cc @@ -1025,7 +1025,7 @@ int impl_index_init(Table_access /* api_ta */, TA_table api_table, *api_key = nullptr; - for (index = 0; index < share->keys; index++) { + for (index = 0; index < share->total_keys; index++) { key_info = &table->key_info[index]; /* FIXME: KEY::name has no associated length member */ diff --git a/sql/sql_alter.h b/sql/sql_alter.h index 5465e833eb1..5c1382b8cc4 100644 --- a/sql/sql_alter.h +++ b/sql/sql_alter.h @@ -497,7 +497,8 @@ class Alter_info { Value_generator *default_val_expr, const char *opt_after, std::optional srid, Sql_check_constraint_spec_list *check_cons_list, - dd::Column::enum_hidden_type hidden, bool is_array = false); + dd::Column::enum_hidden_type hidden, bool is_vector, + bool is_array = false); private: Alter_info &operator=(const Alter_info &rhs); // not implemented diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 13b7a0454d1..939693152cd 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -989,6 +989,8 @@ void release_table_share(TABLE_SHARE *share) { assert(share->ref_count() != 0); if (share->decrement_ref_count() == 0) { + assert(share->hlindex == nullptr || share->hlindex->ref_count() == 0); + if (share->has_old_version() || table_def_shutdown_in_progress) table_def_cache->erase(to_string(share->table_cache_key)); else { @@ -1117,6 +1119,12 @@ OPEN_TABLE_LIST *list_open_tables(THD *thd, const char *db, const char *wild) { void intern_close_table(TABLE *table) { // Free all structures DBUG_TRACE; + + if (table->hlindex != nullptr) { + intern_close_table(table->hlindex); + table->hlindex = nullptr; + } + DBUG_PRINT("tcache", ("table: '%s'.'%s' %p", table->s ? table->s->db.str : "?", table->s ? table->s->table_name.str : "?", table)); @@ -2398,6 +2406,13 @@ void close_temporary_table(THD *thd, TABLE *table, bool free_share, void close_temporary(THD *thd, TABLE *table, bool free_share, bool delete_table) { + if (table->hlindex != nullptr) { + /* The temp table during DDL may have hlindexes. */ + close_temporary(thd, table->hlindex, free_share, delete_table); + table->hlindex = nullptr; + table->s->hlindex = nullptr; + } + handlerton *table_type = table->s->db_type(); DBUG_TRACE; DBUG_PRINT("tmptable", ("closing table: '%s'.'%s'", table->s->db.str, diff --git a/sql/sql_builtin.cc.in b/sql/sql_builtin.cc.in index 8dc70d0b2f4..6a46bbda111 100644 --- a/sql/sql_builtin.cc.in +++ b/sql/sql_builtin.cc.in @@ -31,7 +31,7 @@ extern "C" extern #endif builtin_plugin - @mysql_mandatory_plugins@ @mysql_optional_plugins@ builtin_binlog_plugin, builtin_mysql_password_plugin, builtin_caching_sha2_password_plugin, builtin_daemon_keyring_proxy_plugin; + @mysql_mandatory_plugins@ @mysql_optional_plugins@ builtin_binlog_plugin, builtin_mysql_password_plugin, builtin_caching_sha2_password_plugin, builtin_daemon_keyring_proxy_plugin, builtin_vidx_plugin; struct st_mysql_plugin *mysql_optional_plugins[]= { @@ -40,5 +40,5 @@ struct st_mysql_plugin *mysql_optional_plugins[]= struct st_mysql_plugin *mysql_mandatory_plugins[]= { - builtin_binlog_plugin, builtin_mysql_password_plugin, builtin_caching_sha2_password_plugin, builtin_daemon_keyring_proxy_plugin, @mysql_mandatory_plugins@ 0 + builtin_binlog_plugin, builtin_mysql_password_plugin, builtin_caching_sha2_password_plugin, builtin_daemon_keyring_proxy_plugin, builtin_vidx_plugin, @mysql_mandatory_plugins@ 0 }; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 07053313a4b..f731d73d3d2 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -3088,6 +3088,8 @@ void THD::notify_hton_post_release_exclusive(const MDL_key *mdl_key) { Then, transform the parse tree further into an AST, ready for resolving. */ bool THD::sql_parser() { + m_query_has_vector_column = false; + /* SQL parser function generated by YACC from sql_yacc.yy. diff --git a/sql/sql_class.h b/sql/sql_class.h index c3dd6188b79..809f7f6cee7 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -4742,8 +4742,12 @@ class THD : public MDL_context_owner, std::unordered_map external_store_; THD_rds_context m_rds_context; + public: THD_rds_context &get_rds_context() { return m_rds_context; } + + /* Indicates if query has vector column. */ + bool m_query_has_vector_column{false}; }; // End of class THD /** diff --git a/sql/sql_executor.cc b/sql/sql_executor.cc index a905c1f0b50..4e5eb1f83a5 100644 --- a/sql/sql_executor.cc +++ b/sql/sql_executor.cc @@ -3791,7 +3791,7 @@ AccessPath *QEP_TAB::access_path() { case JT_INDEX_SCAN: path = NewIndexScanAccessPath(join()->thd, table(), index(), use_order(), m_reversed_access, - /*count_examined_rows=*/true); + /*count_examined_rows=*/true, vec_func()); break; case JT_ALL: case JT_RANGE: diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 6520da9901f..4d735ca370f 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -75,6 +75,7 @@ #include "sql/window.h" #include "sql_update.h" // Sql_cmd_update #include "template_utils.h" +#include "vidx/vidx_common.h" // RDS_COMMENT_VERSION class PT_hint_list; @@ -1872,7 +1873,7 @@ static int lex_one_token(Lexer_yystype *yylval, THD *thd) { ulong version; version = strtol(version_str, nullptr, 10); - if (version <= MYSQL_VERSION_ID) { + if (version <= MYSQL_VERSION_ID || version >= RDS_COMMENT_VERSION) { /* Accept 'M' 'm' 'm' 'd' 'd' */ lip->yySkipn(5); /* Expand the content of the special comment as real code */ diff --git a/sql/sql_opt_exec_shared.h b/sql/sql_opt_exec_shared.h index fca770253da..0ee65e8894d 100644 --- a/sql/sql_opt_exec_shared.h +++ b/sql/sql_opt_exec_shared.h @@ -311,6 +311,8 @@ class QEP_shared { table_map added_tables() const { return added_tables_map; } Item_func_match *ft_func() const { return m_ft_func; } void set_ft_func(Item_func_match *f) { m_ft_func = f; } + void *vec_func() const { return m_vec_func; } + void set_vec_func(void *f) { m_vec_func = f; } // More elaborate functions: @@ -467,6 +469,9 @@ class QEP_shared { /** FT function */ Item_func_match *m_ft_func; + /** Vector distance function */ + void *m_vec_func = nullptr; + /** Set if index dive can be skipped for this query. See comments for check_skip_records_in_range_qualification. @@ -540,6 +545,8 @@ class QEP_shared_owner { table_map added_tables() const { return m_qs->added_tables(); } Item_func_match *ft_func() const { return m_qs->ft_func(); } void set_ft_func(Item_func_match *f) { return m_qs->set_ft_func(f); } + void *vec_func() const { return m_qs->vec_func(); } + void set_vec_func(void *f) { return m_qs->set_vec_func(f); } void set_prefix_tables(table_map prefix_tables, table_map prev_tables) { return m_qs->set_prefix_tables(prefix_tables, prev_tables); } diff --git a/sql/sql_optimizer.cc b/sql/sql_optimizer.cc index a0bdecdee62..94185ce5d82 100644 --- a/sql/sql_optimizer.cc +++ b/sql/sql_optimizer.cc @@ -73,6 +73,7 @@ #include "sql/item_cmpfunc.h" #include "sql/item_func.h" #include "sql/item_row.h" +#include "sql/item_strfunc.h" // vidx::Item_func_vec_distance #include "sql/item_subselect.h" #include "sql/item_sum.h" // Item_sum #include "sql/iterators/basic_row_iterators.h" @@ -113,6 +114,7 @@ #include "sql/window.h" #include "sql_string.h" #include "template_utils.h" +#include "vidx/vidx_index.h" // vidx::test_if_cheaper_vector_ordering using std::ceil; using std::max; @@ -1194,7 +1196,7 @@ bool substitute_gc(THD *thd, Query_block *query_block, Item *where_cond, // Collect all GCs that are a part of a key for (Table_ref *tl = query_block->leaf_tables; tl; tl = tl->next_leaf) { - if (tl->table->s->keys == 0) continue; + if (tl->table->s->total_keys == 0) continue; for (uint i = 0; i < tl->table->s->fields; i++) { Field *fld = tl->table->field[i]; if (fld->is_gcol() && @@ -1759,7 +1761,7 @@ bool is_prefix_index(TABLE *table, uint idx) { if (key_part->field && !(table->field[key_part->fieldnr - 1] ->part_of_prefixkey.is_clear_all()) && - !(key_info->flags & (HA_FULLTEXT | HA_SPATIAL))) { + !(key_info->flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR))) { return true; } } @@ -1967,7 +1969,7 @@ uint find_shortest_key(TABLE *table, const Key_map *usable_keys) { : MAX_KEY; if (!usable_keys->is_clear_all()) { uint min_length = (uint)~0; - for (uint nr = 0; nr < table->s->keys; nr++) { + for (uint nr = 0; nr < table->s->total_keys; nr++) { if (nr == usable_clustered_pk) continue; if (usable_keys->is_set(nr)) { /* @@ -2078,7 +2080,7 @@ static uint test_if_subkey(ORDER_with_src *order, JOIN_TAB *tab, uint ref, KEY_PART_INFO *ref_key_part = table->key_info[ref].key_part; KEY_PART_INFO *ref_key_part_end = ref_key_part + ref_key_parts; - for (nr = 0; nr < table->s->keys; nr++) { + for (nr = 0; nr < table->s->total_keys; nr++) { bool skip_quick; if (usable_keys->is_set(nr) && table->key_info[nr].key_length < min_length && @@ -2275,6 +2277,11 @@ static bool test_if_skip_sort_order(JOIN_TAB *tab, ORDER_with_src &order, return true; } } + + if (!ft_func && vidx::test_if_cheaper_vector_ordering( + tab, order.order, select_limit, order_idx)) { + return true; + } } /* @@ -7060,7 +7067,7 @@ static void warn_index_not_applicable(THD *thd, const Field *field, if (thd->lex->is_explain() || thd->variables.option_bits & OPTION_SAFE_UPDATES) - for (uint j = 0; j < field->table->s->keys; j++) + for (uint j = 0; j < field->table->s->total_keys; j++) if (cant_use_index.is_set(j)) push_warning_printf(thd, Sql_condition::SL_WARNING, ER_WARN_INDEX_NOT_APPLICABLE, @@ -7779,9 +7786,9 @@ static bool add_key_part(Key_use_array *keyuse_array, Key_field *key_field) { Table_ref *const tl = key_field->item_field->table_ref; TABLE *const table = tl->table; - for (uint key = 0; key < table->s->keys; key++) { + for (uint key = 0; key < table->s->total_keys; key++) { if (!(table->keys_in_use_for_query.is_set(key))) continue; - if (table->key_info[key].flags & (HA_FULLTEXT | HA_SPATIAL)) + if (table->key_info[key].flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR)) continue; // ToDo: ft-keys in non-ft queries. SerG uint key_parts = actual_key_parts(&table->key_info[key]); @@ -8125,7 +8132,7 @@ static void trace_indexes_added_group_distinct(Opt_trace_context *trace, KEY *key_info = join_tab->table()->key_info; Key_map existing_keys = join_tab->const_keys; - uint nbrkeys = join_tab->table()->s->keys; + uint nbrkeys = join_tab->table()->s->total_keys; Opt_trace_object trace_summary(trace, "const_keys_added"); { @@ -9306,7 +9313,7 @@ void JOIN::finalize_derived_keys() { */ if (table == nullptr || !tr->uses_materialization() || // (1) table->is_created() || // (2) - table->s->keys == 0 || // (3) + table->s->total_keys == 0 || // (3) (processed_tables & tr->map())) { // (4) continue; } @@ -9314,9 +9321,9 @@ void JOIN::finalize_derived_keys() { Collect all used keys before starting to shuffle them: First create a map from key number to the table using the key: */ - assert(table->s->keys <= MAX_INDEXES); + assert(table->s->total_keys <= MAX_INDEXES); TABLE *table_map[MAX_INDEXES]; - for (uint j = 0; j < table->s->keys; j++) { + for (uint j = 0; j < table->s->total_keys; j++) { table_map[j] = nullptr; } @@ -9326,7 +9333,7 @@ void JOIN::finalize_derived_keys() { // (deduplication) whether any expression refers to them or not. // In particular, they are used if we want to materialize a UNION DISTINCT // directly into the derived table. - for (uint key_idx = 0; key_idx < table->s->keys; ++key_idx) { + for (uint key_idx = 0; key_idx < table->s->total_keys; ++key_idx) { if (table->key_info[key_idx].flags & HA_NOSAME) { used_keys.set_bit(key_idx); } @@ -9361,7 +9368,7 @@ void JOIN::finalize_derived_keys() { (void)table->s->find_first_unused_tmp_key(used_keys); // Process keys in increasing key order - for (uint j = 0; j < table->s->keys; j++) { + for (uint j = 0; j < table->s->total_keys; j++) { TABLE *const t = table_map[j]; if (t == nullptr) continue; @@ -10632,7 +10639,7 @@ static bool list_contains_unique_index(JOIN_TAB *tab, TABLE *table = tab->table(); if (tab->is_inner_table_of_outer_join()) return false; - for (uint keynr = 0; keynr < table->s->keys; keynr++) { + for (uint keynr = 0; keynr < table->s->total_keys; keynr++) { if (keynr == table->s->primary_key || (table->key_info[keynr].flags & HA_NOSAME)) { KEY *keyinfo = table->key_info + keynr; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index ffecbf0266b..be8c24857cd 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -5516,6 +5516,7 @@ bool mysql_test_parse_for_slave(THD *thd) { @param col_check_const_spec_list List of column check constraints. @param hidden Column hidden type. @param is_array Whether it's a typed array field + @param is_vector Whether it's a typed vector field @return Return 0 if ok @@ -5529,7 +5530,7 @@ bool Alter_info::add_field( Value_generator *gcol_info, Value_generator *default_val_expr, const char *opt_after, std::optional srid, Sql_check_constraint_spec_list *col_check_const_spec_list, - dd::Column::enum_hidden_type hidden, bool is_array) { + dd::Column::enum_hidden_type hidden, bool is_vector, bool is_array) { uint8 datetime_precision = decimals ? atoi(decimals) : 0; DBUG_TRACE; assert(!is_array || hidden == dd::Column::enum_hidden_type::HT_HIDDEN_SQL); @@ -5630,7 +5631,7 @@ bool Alter_info::add_field( type_modifier, default_value, on_update_value, comment, change, interval_list, cs, has_explicit_collation, uint_geom_type, gcol_info, default_val_expr, srid, hidden, - is_array)) + is_array, is_vector)) return true; for (const auto &a : cf_appliers) { diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 5369a993367..73ca53d89b3 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -3802,6 +3802,10 @@ void JOIN::join_free() { } static void cleanup_table(TABLE *table) { + if (table->hlindex && table->hlindex->context) { + table->hlindex_read_end(); + } + if (table->is_created()) { table->file->ha_index_or_rnd_end(); } @@ -4989,7 +4993,7 @@ bool JOIN::make_tmp_tables_info() { /*sort_before_group=*/false)) return true; } - if (!tab->filesort && !tab->table()->s->keys && + if (!tab->filesort && !tab->table()->s->total_keys && (!(query_block->active_options() & OPTION_BUFFER_RESULT) || need_tmp_before_win || wno >= 1)) { /* @@ -5252,7 +5256,7 @@ bool test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER_with_src *order, assert(refkey_rows_estimate >= 1.0); } - for (nr = 0; nr < table->s->keys; nr++) { + for (nr = 0; nr < table->s->total_keys; nr++) { int direction = 0; uint used_key_parts; bool skip_quick; diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 1fecaeb94a0..1ccbacab30d 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -140,6 +140,7 @@ #include "sql_string.h" #include "template_utils.h" #include "thr_lock.h" +#include "vidx/vidx_index.h" /* @see dynamic_privileges_table.cc */ bool iterate_all_dynamic_privileges(THD *thd, @@ -2225,9 +2226,13 @@ bool store_create_info(THD *thd, Table_ref *table_list, String *packet, file->update_create_info(&create_info); primary_key = share->primary_key; - for (uint i = skip_gipk ? 1 : 0; i < share->keys; i++, key_info++) { + for (uint i = skip_gipk ? 1 : 0; i < share->total_keys; i++, key_info++) { KEY_PART_INFO *key_part = key_info->key_part; bool found_primary = false; + + if (key_info->flags & HA_VECTOR) + packet->append(STRING_WITH_LEN(RDS_COMMENT_VIDX_START)); + packet->append(STRING_WITH_LEN(",\n ")); if (i == primary_key && !strcmp(key_info->name, primary_key_name)) { @@ -2241,6 +2246,8 @@ bool store_create_info(THD *thd, Table_ref *table_list, String *packet, packet->append(STRING_WITH_LEN("UNIQUE KEY ")); else if (key_info->flags & HA_FULLTEXT) packet->append(STRING_WITH_LEN("FULLTEXT KEY ")); + else if (key_info->flags & HA_VECTOR) + packet->append(STRING_WITH_LEN("VECTOR KEY ")); else if (key_info->flags & HA_SPATIAL) packet->append(STRING_WITH_LEN("SPATIAL KEY ")); else @@ -2275,7 +2282,7 @@ bool store_create_info(THD *thd, Table_ref *table_list, String *packet, if (key_part->field && (key_part->length != table->field[key_part->fieldnr - 1]->key_length() && - !(key_info->flags & (HA_FULLTEXT | HA_SPATIAL)))) { + !(key_info->flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR)))) { packet->append_parenthesized((long)key_part->length / key_part->field->charset()->mbmaxlen); } @@ -2619,7 +2626,7 @@ bool store_create_info(THD *thd, Table_ref *table_list, String *packet, static void store_key_options(THD *thd, String *packet, TABLE *table, KEY *key_info) { bool foreign_db_mode = (thd->variables.sql_mode & MODE_ANSI) != 0; - char *end, buff[32]; + char *end, buff[40]; if (!foreign_db_mode) { /* @@ -2640,6 +2647,13 @@ static void store_key_options(THD *thd, String *packet, TABLE *table, } } + if (key_info->flags & HA_VECTOR) { + uint len = vidx::hnsw::index_options_print( + key_info->vector_distance, key_info->vector_m, buff, sizeof(buff)); + + packet->append(buff, len); + } + if ((key_info->flags & HA_USES_BLOCK_SIZE) && table->s->key_block_size != key_info->block_size) { packet->append(STRING_WITH_LEN(" KEY_BLOCK_SIZE=")); @@ -4287,7 +4301,7 @@ static int get_schema_tmp_table_keys_record(THD *thd, Table_ref *tables, key_info++; } - for (; i < show_table->s->keys; i++, key_info++) { + for (; i < show_table->s->total_keys; i++, key_info++) { KEY_PART_INFO *key_part = key_info->key_part; const char *str; for (uint j = 0; j < key_info->user_defined_key_parts; j++, key_part++) { @@ -4367,7 +4381,7 @@ static int get_schema_tmp_table_keys_record(THD *thd, Table_ref *tables, table->field[TMP_TABLE_KEYS_INDEX_TYPE]->store(str, strlen(str), cs); // SUB_PART - if (!(key_info->flags & HA_FULLTEXT) && + if (!(key_info->flags & (HA_FULLTEXT | HA_VECTOR)) && (key_part->field && key_part->length != show_table->s->field[key_part->fieldnr - 1]->key_length())) { diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 8e804d3d790..f5f298b204c 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -191,6 +191,7 @@ #include "sql/duckdb/duckdb_table.h" #include "sql/sql_table_ext.h" +#include "vidx/vidx_index.h" namespace dd { class View; @@ -511,7 +512,7 @@ class Disable_slave_info_update_guard { } }; -static bool trans_intermediate_ddl_commit(THD *thd, bool error) { +bool trans_intermediate_ddl_commit(THD *thd, bool error) { // Must be used for intermediate (but not final) DDL commits. Implicit_substatement_state_guard substatement_guard(thd); if (error) { @@ -998,9 +999,12 @@ static bool rea_create_tmp_table( return false; } + assert(!vidx::dd_table_has_hlindexes(tmp_table_ptr.get())); + // Create the table in the storage engine. if (ha_create_table(thd, path, db, table_name, create_info, false, false, - tmp_table_ptr.get())) { + tmp_table_ptr.get(), + keys == 0 ? nullptr : key_info + keys - 1)) { return true; } @@ -1178,7 +1182,7 @@ static bool rea_create_base_table( *post_ddl_ht = create_info->db_type; if (ha_create_table(thd, path, db, table_name, create_info, false, false, - table_def)) { + table_def, keys == 0 ? nullptr : key_info + keys - 1)) { /* Remove table from data-dictionary if it was added and rollback won't do this automatically. @@ -3922,8 +3926,12 @@ namespace { struct sort_keys { bool operator()(const KEY &a, const KEY &b) const { + // Sort VECTOR after not VECTOR. + if ((a.flags ^ b.flags) & HA_VECTOR) return b.flags & HA_VECTOR; + // std::sort may compare an element to itself: if (&a == &b) return false; + // Sort UNIQUE before not UNIQUE. if ((a.flags ^ b.flags) & HA_NOSAME) return a.flags & HA_NOSAME; @@ -4828,7 +4836,7 @@ static bool count_keys(const Mem_root_array &key_list, */ if ((key2->type != KEYTYPE_FOREIGN && key->type != KEYTYPE_FOREIGN && key2->type != KEYTYPE_SPATIAL && key2->type != KEYTYPE_FULLTEXT && - !redundant_keys->at(key2_counter) && + key2->type != KEYTYPE_VECTOR && !redundant_keys->at(key2_counter) && !foreign_key_prefix(key, key2))) { /* TODO: issue warning message */ /* mark that the generated key should be ignored */ @@ -4954,6 +4962,14 @@ static bool prepare_key_column(THD *thd, HA_CREATE_INFO *create_info, return true; } + if (key->type == KEYTYPE_VECTOR && + (!sql_field->is_vector || (sql_field->flags & FIELD_IS_INVISIBLE) || + sql_field->is_virtual_gcol())) { + my_error(ER_VECTOR_INDEX_USAGE, MYF(0), + "Only support one visible VECTOR column"); + return true; + } + if (sql_field->auto_flags & Field::NEXT_NUMBER) { if (column_nr == 0 || (file->ha_table_flags() & HA_AUTO_PART_KEY)) (*auto_increment)--; // Field is used @@ -5233,7 +5249,7 @@ static bool prepare_key_column(THD *thd, HA_CREATE_INFO *create_info, } if (key_part_length > file->max_key_part_length(create_info) && - key->type != KEYTYPE_FULLTEXT) { + key->type != KEYTYPE_FULLTEXT && key->type != KEYTYPE_VECTOR) { key_part_length = file->max_key_part_length(create_info); if (key->type == KEYTYPE_MULTIPLE) { /* not a critical problem */ @@ -5531,7 +5547,7 @@ static bool prepare_self_ref_fk_parent_key( for (const KEY *key = key_info_buffer; key < key_info_buffer + key_count; key++) { // We can't use FULLTEXT or SPATIAL indexes. - if (key->flags & (HA_FULLTEXT | HA_SPATIAL)) continue; + if (key->flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR)) continue; if (hton->foreign_keys_flags & HTON_FKS_NEED_DIFFERENT_PARENT_AND_SUPPORTING_KEYS) { @@ -5723,7 +5739,7 @@ static const KEY *find_fk_supporting_key(handlerton *hton, for (const KEY *key = key_info_buffer; key < key_info_buffer + key_count; key++) { // We can't use FULLTEXT or SPATIAL indexes. - if (key->flags & (HA_FULLTEXT | HA_SPATIAL)) continue; + if (key->flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR)) continue; if (key->algorithm == HA_KEY_ALG_HASH) { if (hton->foreign_keys_flags & HTON_FKS_WITH_SUPPORTING_HASH_KEYS) { @@ -7324,6 +7340,26 @@ static bool prepare_key( } key_info->flags |= HA_SPATIAL; break; + case KEYTYPE_VECTOR: + if (vidx::feature_disabled) { + my_error(ER_VECTOR_DISABLED, MYF(0)); + return true; + } + + if (vidx::copy_index_option_distance( + thd, &(key_info->vector_distance), + key->key_create_info.vector_distance) || + vidx::hnsw::copy_index_option_m(thd, &(key_info->vector_m), + key->key_create_info.vector_m)) { + my_error(ER_VECTOR_INDEX_USAGE, MYF(0), "Invalid options."); + return true; + } + if (!key->key_create_info.is_visible) { + my_error(ER_VECTOR_INDEX_USAGE, MYF(0), "Must be visible."); + return true; + } + key_info->flags |= HA_VECTOR; + break; case KEYTYPE_PRIMARY: case KEYTYPE_UNIQUE: key_info->flags |= HA_NOSAME; @@ -7359,6 +7395,9 @@ static bool prepare_key( assert(!key->key_create_info.is_algorithm_explicit); key_info->algorithm = HA_KEY_ALG_FULLTEXT; } else { + assert(!(key_info->flags & HA_VECTOR) || + !key->key_create_info.is_algorithm_explicit); + if (key->key_create_info.is_algorithm_explicit) { if (key->key_create_info.algorithm != HA_KEY_ALG_RTREE) { /* @@ -7421,7 +7460,7 @@ static bool prepare_key( key_info->actual_flags = key_info->flags; if (key_info->key_length > file->max_key_length() && - key->type != KEYTYPE_FULLTEXT) { + key->type != KEYTYPE_FULLTEXT && key->type != KEYTYPE_VECTOR) { my_error(ER_TOO_LONG_KEY, MYF(0), file->max_key_length()); if (thd->is_error()) // May be silenced - see Bug#20629014 return true; @@ -8076,6 +8115,28 @@ bool mysql_prepare_create_table( for (Key_spec *key : alter_info->key_list) { if (key->type == KEYTYPE_FOREIGN) continue; + if (key->type == KEYTYPE_VECTOR) { +#ifndef NDEBUG + assert(key->columns.size() == 1); + assert(!key->columns[0]->has_expression()); + assert(key->columns[0]->get_field_name() != nullptr); +#endif /* !NDEBUG */ + + if (is_partitioned || create_info->options & HA_LEX_CREATE_TMP_TABLE) { + my_error(ER_NOT_SUPPORTED_YET, MYF(0), + "the VECTOR index in partitioned or temp tables"); + return true; + } + + if (!(create_info->db_type->flags & HTON_SUPPORTS_ATOMIC_DDL)) { + my_error(ER_NOT_SUPPORTED_YET, MYF(0), + "the VECTOR index in engine not supporting atomic DDL"); + return true; + } + + continue; + } + for (size_t j = 0; j < key->columns.size(); ++j) { Key_part_spec *key_part_spec = key->columns[j]; // In the case of procedures, the Key_part_spec may both have an @@ -8248,6 +8309,12 @@ bool mysql_prepare_create_table( /* Sort keys in optimized order */ std::sort(*key_info_buffer, *key_info_buffer + *key_count, sort_keys()); + /* Check if there are multiple VECTOR indexes. */ + if (*key_count >= 2 && (*key_info_buffer)[*key_count - 2].flags & HA_VECTOR) { + my_error(ER_NOT_SUPPORTED_YET, MYF(0), "multiple VECTOR indexes"); + return true; + } + /* Normal keys are done, now prepare foreign keys. @@ -10232,7 +10299,9 @@ bool mysql_create_table(THD *thd, Table_ref *create_table, variable, but rely on logging what really has been done instead. */ if ((create_table->table == nullptr && !create_table->is_view()) && - is_pk_generated) { + (is_pk_generated || thd->m_query_has_vector_column)) { + thd->m_query_has_vector_column = false; + /* Open table to generate CREATE TABLE statement. For non-temporary table we already have exclusive lock here. @@ -10709,7 +10778,9 @@ bool mysql_rename_table(THD *thd, handlerton *base, const char *old_db, to_table_def->set_name(new_name); to_table_def->set_hidden((flags & FN_TO_IS_TMP) ? dd::Abstract_table::HT_HIDDEN_DDL - : dd::Abstract_table::HT_VISIBLE); + : ((flags & VIDX_RENAME) + ? dd::Abstract_table::HT_HIDDEN_HLINDEX + : dd::Abstract_table::HT_VISIBLE)); /* Adjust parent table for self-referencing foreign keys. */ for (dd::Foreign_key *fk : *(to_table_def->foreign_keys())) { @@ -10801,6 +10872,15 @@ bool mysql_rename_table(THD *thd, handlerton *base, const char *old_db, return true; } + /* Rename vidx table if db is changed. the vidx table name is not changed + because the base table id is not changed. */ + if (vidx::dd_table_has_hlindexes(to_table_def) && + my_strcasecmp(table_alias_charset, old_db, new_db) != 0 && + vidx::rename_table(thd, to_table_def, base, new_schema, old_db, new_db, + flags)) { + return true; + } + /* Note that before WL#7743 we have renamed table in the data-dictionary before renaming it in storage engine. However with WL#7743 engines @@ -11979,7 +12059,7 @@ static bool fill_alter_inplace_info(THD *thd, TABLE *table, /* Allocate result buffers. */ if (!(ha_alter_info->index_drop_buffer = - (KEY **)thd->alloc(sizeof(KEY *) * table->s->keys)) || + (KEY **)thd->alloc(sizeof(KEY *) * table->s->total_keys)) || !(ha_alter_info->index_add_buffer = (uint *)thd->alloc(sizeof(uint) * alter_info->key_list.size())) || !(ha_alter_info->index_rename_buffer = (KEY_PAIR *)thd->alloc( @@ -12338,11 +12418,11 @@ static bool fill_alter_inplace_info(THD *thd, TABLE *table, with new table. */ KEY *table_key; - KEY *table_key_end = table->key_info + table->s->keys; + KEY *table_key_end = table->key_info + table->s->total_keys; KEY *new_key; KEY *new_key_end = ha_alter_info->key_info_buffer + ha_alter_info->key_count; - DBUG_PRINT("info", ("index count old: %d new: %d", table->s->keys, + DBUG_PRINT("info", ("index count old: %d new: %d", table->s->total_keys, ha_alter_info->key_count)); /* @@ -12682,7 +12762,7 @@ bool mysql_compare_tables(THD *thd, TABLE *table, Alter_info *alter_info, /* Go through keys and check if they are compatible. */ KEY *table_key; - KEY *table_key_end = table->key_info + table->s->keys; + KEY *table_key_end = table->key_info + table->s->total_keys; KEY *new_key; KEY *new_key_end = key_info_buffer + key_count; @@ -15011,7 +15091,7 @@ bool prepare_fields_and_keys(THD *thd, const dd::Table *src_table, TABLE *table, for which some fields exists. */ - for (uint i = 0; i < table->s->keys; i++, key_info++) { + for (uint i = 0; i < table->s->total_keys; i++, key_info++) { const char *key_name = key_info->name; bool index_column_dropped = false; size_t drop_idx = 0; @@ -15202,7 +15282,11 @@ bool prepare_fields_and_keys(THD *thd, const dd::Table *src_table, TABLE *table, key_type = KEYTYPE_UNIQUE; } else if (key_info->flags & HA_FULLTEXT) key_type = KEYTYPE_FULLTEXT; - else + else if (key_info->flags & HA_VECTOR) { + key_type = KEYTYPE_VECTOR; + key_create_info.vector_m = key_info->vector_m; + key_create_info.vector_distance = key_info->vector_distance; + } else key_type = KEYTYPE_MULTIPLE; /* @@ -16117,7 +16201,7 @@ static bool is_alter_geometry_column_valid(Alter_info *alter_info) { Check if there is a spatial index on this column. If that is the case, reject the change. */ - for (uint i = 0; i < share->keys; ++i) { + for (uint i = 0; i < share->total_keys; ++i) { if (geom_field->key_start.is_set(i) && share->key_info[i].flags & HA_SPATIAL) { my_error(ER_CANNOT_ALTER_SRID_DUE_TO_INDEX, MYF(0), @@ -16197,7 +16281,7 @@ static bool handle_drop_functional_index(THD *thd, Alter_info *alter_info, // the source column as well. for (const Alter_drop *drop : alter_info->drop_list) { if (drop->type == Alter_drop::KEY) { - for (uint j = 0; j < table_list->table->s->keys; j++) { + for (uint j = 0; j < table_list->table->s->total_keys; j++) { const KEY &key_info = table_list->table->s->key_info[j]; if (my_strcasecmp(system_charset_info, key_info.name, drop->name) == 0) { @@ -16251,7 +16335,7 @@ static bool handle_rename_functional_index(THD *thd, Alter_info *alter_info, for (const Alter_rename_key *alter_rename_key : alter_info->alter_rename_key_list) { // Find the matching existing index - for (uint j = 0; j < table_list->table->s->keys; ++j) { + for (uint j = 0; j < table_list->table->s->total_keys; ++j) { const KEY &key = table_list->table->s->key_info[j]; if (my_strcasecmp(system_charset_info, key.name, alter_rename_key->old_name) == 0) { @@ -17434,6 +17518,11 @@ bool mysql_alter_table(THD *thd, const char *new_db, const char *new_name, goto err_new_table_cleanup; } + if (vidx::check_vector_ddl_and_rewrite_sql(thd, alter_info, key_info, + key_count, table)) { + goto err_new_table_cleanup; + } + if (alter_info->requested_algorithm != Alter_info::ALTER_TABLE_ALGORITHM_COPY) { Alter_inplace_info ha_alter_info(create_info, alter_info, @@ -17511,8 +17600,11 @@ bool mysql_alter_table(THD *thd, const char *new_db, const char *new_name, // Ask storage engine whether to use copy or in-place enum_alter_inplace_result inplace_supported = - table->file->check_if_supported_inplace_alter(altered_table, - &ha_alter_info); + ((key_count > 0 && vidx::key_is_vector(key_info + key_count - 1)) || + table->s->hlindexes() > 0) + ? HA_ALTER_INPLACE_NOT_SUPPORTED + : table->file->check_if_supported_inplace_alter(altered_table, + &ha_alter_info); // If INSTANT was requested but it is not supported, report error. if (alter_info->requested_algorithm == @@ -17759,8 +17851,8 @@ bool mysql_alter_table(THD *thd, const char *new_db, const char *new_name, } if (ha_create_table(thd, alter_ctx.get_tmp_path(), alter_ctx.new_db, - alter_ctx.tmp_name, create_info, false, true, - table_def)) + alter_ctx.tmp_name, create_info, false, true, table_def, + key_count == 0 ? nullptr : key_info + key_count - 1)) goto err_new_table_cleanup; /* Mark that we have created table in storage engine. */ diff --git a/sql/sql_table.h b/sql/sql_table.h index 5894f4a9723..d201842eb55 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -89,6 +89,9 @@ static const uint NO_FK_RENAME = 1 << 4; /** Don't change generated check constraint names while renaming table. */ static const uint NO_CC_RENAME = 1 << 5; +/** rename aux table of vector index. */ +static const uint VIDX_RENAME = 1 << 6; + handlerton *get_viable_handlerton_for_create(THD *thd, const char *table_name, const HA_CREATE_INFO &ci); diff --git a/sql/sql_tmp_table.cc b/sql/sql_tmp_table.cc index 1a84bf55816..df69e9803ce 100644 --- a/sql/sql_tmp_table.cc +++ b/sql/sql_tmp_table.cc @@ -1253,7 +1253,7 @@ TABLE *create_tmp_table(THD *thd, Temp_table_param *param, if (group) { DBUG_PRINT("info", ("Creating group key in temporary table")); table->group = group; /* Table is grouped by key */ - share->keys = 1; + share->total_keys = share->keys = 1; // Let each group expression know the column which materializes its value for (ORDER *cur_group = group; cur_group; cur_group = cur_group->next) { Field *field = (*cur_group->item)->get_tmp_table_field(); @@ -1321,7 +1321,7 @@ TABLE *create_tmp_table(THD *thd, Temp_table_param *param, in the first 'hidden_null_pack_length' bytes of the row. */ DBUG_PRINT("info", ("hidden_field_count: %d", param->hidden_field_count)); - share->keys = 1; + share->total_keys = share->keys = 1; share->is_distinct = distinct || param->m_operation == Temp_table_param::TTP_INTERSECT || param->m_operation == Temp_table_param::TTP_EXCEPT; @@ -1433,7 +1433,7 @@ TABLE *create_tmp_table(THD *thd, Temp_table_param *param, param->schema_table)) return nullptr; /* purecov: inspected */ - if (table->s->keys == 1 && table->key_info) + if (table->s->total_keys == 1 && table->key_info) table->key_info->algorithm = table->file->get_default_index_algorithm(); table->hidden_field_count = param->hidden_field_count; @@ -1816,7 +1816,7 @@ TABLE *create_duplicate_weedout_tmp_table(THD *thd, uint uniq_tuple_length_arg, KEY *hash_key = keyinfo; KEY_PART_INFO *hash_kpi = key_part_info; - share->keys = 1; + share->total_keys = share->keys = 1; table->key_info = share->key_info = hash_key; hash_key->table = table; hash_key->key_part = hash_kpi; @@ -1825,7 +1825,7 @@ TABLE *create_duplicate_weedout_tmp_table(THD *thd, uint uniq_tuple_length_arg, hash_key->key_length = hash_kpi->store_length; } else { DBUG_PRINT("info", ("Creating group key in temporary table")); - share->keys = 1; + share->total_keys = share->keys = 1; table->key_info = table->s->key_info = keyinfo; keyinfo->key_part = key_part_info; keyinfo->actual_flags = keyinfo->flags = HA_NOSAME; @@ -2326,7 +2326,8 @@ static void trace_tmp_table(Opt_trace_context *trace, const TABLE *table) { trace_tmp.add("in_plan_at_position", tab->idx()); trace_tmp.add("columns", s->fields) .add("row_length", s->reclength) - .add("key_length", table->s->keys > 0 ? table->key_info->key_length : 0) + .add("key_length", + table->s->total_keys > 0 ? table->key_info->key_length : 0) .add("unique_constraint", table->hash_field ? true : false) .add("makes_grouped_rows", table->group != nullptr) .add("cannot_insert_duplicates", s->is_distinct); @@ -2397,7 +2398,7 @@ bool instantiate_tmp_table(THD *thd, TABLE *table) { return true; } - if (share->first_unused_tmp_key < share->keys) { + if (share->first_unused_tmp_key < share->total_keys) { /* Some other clone of this materialized temporary table has defined "possible" keys; as we are here creating the table in the engine, we must @@ -2405,7 +2406,7 @@ bool instantiate_tmp_table(THD *thd, TABLE *table) { now. As the other clone assumes they will be available if the Optimizer chooses them, we make them existing. */ - share->find_first_unused_tmp_key(Key_map(share->keys)); + share->find_first_unused_tmp_key(Key_map(share->total_keys)); } Opt_trace_context *const trace = &thd->opt_trace; diff --git a/sql/sql_truncate.cc b/sql/sql_truncate.cc index 46df886354f..30eb5a9835e 100644 --- a/sql/sql_truncate.cc +++ b/sql/sql_truncate.cc @@ -65,6 +65,7 @@ #include "sql/transaction_info.h" #include "sql_string.h" #include "thr_lock.h" +#include "vidx/vidx_index.h" namespace dd { class Table; @@ -694,6 +695,9 @@ void Sql_cmd_truncate_table::truncate_temporary(THD *thd, // Create a clone of the tdef which can be manipulated by ha_create_table Up_table tdef_clone = Up_table{tdef_holder->clone()}; + + assert(!vidx::dd_table_has_hlindexes(tdef_clone.get())); + m_error = ha_create_table(thd, saved_norm_path.c_str(), table_ref->db, table_ref->table_name, &create_info, true, true, tdef_clone.get()); diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 364e0b016ab..f3f424b16c8 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -2034,7 +2034,7 @@ static void cleanup_tmp_tables(Table_ref *list) { } else { // Clear indexes added during optimization, keep possible unique index TABLE *t = tl->table; - t->s->keys = t->s->is_distinct ? 1 : 0; + t->s->total_keys = t->s->keys = t->s->is_distinct ? 1 : 0; t->s->first_unused_tmp_key = 0; t->keys_in_use_for_query.clear_all(); t->keys_in_use_for_group_by.clear_all(); diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 2f12da5b7ca..0542e17220b 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1401,6 +1401,8 @@ void warn_on_deprecated_user_defined_collation( %token URL_SYM 1202 /* MYSQL */ %token GENERATE_SYM 1203 /* MYSQL */ +%token VECTOR_SYM 1215 + /* Precedence rules used to resolve the ambiguity when using keywords as idents in the case e.g.: @@ -1423,6 +1425,10 @@ void warn_on_deprecated_user_defined_collation( /* Toekns for RDS begin from 1300 */ %token REPLICATE_REWRITE_WILD_TABLE 1300 +%token M_SYM 1301 /* MYSQL */ +%token DISTANCE_SYM 1302 /* MYSQL */ +%token EUCLIDEAN_SYM 1303 /* MYSQL */ +%token COSINE_SYM 1304 /* MYSQL */ /* Resolve column attribute ambiguity -- force precedence of "UNIQUE KEY" against @@ -1512,6 +1518,7 @@ void warn_on_deprecated_user_defined_collation( view_check_option signed_num opt_ignore_unknown_user + vector_distance_name %type @@ -1994,10 +2001,11 @@ void warn_on_deprecated_user_defined_collation( %type alter_instance_action -%type key_list key_list_with_expression +%type key_list key_list_with_expression key_part_simple %type opt_index_options index_options opt_fulltext_index_options fulltext_index_options opt_spatial_index_options spatial_index_options + opt_vector_index_options vector_index_options %type opt_index_lock_and_algorithm @@ -2005,6 +2013,7 @@ void warn_on_deprecated_user_defined_collation( spatial_index_option index_type_clause opt_index_type_clause + vector_index_option %type alter_algorithm_option_value alter_algorithm_option @@ -3695,6 +3704,14 @@ create_index_stmt: $11.algo.get_or_default(), $11.lock.get_or_default()); } + | CREATE VECTOR_SYM INDEX_SYM ident ON_SYM table_ident + '(' key_part_simple ')' opt_vector_index_options opt_index_lock_and_algorithm + { + $$= NEW_PTN PT_create_index_stmt(YYMEM_ROOT, KEYTYPE_VECTOR, $4, + NULL, $6, $8, $10, + $11.algo.get_or_default(), + $11.lock.get_or_default()); + } | CREATE SPATIAL_SYM INDEX_SYM ident ON_SYM table_ident '(' key_list_with_expression ')' opt_spatial_index_options opt_index_lock_and_algorithm { @@ -4086,7 +4103,8 @@ sp_fdparam: cs ? cs : thd->variables.collation_database, $3 != nullptr, $2->get_uint_geom_type(), nullptr, nullptr, {}, - dd::Column::enum_hidden_type::HT_VISIBLE)) + dd::Column::enum_hidden_type::HT_VISIBLE, + false, $2->is_vector())) { MYSQL_YYABORT; } @@ -4147,7 +4165,8 @@ sp_pdparam: cs ? cs : thd->variables.collation_database, $4 != nullptr, $3->get_uint_geom_type(), nullptr, nullptr, {}, - dd::Column::enum_hidden_type::HT_VISIBLE)) + dd::Column::enum_hidden_type::HT_VISIBLE, + false, $3->is_vector())) { MYSQL_YYABORT; } @@ -4277,7 +4296,8 @@ sp_decl: cs ? cs : thd->variables.collation_database, $4 != nullptr, $3->get_uint_geom_type(), nullptr, nullptr, {}, - dd::Column::enum_hidden_type::HT_VISIBLE)) + dd::Column::enum_hidden_type::HT_VISIBLE, + false, $3->is_vector())) { MYSQL_YYABORT; } @@ -7024,6 +7044,11 @@ table_constraint_def: $$= NEW_PTN PT_inline_index_definition(KEYTYPE_FULLTEXT, $3, NULL, $5, $7); } + | VECTOR_SYM opt_key_or_index opt_ident '(' key_part_simple ')' + opt_vector_index_options + { + $$= NEW_PTN PT_inline_index_definition(KEYTYPE_VECTOR, $3, NULL, $5, $7); + } | SPATIAL_SYM opt_key_or_index opt_ident '(' key_list_with_expression ')' opt_spatial_index_options { @@ -7197,6 +7222,10 @@ type: { $$= NEW_PTN PT_char_type(Char_type::VARCHAR, $2, &my_charset_bin); } + | VECTOR_SYM field_length opt_vector_field_options + { + $$= NEW_PTN PT_vector_type(YYTHD, $2); + } | YEAR_SYM opt_field_length field_options { if ($2) @@ -7955,6 +7984,40 @@ fulltext_index_option: $$= NEW_PTN PT_fulltext_index_parser_name(to_lex_cstring($3)); } ; +opt_vector_field_options: + %empty + | VARBINARY_SYM field_length + ; + +opt_vector_index_options: + %empty { $$.init(YYMEM_ROOT); } + | vector_index_options + ; + +vector_index_options: + vector_index_option + { + $$.init(YYMEM_ROOT); + if ($$.push_back($1)) + MYSQL_YYABORT; // OOM + } + | vector_index_options vector_index_option + { + if ($1.push_back($2)) + MYSQL_YYABORT; // OOM + $$= $1; + } + ; + +vector_index_option: + common_index_option + | M_SYM opt_equal ulong_num { $$= NEW_PTN PT_index_vector_m($3); } + | DISTANCE_SYM opt_equal vector_distance_name { $$= NEW_PTN PT_index_vector_distance($3); } + ; + +vector_distance_name: + EUCLIDEAN_SYM { $$= 0; } + | COSINE_SYM { $$= 1; } opt_spatial_index_options: %empty { $$.init(YYMEM_ROOT); } @@ -8107,6 +8170,15 @@ key_part: } ; +key_part_simple: + ident + { + $$= NEW_PTN List; + if ($$ == NULL || $$->push_back(NEW_PTN PT_key_part_specification(to_lex_cstring($1), ORDER_NOT_RELEVANT, 0))) + MYSQL_YYABORT; + } + ; + key_list_with_expression: key_list_with_expression ',' key_part_with_expression { @@ -15626,6 +15698,10 @@ ident_keywords_unambiguous: | SECONDARY_UNLOAD_SYM | SECOND_SYM | SECURITY_SYM + | M_SYM + | DISTANCE_SYM + | EUCLIDEAN_SYM + | COSINE_SYM | SERIALIZABLE_SYM | SERIAL_SYM | SERVER_SYM @@ -17840,7 +17916,8 @@ sf_tail: cs ? cs : YYTHD->variables.collation_database, $11 != nullptr, $10->get_uint_geom_type(), nullptr, nullptr, {}, - dd::Column::enum_hidden_type::HT_VISIBLE)) + dd::Column::enum_hidden_type::HT_VISIBLE, + false, $10->is_vector())) { MYSQL_YYABORT; } diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 716b0452698..71e80ee851e 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -139,6 +139,7 @@ #include "sql/xa.h" #include "template_utils.h" // pointer_cast #include "thr_lock.h" +#include "vidx/vidx_common.h" #ifdef _WIN32 #include "sql/named_pipe.h" #endif diff --git a/sql/sys_vars_ext.cc b/sql/sys_vars_ext.cc index 023717db757..25f201c11e7 100644 --- a/sql/sys_vars_ext.cc +++ b/sql/sys_vars_ext.cc @@ -24,12 +24,13 @@ this program; if not, write to the Free Software Foundation, Inc., *****************************************************************************/ -#include "sql/sys_vars.h" #include "sql/duckdb/duckdb_config.h" +#include "sql/duckdb/log.h" #include "sql/rpl_applier_reader.h" #include "sql/rpl_rli.h" -#include "sql/duckdb/log.h" #include "sql/sql_table_ext.h" +#include "sql/sys_vars.h" +#include "vidx/vidx_index.h" /** DuckDB related variables begin. */ static Sys_var_bool Sys_duckdb_require_primary_key( diff --git a/sql/system_variables.h b/sql/system_variables.h index a91d161284a..7aceee19f6b 100644 --- a/sql/system_variables.h +++ b/sql/system_variables.h @@ -34,6 +34,7 @@ #include "my_thread_local.h" // my_thread_id #include "sql/rpl_gtid.h" // Gitd_specification #include "sql/sql_plugin_ref.h" // plugin_ref +#include "vidx/vidx_common.h" class MY_LOCALE; class Time_zone; diff --git a/sql/table.cc b/sql/table.cc index 33ae7d9d05c..d616dd6e2aa 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -79,7 +79,8 @@ #include "sql/dd/types/view.h" // dd::View #include "sql/debug_sync.h" // DEBUG_SYNC #include "sql/derror.h" // ER_THD -#include "sql/error_handler.h" // Strict_error_handler +#include "sql/duckdb/duckdb_config.h" +#include "sql/error_handler.h" // Strict_error_handler #include "sql/field.h" #include "sql/filesort.h" // filesort_free_buffers #include "sql/gis/srid.h" @@ -124,7 +125,7 @@ #include "sql_string.h" #include "template_utils.h" // down_cast #include "thr_mutex.h" -#include "sql/duckdb/duckdb_config.h" +#include "vidx/vidx_hnsw.h" /* INFORMATION_SCHEMA name */ LEX_CSTRING INFORMATION_SCHEMA_NAME = {STRING_WITH_LEN("information_schema")}; @@ -432,6 +433,8 @@ TABLE_SHARE *alloc_table_share(const char *db, const char *table_name, share->mem_root = std::move(mem_root); mysql_mutex_init(key_TABLE_SHARE_LOCK_ha_data, &share->LOCK_ha_data, MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_TABLE_SHARE_LOCK_share, &share->LOCK_share, + MY_MUTEX_INIT_FAST); } return share; } @@ -547,8 +550,16 @@ void TABLE_SHARE::destroy() { ::destroy(m_part_info); m_part_info = nullptr; } + if (hlindex) { + vidx::hnsw::mhnsw_free(this); + hlindex->destroy(); + hlindex = nullptr; + } /* The mutex is initialized only for shares that are part of the TDC */ - if (tmp_table == NO_TMP_TABLE) mysql_mutex_destroy(&LOCK_ha_data); + if (tmp_table == NO_TMP_TABLE) { + mysql_mutex_destroy(&LOCK_ha_data); + mysql_mutex_destroy(&LOCK_share); + } delete m_histograms; m_histograms = nullptr; @@ -738,8 +749,9 @@ void setup_key_part_field(TABLE_SHARE *share, handler *handler_file, if (key_part_n == 0) field->key_start.set_bit(key_n); field->m_indexed = true; - const bool full_length_key_part = - field->key_length() == key_part->length && !field->is_flag_set(BLOB_FLAG); + const bool full_length_key_part = field->key_length() == key_part->length && + !field->is_flag_set(BLOB_FLAG) && + key_n < share->keys; const bool is_spatial_key = Overlaps(keyinfo->flags, HA_SPATIAL); /* part_of_key contains all non-prefix keys, part_of_prefixkey @@ -758,7 +770,7 @@ void setup_key_part_field(TABLE_SHARE *share, handler *handler_file, // R-tree indexes do not allow index scans and therefore cannot be // marked as keys for index only access. if ((handler_file->index_flags(key_n, key_part_n, false) & HA_KEYREAD_ONLY) && - !is_spatial_key) { + !is_spatial_key && key_n < share->keys) { // Set the key as 'keys_for_keyread' even if it is prefix key. share->keys_for_keyread.set_bit(key_n); } @@ -1353,7 +1365,8 @@ static int make_field_from_frm(THD *thd, TABLE_SHARE *share, f_is_zerofill(pack_flag) != 0, f_is_dec(pack_flag) == 0, f_decimals(pack_flag), f_bit_as_char(pack_flag), 0, {}, // Array fields aren't supported in .frm-based tables - false); + // Vector fields aren't supported in .frm-based tables + false, false); if (!reg_field) { // Not supported field type return 4; @@ -1475,6 +1488,7 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, DBUG_PRINT("info", ("default_part_db_type = %u", head[61])); legacy_db_type = (enum legacy_db_type)(uint) * (head + 3); assert(share->db_plugin == nullptr); + assert(share->keys == share->total_keys); /* if the storage engine is dynamic, no point in resolving it by its dynamically allocated legacy_db_type. We will resolve it later by name. @@ -1518,10 +1532,11 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, if (read_string(file, &disk_buff, key_info_length)) goto err; /* purecov: inspected */ if (disk_buff[0] & 0x80) { - share->keys = keys = (disk_buff[1] << 7) | (disk_buff[0] & 0x7f); + share->total_keys = share->keys = keys = + (disk_buff[1] << 7) | (disk_buff[0] & 0x7f); share->key_parts = key_parts = uint2korr(disk_buff + 2); } else { - share->keys = keys = disk_buff[0]; + share->total_keys = share->keys = keys = disk_buff[0]; share->key_parts = key_parts = disk_buff[1]; } share->visible_indexes.init(0); @@ -1896,13 +1911,13 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, DBUG_PRINT("info", ("i_count: %d i_parts: %d index: %d n_length: %d " "int_length: %d com_length: %d gcol_screen_length: %d", - interval_count, interval_parts, share->keys, n_length, - int_length, com_length, gcol_screen_length)); - if (!(field_ptr = (Field **)share->mem_root.Alloc(( - uint)((share->fields + 1) * sizeof(Field *) + - interval_count * sizeof(TYPELIB) + - (share->fields + interval_parts + keys + 3) * sizeof(char *) + - (n_length + int_length + com_length + gcol_screen_length))))) + interval_count, interval_parts, share->total_keys, + n_length, int_length, com_length, gcol_screen_length)); + if (!(field_ptr = (Field **)share->mem_root.Alloc((uint)( + (share->fields + 1) * sizeof(Field *) + + interval_count * sizeof(TYPELIB) + + (share->fields + interval_parts + keys + 3) * sizeof(char *) + + (n_length + int_length + com_length + gcol_screen_length))))) goto err; /* purecov: inspected */ share->field = field_ptr; @@ -2051,7 +2066,7 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, keyinfo = share->key_info; key_part = keyinfo->key_part; - for (uint key = 0; key < share->keys; key++, keyinfo++) { + for (uint key = 0; key < share->total_keys; key++, keyinfo++) { uint usable_parts = 0; keyinfo->name = share->keynames.type_names[key]; /* Fix fulltext keys for old .frm files */ @@ -2227,7 +2242,7 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, if (share->found_next_number_field) { Field *reg_field = *share->found_next_number_field; if ((int)(share->next_number_index = (uint)find_ref_key( - share->key_info, share->keys, share->default_values, + share->key_info, share->total_keys, share->default_values, reg_field, &share->next_number_key_offset, &share->next_number_keypart)) < 0) { /* Wrong field definition */ @@ -2801,22 +2816,22 @@ bool create_key_part_field_with_prefix_length(TABLE *table, MEM_ROOT *root) { assert(share->key_parts); - n_length = - share->keys * sizeof(KEY) + share->key_parts * sizeof(KEY_PART_INFO); + n_length = share->total_keys * sizeof(KEY) + + share->key_parts * sizeof(KEY_PART_INFO); // Allocate new memory for table.key_info if (!(key_info = static_cast(root->Alloc(n_length)))) return true; table->key_info = key_info; - key_part = (reinterpret_cast(key_info + share->keys)); + key_part = (reinterpret_cast(key_info + share->total_keys)); // Copy over the key_info from share to table. - memcpy(key_info, share->key_info, sizeof(*key_info) * share->keys); + memcpy(key_info, share->key_info, sizeof(*key_info) * share->total_keys); memcpy(key_part, share->key_info[0].key_part, (sizeof(*key_part) * share->key_parts)); - for (KEY *key_info_end = key_info + share->keys; key_info < key_info_end; - key_info++) { + for (KEY *key_info_end = key_info + share->total_keys; + key_info < key_info_end; key_info++) { key_info->table = table; key_info->key_part = key_part; @@ -3009,8 +3024,8 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, if (share->key_parts) { if (create_key_part_field_with_prefix_length(outparam, root)) goto err; KEY *key_info = outparam->key_info; - for (KEY *key_info_end = key_info + share->keys; key_info < key_info_end; - key_info++) { + for (KEY *key_info_end = key_info + share->total_keys; + key_info < key_info_end; key_info++) { /* Set TABLE::fts_doc_id_field for tables with FT KEY */ if ((key_info->flags & HA_FULLTEXT)) outparam->fts_doc_id_field = fts_doc_id_field; @@ -3365,7 +3380,9 @@ int closefrm(TABLE *table, bool free_share) { table->part_info = nullptr; } if (free_share) { - if (table->s->tmp_table == NO_TMP_TABLE) + if (table->s->is_hlindex) { + table->s->decrement_ref_count(); + } else if (table->s->tmp_table == NO_TMP_TABLE) release_table_share(table->s); else free_table_share(table->s); @@ -4280,7 +4297,7 @@ bool TABLE::init_tmp_table(THD *thd, TABLE_SHARE *share, MEM_ROOT *m_root, share->visible_indexes.init(); share->keys_for_keyread.init(); share->keys_in_use.init(); - share->keys = 0; + share->total_keys = share->keys = 0; share->field = field = fld; share->table_charset = charset; set_not_started(); @@ -5998,6 +6015,7 @@ bool TABLE::add_tmp_key(Field_map *key_parts, bool invisible, assert(s->keys < s->max_tmp_keys); sprintf(s->key_names[s->keys].name, "", s->keys); s->keys++; + s->total_keys++; } const uint keyno = s->keys - 1; @@ -6151,7 +6169,7 @@ void TABLE::move_tmp_key(int old_idx, bool modify_share) { void TABLE::drop_unused_tmp_keys(bool modify_share) { if (modify_share) { assert(s->first_unused_tmp_key <= s->keys); - s->keys = s->first_unused_tmp_key; + s->total_keys = s->keys = s->first_unused_tmp_key; s->key_parts = 0; for (uint i = 0; i < s->keys; i++) s->key_parts += s->key_info[i].user_defined_key_parts; @@ -7973,6 +7991,8 @@ int create_table_share_for_upgrade(THD *thd, const char *path, share->tmp_table = NO_TMP_TABLE; mysql_mutex_init(key_TABLE_SHARE_LOCK_ha_data, &share->LOCK_ha_data, MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_TABLE_SHARE_LOCK_share, &share->LOCK_share, + MY_MUTEX_INIT_FAST); int r = read_frm_file(thd, share, frm_context, table_name, is_fix_view_cols_and_deps); @@ -8016,7 +8036,7 @@ bool TABLE::empty_result_table() { void TABLE::update_covering_prefix_keys(Field *field, uint16 key_read_length, Key_map *covering_prefix_keys) { - for (uint keyno = 0; keyno < s->keys; keyno++) + for (uint keyno = 0; keyno < s->total_keys; keyno++) if (covering_prefix_keys->is_set(keyno)) { KEY *key_info = &this->key_info[keyno]; for (KEY_PART_INFO *part = key_info->key_part, diff --git a/sql/table.h b/sql/table.h index 5034da1538d..405bb0d9ac1 100644 --- a/sql/table.h +++ b/sql/table.h @@ -45,7 +45,8 @@ #include "my_table_map.h" #include "mysql/components/services/bits/mysql_mutex_bits.h" #include "mysql/components/services/bits/psi_table_bits.h" -#include "sql/auth/auth_acls.h" // Access_bitmask +#include "sql/auth/auth_acls.h" // Access_bitmask +#include "sql/dd/object_id.h" #include "sql/dd/types/foreign_key.h" // dd::Foreign_key::enum_rule #include "sql/enum_query_type.h" // enum_query_type #include "sql/key.h" @@ -1045,6 +1046,38 @@ struct TABLE_SHARE { enum class Schema_read_only { NOT_SET, RO_OFF, RO_ON }; Schema_read_only schema_read_only{Schema_read_only::NOT_SET}; + bool is_hlindex{false}; + /* for hlindex tables */ + void *hlindex_data{nullptr}; + /* for normal tables */ + TABLE_SHARE *hlindex{nullptr}; + /* Number of keys including vector keys. */ + uint total_keys{0}; + + mysql_mutex_t LOCK_share; /* MariaDB use it to protect TABLE_SHARE. RDS only + protect hlindex and hlindex_data */ + + dd::Object_id m_se_private_id{dd::INVALID_OBJECT_ID}; + + inline void lock_share() { + if (!tmp_table) mysql_mutex_lock(&LOCK_share); + } + + inline void unlock_share() { + if (!tmp_table) mysql_mutex_unlock(&LOCK_share); + } + + inline uint hlindexes() { + assert(total_keys >= keys); + /* The total_keys is not expected as less than keys. This abnormal + situation returns 0 for stability。 */ + return total_keys >= keys ? total_keys - keys : 0; + } + + inline KEY *get_vec_key() { + return hlindexes() ? (key_info + keys) : nullptr; + } + /** Set share's table cache key and update its db and table name appropriately. @@ -1954,7 +1987,7 @@ struct TABLE { column */ inline bool index_contains_some_virtual_gcol(uint index_no) const { - assert(index_no < s->keys); + assert(index_no < s->total_keys); return key_info[index_no].flags & HA_VIRTUAL_GEN_KEY; } void update_const_key_parts(Item *conds); @@ -2414,6 +2447,22 @@ struct TABLE { set or not */ bool should_binlog_drop_if_temp(void) const; + + TABLE *hlindex{nullptr}; /* Vector key table */ + void *context; /* only for hlindexes */ + + int hlindex_open(uint nr); + int hlindex_lock(uint nr); + int reset_hlindexes(); + + int hlindexes_on_insert(); + int hlindexes_on_update(); + int hlindexes_on_delete(const uchar *buf); + int hlindexes_on_delete_all(); + + int hlindex_read_first(uint key, void *item); + int hlindex_read_next(); + int hlindex_read_end(); }; static inline void empty_record(TABLE *table) { diff --git a/sql/table_cache.h b/sql/table_cache.h index 2287a078ae7..dfd23e5dae3 100644 --- a/sql/table_cache.h +++ b/sql/table_cache.h @@ -459,6 +459,7 @@ TABLE *Table_cache::get_table(THD *thd, const char *key, size_t key_length, if ((table = el->free_tables.front())) { assert(!table->in_use); + assert(table->hlindex == nullptr); /* Unlink table from list of unused TABLE objects for this @@ -494,6 +495,14 @@ TABLE *Table_cache::get_table(THD *thd, const char *key, size_t key_length, */ void Table_cache::release_table(THD *thd, TABLE *table) { + if (table->hlindex != nullptr) { + mysql_mutex_lock(&LOCK_open); + intern_close_table(table->hlindex); + mysql_mutex_unlock(&LOCK_open); + + table->hlindex = nullptr; + } + Table_cache_element *el = table->s->cache_element[table_cache_manager.cache_index(this)]; diff --git a/sql/vidx/vidx_field.cc b/sql/vidx/vidx_field.cc new file mode 100644 index 00000000000..5a0d064ae92 --- /dev/null +++ b/sql/vidx/vidx_field.cc @@ -0,0 +1,146 @@ +/* Copyright (c) 2025, 2025, Alibaba and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include "sql/create_field.h" // Create_field +#include "sql/current_thd.h" // vidx/vidx_index.h +#include "sql/derror.h" // ER_THD +#include "sql/my_decimal.h" // my_decimal +#include "sql/sql_class.h" // THD + +#include "vidx/vidx_field.h" + +namespace vidx { +uint32 Field_vector::get_dimensions() const { + return get_dimensions_low(field_length, VECTOR_PRECISION); +} + +type_conversion_status Field_vector::store(double) { + my_error(ER_DATA_INCOMPATIBLE_WITH_VECTOR, MYF(0), "double", sizeof(double), + get_dimensions()); + return TYPE_ERR_BAD_VALUE; +} + +type_conversion_status Field_vector::store(longlong, bool) { + my_error(ER_DATA_INCOMPATIBLE_WITH_VECTOR, MYF(0), "longlong", + sizeof(longlong), get_dimensions()); + return TYPE_ERR_BAD_VALUE; +} + +type_conversion_status Field_vector::store_decimal(const my_decimal *) { + my_error(ER_DATA_INCOMPATIBLE_WITH_VECTOR, MYF(0), "decimal", + sizeof(my_decimal), get_dimensions()); + return TYPE_ERR_BAD_VALUE; +} + +type_conversion_status Field_vector::store(const char *from, size_t length, + const CHARSET_INFO *cs) { + if (cs != &my_charset_bin) { + THD *thd = current_thd; + ErrConvString err(from, length, cs); + push_warning_printf( + thd, Sql_condition::SL_WARNING, ER_TRUNCATED_WRONG_VALUE_FOR_FIELD, + ER_THD(thd, ER_TRUNCATED_WRONG_VALUE_FOR_FIELD), "vector", err.ptr(), + field_name, thd->get_stmt_da()->current_row_for_condition()); + } + + if (length != field_length) { + /* Validate length should be the same as the field length. */ + wrong_length_return: + my_error(ER_DATA_INCOMPATIBLE_WITH_VECTOR, MYF(0), "string", length, + get_dimensions()); + return TYPE_ERR_BAD_VALUE; + } + + uint32 dimensions = get_dimensions_low(length, VECTOR_PRECISION); + if (dimensions == UINT_MAX32 || dimensions > get_dimensions()) { + goto wrong_length_return; + } + + /* Validate every dimension and abs value of the vector. */ + float abs2 = 0.0f; + for (uint32 i = 0; i < dimensions; i++) { + float to_store = 0; + memcpy(&to_store, from + sizeof(float) * i, sizeof(float)); + if (std::isnan(to_store) || std::isinf(to_store)) { + goto wrong_data_return; + } + float val = get_float(from + sizeof(float) * i); + abs2 += val * val; + } + + if (!std::isfinite(abs2)) { + wrong_data_return: + THD *thd = current_thd; + ErrConvString err(from, length, cs); + my_error(ER_TRUNCATED_WRONG_VALUE_FOR_FIELD, MYF(0), "vector", err.ptr(), + field_name, thd->get_stmt_da()->current_row_for_condition()); + return TYPE_ERR_BAD_VALUE; + } + +#ifdef WORDS_BIGENDIAN + if (value.alloc(length)) { + reset(); + return TYPE_ERR_OOM; + } + for (uint32 i = 0; i < dimensions; i++) { + float to_store = 0; + memcpy(&to_store, from + sizeof(float) * i, sizeof(float)); + float4store(value.ptr() + i * sizeof(float), to_store); + } + from = value.ptr(); +#endif + + return Field_varstring::store(from, length, cs); +} + +uint Field_vector::is_equal(const Create_field *new_field) const { + if (new_field->sql_type != type() || + new_field->max_display_width_in_codepoints() != field_length || + new_field->charset != field_charset) { + return IS_EQUAL_NO; + } + return IS_EQUAL_YES; +} + +String *Field_vector::val_str(String *, String *val_ptr) const { + ASSERT_COLUMN_MARKED_FOR_READ; + + const char *data = pointer_cast(data_ptr()); + if (data == nullptr) { + val_ptr->set("", 0, charset()); // A bit safer than ->length(0) + } else { + uint32 length = data_length(); +#ifdef WORDS_BIGENDIAN + val_ptr->alloc(length); + uint32 dimensions = get_dimensions_low(length, VECTOR_PRECISION); + float *to_store = (float *)(val_ptr->ptr()); + for (uint32 i = 0; i < dimensions; i++) { + to_store[i] = float4get((const uchar *)(data + i * sizeof(float))); + } + val_ptr->length(length); +#else + val_ptr->set(data, length, charset()); +#endif + } + return val_ptr; +} +} // namespace vidx diff --git a/sql/vidx/vidx_func.cc b/sql/vidx/vidx_func.cc new file mode 100644 index 00000000000..9172d7fd3a3 --- /dev/null +++ b/sql/vidx/vidx_func.cc @@ -0,0 +1,358 @@ +/* Copyright (c) 2025, 2025, Alibaba and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include // std::isinf, std::isnan + +#include "my_byteorder.h" // get_float +#include "sql/sql_const.h" // MAX_FLOAT_STR_LENGTH +#include "sql_string.h" // String + +#include "sql/item_func.h" +#include "sql/item_strfunc.h" +#include "vidx/vidx_field.h" + +namespace vidx { +static double calc_distance_euclidean(float *v1, float *v2, size_t v_len) { + double d = 0; + for (size_t i = 0; i < v_len; i++, v1++, v2++) { + double dist = get_float(v1) - get_float(v2); + d += dist * dist; + } + return sqrt(d); +} + +static double calc_distance_cosine(float *v1, float *v2, size_t v_len) { + double dotp = 0, abs1 = 0, abs2 = 0; + for (size_t i = 0; i < v_len; i++, v1++, v2++) { + float f1 = get_float(v1), f2 = get_float(v2); + abs1 += f1 * f1; + abs2 += f2 * f2; + dotp += f1 * f2; + } + return 1 - dotp / sqrt(abs1 * abs2); +} + +static distance_kind mhnsw_uses_distance(KEY *keyinfo) { + if (keyinfo->vector_distance == (uint)EUCLIDEAN) return EUCLIDEAN; + return COSINE; +} + +static inline bool from_string_to_vector(const char *input, uint32_t input_len, + char *const output, + uint32_t *max_output_dims) { + if (input == nullptr || input_len == 0 || input[0] != '[' || + input[input_len - 1] != ']') { + *max_output_dims = 0; + return true; + } + + // Check for memory region overlap + size_t output_len = sizeof(float) * (*max_output_dims); + String temp_output(output, output_len, nullptr); + if (output + output_len >= input && input + input_len >= output) { + temp_output = String(output_len); + } + + const char *const input_end = input + input_len - 1; + input = input + 1; + uint32_t dim = 0; + char *end = nullptr; + bool with_success = false; + errno = 0; + for (float fnum = strtof(input, &end); input != end; + fnum = strtof(input, &end)) { + input = end; + if (errno == ERANGE || dim >= *max_output_dims || std::isnan(fnum) || + std::isinf(fnum)) { + errno = 0; + break; + } + memcpy(temp_output.ptr() + dim * sizeof(float), &fnum, sizeof(float)); + + if (*input == ',') { + input = input + 1; + dim++; + } else if (*input == ']' && input == input_end) { + with_success = true; + dim++; + break; + } else { + break; + } + } + + if (temp_output.ptr() != output) { + memcpy(output, temp_output.ptr(), dim * sizeof(float)); + } + + *max_output_dims = dim; + return !with_success; +} + +static inline bool from_vector_to_string(String *input, const uint32 precision, + CHARSET_INFO *cs, String *output) { + assert(input != nullptr && input->ptr() != nullptr); + assert(output != nullptr); + + const uint32 input_dims = get_dimensions_low(input->length(), precision); + + if (input_dims == UINT_MAX32) { + return true; + } + + output->length(0); + output->set_charset(cs); + output->reserve(input_dims * (MAX_FLOAT_STR_LENGTH + 1) + 2); + + if (input_dims == 0) { + return false; + } + + float val; + size_t len; + char buf[MAX_FLOAT_STR_LENGTH + 1]; + auto ptr = (const uchar *)input->ptr(); + + output->append('['); + + for (size_t i = 0; i < input_dims; i++) { + if (i != 0) { + output->append(','); + } + + val = float4get(ptr); + if (std::isinf(val)) + if (val < 0) + output->append(STRING_WITH_LEN("-Inf")); + else + output->append(STRING_WITH_LEN("Inf")); + else if (std::isnan(val)) + output->append(STRING_WITH_LEN("NaN")); + else { + len = my_gcvt(val, MY_GCVT_ARG_FLOAT, MAX_FLOAT_STR_LENGTH, buf, 0); + output->append(buf, len); + } + + ptr += precision; + } + + output->append(']'); + + return false; +} + +bool Item_func_vec_distance::resolve_type(THD *thd) { + switch (kind) { + case EUCLIDEAN: { + calc_distance_func = calc_distance_euclidean; + break; + } + case COSINE: { + calc_distance_func = calc_distance_cosine; + break; + } + case AUTO: { + for (uint fno = 0; fno < 2; fno++) { + if (args[fno]->type() == Item::FIELD_ITEM) { + Field *f = ((Item_field *)args[fno])->field; + KEY *key_info = f->table->s->key_info; + for (uint i = f->table->s->keys; i < f->table->s->total_keys; i++) { + assert(key_info[i].flags & HA_VECTOR); + assert(key_info[i].user_defined_key_parts == 1); + if (f->key_start.is_set(i)) { + kind = mhnsw_uses_distance(key_info + i); + return resolve_type(thd); + } + } + } + } + } + [[fallthrough]]; + default: + my_error(ER_VEC_DISTANCE_TYPE, MYF(0)); + return true; + } + + return Item_real_func::resolve_type(thd); +} + +int Item_func_vec_distance::get_key() { + if (check_args()) { + Field *f = field_arg->field; + String tmp; + String *r = const_arg->val_str(&tmp); + + if (!r || r->length() != f->field_length || r->length() % sizeof(float)) { + my_error(ER_WRONG_ARGUMENTS, MYF(0), func_name()); + return -1; + } + + KEY *keyinfo = f->table->s->key_info; + for (uint i = f->table->s->keys; i < f->table->s->total_keys; i++) { + assert(keyinfo[i].flags & HA_VECTOR); + assert(keyinfo[i].user_defined_key_parts == 1); + + if (f->key_start.is_set(i) && kind == mhnsw_uses_distance(keyinfo + i)) { + return i; + } + } + } + + return -1; +} + +double Item_func_vec_distance::val_real() { + String tmp1, tmp2; + String *r1 = args[0]->val_str(&tmp1); + String *r2 = args[1]->val_str(&tmp2); + + /* If the dimensions of two vectors are not equal, result should be NULL. */ + if (!r1 || !r2 || r1->length() != r2->length() || + r1->length() % sizeof(float)) { + null_value = true; + return 0; + } + + null_value = false; + + float *v1 = (float *)r1->ptr(); + float *v2 = (float *)r2->ptr(); + + return calc_distance_func(v1, v2, (r1->length()) / sizeof(float)); +} + +bool Item_func_vec_distance::check_args() { + assert((field_arg == nullptr) == (const_arg == nullptr)); + + if (field_arg != nullptr) { + return true; + } + + /* MDEV-35922 Server crashes in mhnsw_read_first upon using vector key with + * views */ + if (args[0]->real_item()->type() == Item::FIELD_ITEM && + args[1]->const_for_execution()) { + field_arg = (Item_field *)(args[0]->real_item()); + const_arg = args[1]; + return true; + } + + if (args[1]->real_item()->type() == Item::FIELD_ITEM && + args[0]->const_for_execution()) { + field_arg = (Item_field *)(args[1]->real_item()); + const_arg = args[0]; + return true; + } + + return false; +} + +bool Item_func_vec_fromtext::resolve_type(THD *thd) { + if (Item_str_func::resolve_type(thd)) { + return true; + } + if (args[0]->result_type() != STRING_RESULT || + args[0]->data_type() == MYSQL_TYPE_JSON) { + my_error(ER_WRONG_ARGUMENTS, MYF(0), func_name()); + return true; + } + if (reject_geometry_args(arg_count, args, this)) return true; + set_data_type_vector( + static_cast(Field_vector::dimension_bytes(MAX_DIMENSIONS))); + return false; +} + +String *Item_func_vec_fromtext::val_str(String *str) { + assert(fixed); + null_value = false; + String *res = args[0]->val_str(str); + if (res == nullptr || res->ptr() == nullptr) { + return error_str(); + } + + uint32 output_dims = MAX_DIMENSIONS; + auto dimension_bytes = Field_vector::dimension_bytes(output_dims); + if (buffer.mem_realloc(dimension_bytes)) return error_str(); + + bool err = from_string_to_vector(res->ptr(), res->length(), buffer.ptr(), + &output_dims); + if (err) { + if (output_dims == MAX_DIMENSIONS) { + res->replace(32, 5, "... \0", 5); + my_error(ER_DATA_OUT_OF_RANGE, MYF(0), res->ptr(), func_name()); + } else { + my_error(ER_TO_VECTOR_CONVERSION, MYF(0), res->length(), res->ptr()); + } + return error_str(); + } + + buffer.length(Field_vector::dimension_bytes(output_dims)); + return &buffer; +} + +bool Item_func_vec_totext::resolve_type(THD *thd) { + if (param_type_is_default(thd, 0, 1, MYSQL_TYPE_VARCHAR)) { + return true; + } + bool valid_type = (args[0]->data_type() == MYSQL_TYPE_VARCHAR) || + (args[0]->result_type() == STRING_RESULT && + args[0]->collation.collation == &my_charset_bin); + if (!valid_type) { + my_error(ER_WRONG_ARGUMENTS, MYF(0), func_name()); + return true; + } + set_data_type_string(Item_func_vec_totext::max_output_bytes); + return false; +} + +String *Item_func_vec_totext::val_str(String *str) { + assert(fixed); + String *res = args[0]->val_str(str); + null_value = false; + if (res == nullptr || res->ptr() == nullptr) { + return error_str(); + } + + if (from_vector_to_string(res, VECTOR_PRECISION, &my_charset_numeric, + &buffer)) { + my_error(ER_VECTOR_BINARY_FORMAT_INVALID, MYF(0)); + return error_str(); + } + + return &buffer; +} + +longlong Item_func_vector_dim::val_int() { + assert(fixed); + String *res = args[0]->val_str(&value); + null_value = false; + if (res == nullptr || res->ptr() == nullptr) { + return error_int(); /* purecov: inspected */ + } + uint32 dimensions = get_dimensions_low(res->length(), VECTOR_PRECISION); + if (dimensions == UINT_MAX32) { + my_error(ER_TO_VECTOR_CONVERSION, MYF(0), res->length(), res->ptr()); + return error_int(); /* purecov: inspected */ + } + return (longlong)dimensions; +} +} // namespace vidx diff --git a/sql/vidx/vidx_hnsw.cc b/sql/vidx/vidx_hnsw.cc new file mode 100644 index 00000000000..00a751370de --- /dev/null +++ b/sql/vidx/vidx_hnsw.cc @@ -0,0 +1,1551 @@ +/* Copyright (c) 2025, 2025, Alibaba and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include "m_ctype.h" // my_hash_sort_bin +#include "my_byteorder.h" // get_float +#include "my_inttypes.h" // uintxx_t +#include "mysql/plugin.h" // THDVAR +#include "scope_guard.h" // Scope_guard +#include "sql/dd/cache/dictionary_client.h" // dd::cache::Dictionary_client +#include "sql/dd/properties.h" // dd::Properties +#include "sql/dd/string_type.h" // dd::String_type +#include "sql/dd/types/index.h" // dd::Index +#include "sql/dd/types/index_element.h" // dd::Index_element +#include "sql/dd/types/schema.h" // dd::Schema +#include "sql/dd/types/table.h" // dd::Table +#include "sql/field.h" // Field +#include "sql/item_strfunc.h" // vidx::Item_func_vec_distance +#include "sql/key.h" // KEY +#include "sql/psi_memory_key.h" // key_memory_vidx_mem +#include "sql/sql_base.h" // get_from_share +#include "sql/sql_class.h" // THD +#include "sql/table.h" // TABLE +#include "vidx/my_atomic_wrapper.h" // Atomic_relaxed +#include "vidx/sql_hset.h" // Hash_set +#include "vidx/sql_queue.h" // Queue + +#include "sql/item_strfunc.h" +#include "vidx/bloom_filters.h" +#include "vidx/vidx_common.h" +#include "vidx/vidx_hnsw.h" + +namespace vidx { +namespace hnsw { +/* -------------------- Macros -------------------- */ +// distance can be a little bit < 0 because of fast math +static constexpr float NEAREST = -1.0f; + +// Algorithm parameters +static constexpr float alpha = 1.1f; +static constexpr uint ef_construction = 10; + +/* simplify and unify my_safe_alloca usage */ +#define MAX_ALLOCA_SZ 4096 + +uint get_ef_search(THD *thd); + +/* -------------------- Structure Definition -------------------- */ +class MHNSW_Share; +class FVectorNode; + +enum Graph_table_fields { FIELD_LAYER, FIELD_TREF, FIELD_VEC, FIELD_NEIGHBORS }; +enum Graph_table_indices { IDX_TREF, IDX_LAYER }; + +/* + One vector, an array of coordinates in ctx->vec_len dimensions +*/ +#pragma pack(push, 1) +struct FVector { + static constexpr size_t data_header = sizeof(float); + static constexpr size_t alloc_header = data_header + sizeof(float); + + float abs2, scale; + int16_t dims[4]; + + uchar *data() const { return (uchar *)(&scale); } + + static size_t data_size(size_t n) { return data_header + n * 2; } + + static size_t data_to_value_size(size_t data_size) { + return (data_size - data_header) * 2; + } + + static const FVector *create(distance_kind metric, void *mem, const void *src, + size_t src_len) { + float scale = 0, *v = (float *)src; + size_t vec_len = src_len / sizeof(float); + for (size_t i = 0; i < vec_len; i++) + scale = std::max(scale, std::abs(get_float(v + i))); + + FVector *vec = align_ptr(mem); + vec->scale = scale ? scale / 32767 : 1; + /* MDEV-37055 */ + if (std::round(scale / vec->scale) > 32767) + vec->scale = std::nextafter(vec->scale, FLT_MAX); + for (size_t i = 0; i < vec_len; i++) + vec->dims[i] = + static_cast(std::round(get_float(v + i) / vec->scale)); + vec->postprocess(vec_len); + if (metric == COSINE) { + /* fix vector length when cosine */ + if (vec->abs2 > 0.0f) vec->scale /= std::sqrt(2 * vec->abs2); + vec->abs2 = 0.5f; + } + return vec; + } + + void postprocess(size_t vec_len) { + fix_tail(vec_len); + abs2 = scale * scale * dot_product(dims, dims, vec_len) / 2; + } + +#ifdef AVX2_IMPLEMENTATION + /************* AVX2 *****************************************************/ + static constexpr size_t AVX2_bytes = 256 / 8; + static constexpr size_t AVX2_dims = AVX2_bytes / sizeof(int16_t); + + AVX2_IMPLEMENTATION + static float dot_product(const int16_t *v1, const int16_t *v2, size_t len) { + typedef float v8f __attribute__((vector_size(AVX2_bytes))); + union { + v8f v; + __m256 i; + } tmp; + __m256i *p1 = (__m256i *)v1; + __m256i *p2 = (__m256i *)v2; + v8f d = {0}; + for (size_t i = 0; i < (len + AVX2_dims - 1) / AVX2_dims; p1++, p2++, i++) { + tmp.i = _mm256_cvtepi32_ps(_mm256_madd_epi16(*p1, *p2)); + d += tmp.v; + } + return d[0] + d[1] + d[2] + d[3] + d[4] + d[5] + d[6] + d[7]; + } + + AVX2_IMPLEMENTATION + static size_t alloc_size(size_t n) { + return alloc_header + MY_ALIGN(n * 2, AVX2_bytes) + AVX2_bytes - 1; + } + + AVX2_IMPLEMENTATION + static FVector *align_ptr(void *ptr) { + return (FVector *)(MY_ALIGN(((intptr)ptr) + alloc_header, AVX2_bytes) - + alloc_header); + } + + AVX2_IMPLEMENTATION + void fix_tail(size_t vec_len) { + bzero(dims + vec_len, (MY_ALIGN(vec_len, AVX2_dims) - vec_len) * 2); + } +#endif + +#ifdef AVX512_IMPLEMENTATION + /************* AVX512 ****************************************************/ + static constexpr size_t AVX512_bytes = 512 / 8; + static constexpr size_t AVX512_dims = AVX512_bytes / sizeof(int16_t); + + AVX512_IMPLEMENTATION + static float dot_product(const int16_t *v1, const int16_t *v2, size_t len) { + __m512i *p1 = (__m512i *)v1; + __m512i *p2 = (__m512i *)v2; + __m512 d = _mm512_setzero_ps(); + for (size_t i = 0; i < (len + AVX512_dims - 1) / AVX512_dims; + p1++, p2++, i++) + d = _mm512_add_ps(d, _mm512_cvtepi32_ps(_mm512_madd_epi16(*p1, *p2))); + return _mm512_reduce_add_ps(d); + } + + AVX512_IMPLEMENTATION + static size_t alloc_size(size_t n) { + return alloc_header + MY_ALIGN(n * 2, AVX512_bytes) + AVX512_bytes - 1; + } + + AVX512_IMPLEMENTATION + static FVector *align_ptr(void *ptr) { + return (FVector *)(MY_ALIGN(((intptr)ptr) + alloc_header, AVX512_bytes) - + alloc_header); + } + + AVX512_IMPLEMENTATION + void fix_tail(size_t vec_len) { + bzero(dims + vec_len, (MY_ALIGN(vec_len, AVX512_dims) - vec_len) * 2); + } +#endif + + /* + ARM NEON implementation. A microbenchmark shows 1.7x dot_product() + performance improvement compared to regular -O2/-O3 builds and 2.4x compared + to builds with auto-vectorization disabled. + + There seem to be no performance difference between vmull+vmull_high and + vmull+vmlal2_high implementations. + */ + +#ifdef NEON_IMPLEMENTATION + static constexpr size_t NEON_bytes = 128 / 8; + static constexpr size_t NEON_dims = NEON_bytes / sizeof(int16_t); + + static float dot_product(const int16_t *v1, const int16_t *v2, size_t len) { + int64_t d = 0; + for (size_t i = 0; i < (len + NEON_dims - 1) / NEON_dims; i++) { + int16x8_t p1 = vld1q_s16(v1); + int16x8_t p2 = vld1q_s16(v2); + d += vaddlvq_s32(vmull_s16(vget_low_s16(p1), vget_low_s16(p2))) + + vaddlvq_s32(vmull_high_s16(p1, p2)); + v1 += NEON_dims; + v2 += NEON_dims; + } + return static_cast(d); + } + + static size_t alloc_size(size_t n) { + return alloc_header + MY_ALIGN(n * 2, NEON_bytes) + NEON_bytes - 1; + } + + static FVector *align_ptr(void *ptr) { + return (FVector *)(MY_ALIGN(((intptr)ptr) + alloc_header, NEON_bytes) - + alloc_header); + } + + void fix_tail(size_t vec_len) { + bzero(dims + vec_len, (MY_ALIGN(vec_len, NEON_dims) - vec_len) * 2); + } +#endif + + /************* no-SIMD default ******************************************/ +#ifdef DEFAULT_IMPLEMENTATION + DEFAULT_IMPLEMENTATION + static float dot_product(const int16_t *v1, const int16_t *v2, size_t len) { + int64_t d = 0; + for (size_t i = 0; i < len; i++) d += int32_t(v1[i]) * int32_t(v2[i]); + return static_cast(d); + } + + DEFAULT_IMPLEMENTATION + static size_t alloc_size(size_t n) { return alloc_header + n * 2; } + + DEFAULT_IMPLEMENTATION + static FVector *align_ptr(void *ptr) { return (FVector *)ptr; } + + DEFAULT_IMPLEMENTATION + void fix_tail(size_t) {} +#endif + + float distance_to(const FVector *other, size_t vec_len) const { + return abs2 + other->abs2 - + scale * other->scale * dot_product(dims, other->dims, vec_len); + } +}; +#pragma pack(pop) + +/* + An array of pointers to graph nodes + + It's mainly used to store all neighbors of a given node on a given layer. + + An array is fixed size, 2*M for the zero layer, M for other layers + see MHNSW_Share::max_neighbors(). + + Number of neighbors is zero-padded to multiples of 8 (for SIMD Bloom filter). + + Also used as a simply array of nodes in search_layer, the array size + then is defined by ef or efConstruction. +*/ +struct Neighborhood { + FVectorNode **links; + size_t num; + FVectorNode **init(FVectorNode **ptr, size_t n) { + num = 0; + links = ptr; + n = MY_ALIGN(n, 8); + bzero(ptr, n * sizeof(*ptr)); + return ptr + n; + } +}; + +/* + One node in a graph = one row in the graph table + + stores a vector itself, ref (= position) in the graph (= hlindex) + table, a ref in the main table, and an array of Neighborhood's, one + per layer. + + It's lazily initialized, may know only gref, everything else is + loaded on demand. + + On the other hand, on INSERT the new node knows everything except + gref - which only becomes known after ha_write_row. + + Allocated on memroot in two chunks. One is the same size for all nodes + and stores FVectorNode object, gref, tref, and vector. The second + stores neighbors, all Neighborhood's together, its size depends + on the number of layers this node is on. + + There can be millions of nodes in the cache and the cache size + is constrained by max_cache_size, so every byte matters here +*/ +#pragma pack(push, 1) +class FVectorNode { + private: + MHNSW_Share *ctx; + + const FVector *make_vec(const void *v); + int alloc_neighborhood(uint8_t layer); + + public: + const FVector *vec = nullptr; + Neighborhood *neighbors = nullptr; + uint8_t max_layer; + bool stored : 1, deleted : 1; + + FVectorNode(MHNSW_Share *ctx_, const void *gref_); + FVectorNode(MHNSW_Share *ctx_, const void *tref_, uint8_t layer, + const void *vec_); + float distance_to(const FVector *other) const; + int load(TABLE *graph); + int load_from_record(TABLE *graph); + int save(TABLE *graph); + size_t tref_len() const; + size_t gref_len() const; + uchar *gref() const; + uchar *tref() const; + void push_neighbor(size_t layer, FVectorNode *v); + + static const uchar *get_key(const void *elem, size_t *key_len, my_bool); +}; +#pragma pack(pop) + +/* + Shared algorithm context. The graph. + + Stored in TABLE_SHARE and on TABLE_SHARE::mem_root. + Stores the complete graph in MHNSW_Share::root, + The mapping gref->FVectorNode is in the node_cache. + Both root and node_cache are protected by a cache_lock, but it's + needed when loading nodes and is not used when the whole graph is in memory. + Graph can be traversed concurrently by different threads, as traversal + changes neither nodes nor the ctx. + Nodes can be loaded concurrently by different threads, this is protected + by a partitioned node_lock. + reference counter allows flushing the graph without interrupting + concurrent searches. + MyISAM automatically gets exclusive write access because of the TL_WRITE, + but InnoDB has to use a dedicated ctx->commit_lock for that +*/ +class MHNSW_Share { + mysql_mutex_t cache_lock; + mysql_mutex_t node_lock[8]; + + void cache_internal(FVectorNode *node) { + assert(node->stored); + node_cache.insert(node); + } + void *alloc_node_internal() { + return root.Alloc(sizeof(FVectorNode) + gref_len + tref_len + + FVector::alloc_size(vec_len)); + } + + protected: + std::atomic refcnt{0}; + MEM_ROOT root; + Hash_set node_cache{key_memory_vidx_mem, FVectorNode::get_key}; + + public: + ulonglong version = 0; // protected by commit_lock + mysql_rwlock_t commit_lock; + size_t vec_len = 0; + size_t byte_len = 0; + Atomic_relaxed ef_power{0.6}; // for the bloom filter size heuristic + Atomic_relaxed diameter{0}; // for the generosity heuristic + FVectorNode *start = 0; + const uint tref_len; + const uint gref_len; + const uint M; + distance_kind metric; + + MHNSW_Share(TABLE *t) + : tref_len(t->file->ref_length), + gref_len(t->hlindex->file->ref_length), + M(static_cast(t->s->key_info[t->s->keys].vector_m)), + metric((distance_kind)t->s->key_info[t->s->keys].vector_distance) { + mysql_rwlock_init(PSI_INSTRUMENT_ME, &commit_lock); + mysql_mutex_init(PSI_INSTRUMENT_ME, &cache_lock, MY_MUTEX_INIT_FAST); + for (uint i = 0; i < array_elements(node_lock); i++) + mysql_mutex_init(PSI_INSTRUMENT_ME, node_lock + i, MY_MUTEX_INIT_SLOW); + init_sql_alloc(key_memory_vidx_mem, &root, 1024 * 1024); + } + + virtual ~MHNSW_Share() { + /* No need to call free_root(&root, MYF(0)), because MEM_ROOT::Clear() will + be called by MEM_ROOT::~MEM_ROOT() */ + mysql_rwlock_destroy(&commit_lock); + mysql_mutex_destroy(&cache_lock); + for (size_t i = 0; i < array_elements(node_lock); i++) + mysql_mutex_destroy(node_lock + i); + } + + uint lock_node(FVectorNode *ptr) { + ulong nr1 = 1, nr2 = 4; + my_hash_sort_bin(0, (const uchar *)&ptr, sizeof(ptr), &nr1, &nr2); + uint ticket = nr1 % array_elements(node_lock); + mysql_mutex_lock(node_lock + ticket); + return ticket; + } + + void unlock_node(uint ticket) { mysql_mutex_unlock(node_lock + ticket); } + + uint max_neighbors(size_t layer) const { + return (layer ? 1 : 2) * M; // heuristic from the paper + } + + void set_lengths(size_t len) { + byte_len = len; + vec_len = len / sizeof(float); + } + + static int acquire(MHNSW_Share **ctx, TABLE *table, bool for_update); + static MHNSW_Share *get_from_share(TABLE_SHARE *share, TABLE *table); + + virtual void reset(TABLE_SHARE *share) { + share->lock_share(); + if (static_cast(share->hlindex->hlindex_data) == this) { + share->hlindex->hlindex_data = nullptr; + --refcnt; + } + share->unlock_share(); + } + + void release(TABLE *table) { + return release(table->file->has_transactions(), table->s); + } + + virtual void release(bool can_commit, TABLE_SHARE *share) { + if (can_commit) mysql_rwlock_unlock(&commit_lock); + if (root.allocated_size() > max_cache_size) reset(share); + if (--refcnt == 0) this->~MHNSW_Share(); // XXX reuse + } + + virtual MHNSW_Share *dup(bool can_commit) { + refcnt++; + if (can_commit) mysql_rwlock_rdlock(&commit_lock); + return this; + } + + FVectorNode *get_node(const void *gref) { + mysql_mutex_lock(&cache_lock); + FVectorNode *node = node_cache.find(gref, gref_len); + if (!node) { + node = new (alloc_node_internal()) FVectorNode(this, gref); + cache_internal(node); + } + mysql_mutex_unlock(&cache_lock); + return node; + } + + /* used on INSERT, gref isn't known, so cannot cache the node yet */ + void *alloc_node() { + mysql_mutex_lock(&cache_lock); + auto p = alloc_node_internal(); + mysql_mutex_unlock(&cache_lock); + return p; + } + + /* explicitly cache the node after alloc_node() */ + void cache_node(FVectorNode *node) { + mysql_mutex_lock(&cache_lock); + cache_internal(node); + mysql_mutex_unlock(&cache_lock); + } + + /* find the node without creating, only used on merging trx->ctx */ + FVectorNode *find_node(const void *gref) { + mysql_mutex_lock(&cache_lock); + FVectorNode *node = node_cache.find(gref, gref_len); + mysql_mutex_unlock(&cache_lock); + return node; + } + + void *alloc_neighborhood(size_t max_layer) { + mysql_mutex_lock(&cache_lock); + auto p = root.Alloc(sizeof(Neighborhood) * (max_layer + 1) + + sizeof(FVectorNode *) * + (MY_ALIGN(M, 4) * 2 + MY_ALIGN(M, 8) * max_layer)); + mysql_mutex_unlock(&cache_lock); + return p; + } +}; + +/* + This is a non-shared context that exists within one transaction. + + At the end of the transaction it's either discarded (on rollback) + or merged into the shared ctx (on commit). + + trx's are stored in thd->ha_data[] in a single-linked list, + one instance of trx per TABLE_SHARE and allocated on the + thd->transaction->mem_root +*/ +class MHNSW_Trx : public MHNSW_Share { + public: + MDL_ticket *table_id; + bool list_of_nodes_is_lost = false; + MHNSW_Trx *next = nullptr; + + MHNSW_Trx(TABLE *table) : MHNSW_Share(table), table_id(table->mdl_ticket) {} + void reset(TABLE_SHARE *) override { + node_cache.clear(); + root.Clear(); + start = 0; + list_of_nodes_is_lost = true; + } + void release(bool, TABLE_SHARE *) override { + if (--refcnt == 0 && root.allocated_size() > max_cache_size) reset(nullptr); + } + + virtual MHNSW_Share *dup(bool) override { + refcnt++; + return this; + } + + static MHNSW_Trx *get_from_thd(TABLE *table, bool for_update); + + // it's okay in a transaction-local cache, there's no concurrent access + Hash_set &get_cache() { return node_cache; } + + /* fake handlerton to use thd->ha_data and to get notified of commits */ + static struct MHNSW_hton : public handlerton { + MHNSW_hton() { + db_type = DB_TYPE_HLINDEX_HELPER; + flags = HTON_NOT_USER_SELECTABLE | HTON_HIDDEN; + /* savepoint_offset will be set in setup_transaction_participant(). */ + savepoint_offset = 0; + savepoint_set = [](handlerton *, THD *, void *) { return 0; }; + savepoint_rollback_can_release_mdl = [](handlerton *, THD *) { + return true; + }; + savepoint_rollback = do_savepoint_rollback; + commit = do_commit; + rollback = do_rollback; + } + static int do_commit(handlerton *, THD *thd, bool); + static int do_rollback(handlerton *, THD *thd, bool); + static int do_savepoint_rollback(handlerton *, THD *thd, void *); + } hnsw_hton; +}; + +/* one visited node during the search. caches the distance to target */ +struct Visited { + FVectorNode *node; + const float distance_to_target; + Visited(FVectorNode *n, float d) : node(n), distance_to_target(d) {} + static int cmp(void *, uchar *a_, uchar *b_) { + const Visited *a = (const Visited *)a_; + const Visited *b = (const Visited *)b_; + return a->distance_to_target < b->distance_to_target ? -1 + : a->distance_to_target > b->distance_to_target ? 1 + : 0; + } +}; + +/* + a factory to create Visited and keep track of already seen nodes + + note that PatternedSimdBloomFilter works in blocks of 8 elements, + so on insert they're accumulated in nodes[], on search the caller + provides 8 addresses at once. we record 0x0 as "seen" so that + the caller could pad the input with nullptr's +*/ +class VisitedSet { + MEM_ROOT *root; + const FVector *target; + PatternedSimdBloomFilter map; + const FVectorNode *nodes[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + size_t idx = 1; // to record 0 in the filter + public: + uint count = 0; + VisitedSet(MEM_ROOT *root, const FVector *target, uint size) + : root(root), target(target), map(size, 0.01f) {} + Visited *create(FVectorNode *node) { + auto *v = new (root) Visited(node, node->distance_to(target)); + insert(node); + count++; + return v; + } + void insert(const FVectorNode *n) { + nodes[idx++] = n; + if (idx == 8) flush(); + } + void flush() { + if (idx) map.Insert(nodes); + idx = 0; + } + uint8_t seen(FVectorNode **nodes) { return map.Query(nodes); } +}; + +struct Search_context { + Neighborhood found; + MHNSW_Share *ctx; + const FVector *target; + ulonglong ctx_version; + size_t pos = 0; + float threshold = NEAREST / 2; + Search_context(Neighborhood *n, MHNSW_Share *s, const FVector *v) + : found(*n), ctx(s->dup(false)), target(v), ctx_version(ctx->version) {} +}; + +/* -------------------- External Vars Definition -------------------- */ +MHNSW_Trx::MHNSW_hton MHNSW_Trx::hnsw_hton; + +void *trx_handler = &MHNSW_Trx::hnsw_hton; + +ulonglong max_cache_size = DEF_CACHE_SIZE; + +/* -------------------- Static Functions Definition -------------------- */ +template +static void copy_option(dd::Properties *to, dd::Properties *from, + const char *key) { + if (from->exists(key)) { + T value; + from->get(key, &value); + to->set(key, value); + } +} + +static dd::Column *fill_dd_add_columns( + dd::Table *hlindex_dd, const char *name, const char *type_name, + dd::enum_column_types type, size_t char_length, bool nullable, + uint numeric_precision, const CHARSET_INFO *charset, + bool is_explicit_collation, bool is_numeric_scale_null, + size_t default_value_len, uint nr [[maybe_unused]]) { + dd::Column *col = hlindex_dd->add_column(); + assert(col->ordinal_position() == nr + 1); + dd::Properties *options = &col->options(); + col->set_name(name); + col->set_column_type_utf8(type_name); + col->set_type(type); + col->set_char_length(char_length); + col->set_numeric_precision(numeric_precision); + col->set_hidden(dd::Column::enum_hidden_type::HT_VISIBLE); + col->set_collation_id(charset->number); + col->set_is_explicit_collation(is_explicit_collation); + if (!is_numeric_scale_null) { + col->set_numeric_scale(0); + } + col->set_nullable(nullable); + col->set_has_no_default(!nullable); + col->set_default_value_null(nullable); + if (!nullable) { + assert(default_value_len > 0); + + char *default_value = new char[default_value_len]; + + memset(default_value, 0, default_value_len); + col->set_default_value(dd::String_type{default_value, default_value_len}); + + delete[] default_value; + } + + options->set("interval_count", 0); + return col; +} + +static dd::Index *fill_dd_add_indexes(dd::Table *hlindex_dd, const char *name, + bool is_unique, uint nr, dd::Column *col, + const uint64_t length) { + dd::Index *index = hlindex_dd->add_index(); + index->set_name(name); + index->set_algorithm(dd::Index::IA_BTREE); + index->set_algorithm_explicit(false); + index->set_visible(true); + index->set_type(is_unique ? dd::Index::IT_UNIQUE : dd::Index::IT_MULTIPLE); + index->set_ordinal_position(nr + 1); + index->set_generated(false); + index->set_engine(hlindex_dd->engine()); + index->options().set("flags", 0); + + col->set_column_key(is_unique ? dd::Column::CK_UNIQUE + : dd::Column::CK_MULTIPLE); + + dd::Index_element *index_elem = index->add_element(col); + index_elem->set_length(length); + + return index; +} + +/* + selects best neighbors from the list of candidates plus one extra candidate + + one extra candidate is specified separately to avoid appending it to + the Neighborhood candidates, which might be already at its max size. +*/ +static int select_neighbors(MHNSW_Share *, TABLE *graph, size_t layer, + FVectorNode &target, const Neighborhood &candidates, + FVectorNode *extra_candidate, + size_t max_neighbor_connections) { + Queue pq; // working queue + + if (pq.init(max_ef, false, Visited::cmp)) return HA_ERR_OUT_OF_MEM; + + MEM_ROOT *const root = graph->in_use->mem_root; + auto discarded = (Visited **)my_safe_alloca( + sizeof(Visited **) * max_neighbor_connections, MAX_ALLOCA_SZ); + size_t discarded_num = 0; + Neighborhood &neighbors = target.neighbors[layer]; + + for (size_t i = 0; i < candidates.num; i++) { + FVectorNode *node = candidates.links[i]; + if (int err = node->load(graph)) return err; + pq.push(new (root) Visited(node, node->distance_to(target.vec))); + } + if (extra_candidate) + pq.push(new (root) Visited(extra_candidate, + extra_candidate->distance_to(target.vec))); + + assert(pq.elements()); + neighbors.num = 0; + + while (pq.elements() && neighbors.num < max_neighbor_connections) { + Visited *vec = pq.pop(); + FVectorNode *const node = vec->node; + const float target_dista = + std::max(32 * FLT_EPSILON, vec->distance_to_target / alpha); + bool discard = false; + for (size_t i = 0; i < neighbors.num; i++) + if ((discard = + node->distance_to(neighbors.links[i]->vec) <= target_dista)) + break; + if (!discard) + target.push_neighbor(layer, node); + else if (discarded_num + neighbors.num < max_neighbor_connections) + discarded[discarded_num++] = vec; + } + + for (size_t i = 0; + i < discarded_num && neighbors.num < max_neighbor_connections; i++) + target.push_neighbor(layer, discarded[i]->node); + + my_safe_afree(discarded, sizeof(Visited **) * max_neighbor_connections, + MAX_ALLOCA_SZ); + return 0; +} + +static int update_second_degree_neighbors(MHNSW_Share *ctx, TABLE *graph, + size_t layer, FVectorNode *node) { + const uint max_neighbors = ctx->max_neighbors(layer); + // it seems that one could update nodes in the gref order + // to avoid InnoDB deadlocks, but it produces no noticeable effect + for (size_t i = 0; i < node->neighbors[layer].num; i++) { + FVectorNode *neigh = node->neighbors[layer].links[i]; + Neighborhood &neighneighbors = neigh->neighbors[layer]; + if (neighneighbors.num < max_neighbors) + neigh->push_neighbor(layer, node); + else if (int err = select_neighbors(ctx, graph, layer, *neigh, + neighneighbors, node, max_neighbors)) + return err; + if (int err = neigh->save(graph)) return err; + } + return 0; +} + +static inline float generous_furthest(const Queue &q, float maxd, + float g) { + float d0 = maxd * g / 2; + float d = q.top()->distance_to_target; + float k = 5; + float x = (d - d0) / d0; + float sigmoid = + k * x / std::sqrt(1 + (k * k - 1) * x * x); // or any other sigmoid + return d * (1 + (g - 1) / 2 * (1 - sigmoid)); +} + +/* + @param[in/out] inout in: start nodes, out: result nodes +*/ +static int search_layer(MHNSW_Share *ctx, TABLE *graph, const FVector *target, + float threshold, uint result_size, size_t layer, + Neighborhood *inout, bool construction) { + assert(inout->num > 0); + + MEM_ROOT *const root = graph->in_use->mem_root; + Queue candidates, best; + bool skip_deleted; + uint ef = result_size; + float generosity = 1.1f + ctx->M / 500.0f; + + if (construction) { + skip_deleted = false; + if (ef > 1) ef = std::max(ef_construction, ef); + } else { + skip_deleted = layer == 0; + if (ef > 1 || layer == 0) ef = std::max(get_ef_search(graph->in_use), ef); + } + + // WARNING! heuristic here + const double est_heuristic = 8 * std::sqrt(ctx->max_neighbors(layer)); + const uint est_size = + static_cast(est_heuristic * std::pow(ef, ctx->ef_power)); + VisitedSet visited(root, target, est_size); + + candidates.init(max_ef, false, Visited::cmp); + best.init(ef, true, Visited::cmp); + + assert(inout->num <= result_size); + float max_distance = ctx->diameter; + for (size_t i = 0; i < inout->num; i++) { + Visited *v = visited.create(inout->links[i]); + max_distance = std::max(max_distance, v->distance_to_target); + candidates.safe_push(v); /* MDEV-35745 */ + if ((skip_deleted && v->node->deleted) || threshold > NEAREST) continue; + best.push(v); + } + + float furthest_best = best.is_empty() + ? FLT_MAX + : generous_furthest(best, max_distance, generosity); + while (candidates.elements()) { + const Visited &cur = *candidates.pop(); + if (cur.distance_to_target > furthest_best && best.is_full()) + break; // All possible candidates are worse than what we have + + visited.flush(); + + Neighborhood &neighbors = cur.node->neighbors[layer]; + FVectorNode **links = neighbors.links, **end = links + neighbors.num; + for (; links < end; links += 8) { + uint8_t res = visited.seen(links); + if (res == 0xff) continue; + + for (size_t i = 0; i < 8; i++) { + if (res & (1 << i)) continue; + if (int err = links[i]->load(graph)) return err; + Visited *v = visited.create(links[i]); + if (v->distance_to_target <= threshold) continue; + if (!best.is_full()) { + max_distance = std::max(max_distance, v->distance_to_target); + candidates.safe_push(v); /* MDEV-35745 */ + if (skip_deleted && v->node->deleted) continue; + best.push(v); + furthest_best = generous_furthest(best, max_distance, generosity); + } else if (v->distance_to_target < furthest_best) { + candidates.safe_push(v); + if (skip_deleted && v->node->deleted) continue; + if (v->distance_to_target < best.top()->distance_to_target) { + best.replace_top(v); + furthest_best = generous_furthest(best, max_distance, generosity); + } + } + } + } + } + set_if_bigger(ctx->diameter, max_distance); // not atomic, but it's ok + if (ef > 1 && visited.count * 2 > est_size) { + double ef_power = + std::log(visited.count * 2 / est_heuristic) / std::log(ef); + set_if_bigger(ctx->ef_power, ef_power); // not atomic, but it's ok + } + + while (best.elements() > result_size) best.pop(); + + inout->num = best.elements(); + for (FVectorNode **links = inout->links + inout->num; best.elements();) + *--links = best.pop()->node; + + return 0; +} + +/* -------------------- Class Functions Definition -------------------- */ +int MHNSW_Trx::MHNSW_hton::do_savepoint_rollback(handlerton *, THD *thd, + void *) { + for (auto trx = static_cast(thd_get_ha_data(thd, &hnsw_hton)); + trx; trx = trx->next) + trx->reset(nullptr); + return 0; +} + +int MHNSW_Trx::MHNSW_hton::do_rollback(handlerton *ht, THD *thd, bool all) { + /* MDEV-37068 Can't find record in 't1' on INSERT to Vector table */ + if (!all && thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) + return do_savepoint_rollback(ht, thd, nullptr); + + MHNSW_Trx *trx_next; + for (auto trx = static_cast(thd_get_ha_data(thd, &hnsw_hton)); + trx; trx = trx_next) { + trx_next = trx->next; + trx->~MHNSW_Trx(); + } + thd_set_ha_data(current_thd, &hnsw_hton, nullptr); + return 0; +} + +int MHNSW_Trx::MHNSW_hton::do_commit(handlerton *, THD *thd, bool all) { + /* MDEV-37068 Can't find record in 't1' on INSERT to Vector table */ + if (!all && thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) + return 0; + + MHNSW_Trx *trx_next; + + for (auto trx = static_cast(thd_get_ha_data(thd, &hnsw_hton)); + trx; trx = trx_next) { + trx_next = trx->next; + if (trx->table_id) { + const MDL_key *key = trx->table_id->get_key(); + mysql_mutex_lock(&LOCK_open); + TABLE_SHARE *share = get_table_share(thd, key->db_name(), key->name(), + (const char *)key->ptr() + 1, + key->length() - 1, false); + mysql_mutex_unlock(&LOCK_open); + + if (share) { + auto ctx = share->hlindex ? MHNSW_Share::get_from_share(share, nullptr) + : nullptr; + if (ctx) { + mysql_rwlock_wrlock(&ctx->commit_lock); + ctx->version++; + if (trx->list_of_nodes_is_lost) + ctx->reset(share); + else { + // consider copying nodes from trx to shared cache when it makes + // sense. for ann_benchmarks it does not. + // also, consider flushing only changed nodes (a flag in the node) + for (FVectorNode &from : trx->get_cache()) + if (FVectorNode *node = ctx->find_node(from.gref())) + node->vec = nullptr; + ctx->start = nullptr; + } + ctx->release(true, share); + } + + mysql_mutex_lock(&LOCK_open); + release_table_share(share); + mysql_mutex_unlock(&LOCK_open); + } + } + + trx->~MHNSW_Trx(); + } + thd_set_ha_data(current_thd, &hnsw_hton, nullptr); + return 0; +} + +MHNSW_Trx *MHNSW_Trx::get_from_thd(TABLE *table, bool for_update) { + if (!table->file->has_transactions()) return NULL; + + THD *thd = table->in_use; + auto trx = static_cast(thd_get_ha_data(thd, &hnsw_hton)); + if (!for_update && !trx) return NULL; + + while (trx && trx->table_id != table->mdl_ticket) trx = trx->next; + if (!trx) { + trx = new (thd->get_transaction()->transaction_memroot()) MHNSW_Trx(table); + trx->next = static_cast(thd_get_ha_data(thd, &hnsw_hton)); + thd_set_ha_data(thd, &hnsw_hton, trx); + if (!trx->next) { + /* MDEV-37068 Can't find record in 't1' on INSERT to Vector table */ + if (thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) + trans_register_ha(thd, true, &hnsw_hton, 0); + trans_register_ha(thd, false, &hnsw_hton, 0); + } + } + trx->refcnt++; + return trx; +} + +MHNSW_Share *MHNSW_Share::get_from_share(TABLE_SHARE *share, TABLE *table) { + share->lock_share(); + auto ctx = static_cast(share->hlindex->hlindex_data); + if (!ctx && table) { + ctx = new (&share->hlindex->mem_root) MHNSW_Share(table); + if (!ctx) return nullptr; + share->hlindex->hlindex_data = ctx; + ctx->refcnt++; + } + if (ctx) ctx->refcnt++; + share->unlock_share(); + return ctx; +} + +int MHNSW_Share::acquire(MHNSW_Share **ctx, TABLE *table, bool for_update) { + TABLE *graph = table->hlindex; + + if (!(*ctx = MHNSW_Trx::get_from_thd(table, for_update))) { + *ctx = MHNSW_Share::get_from_share(table->s, table); + if (table->file->has_transactions()) + mysql_rwlock_rdlock(&(*ctx)->commit_lock); + } + + if ((*ctx)->start) return 0; + + if (int err = graph->file->ha_index_init(IDX_LAYER, 1)) return err; + + int err = graph->file->ha_index_last(graph->record[0]); + graph->file->ha_index_end(); + if (err) return err; + + graph->file->position(graph->record[0]); + (*ctx)->set_lengths( + FVector::data_to_value_size(graph->field[FIELD_VEC]->data_length())); + + /* MDEV-35834 Server crash in FVector::distance_to upon concurrent SELECT */ + auto node = (*ctx)->get_node(graph->file->ref); + if ((err = node->load_from_record(graph))) return err; + + (*ctx)->start = node; // set the shared start only when node is fully loaded + return 0; +} + +/* copy the vector, preprocessed as needed */ +const FVector *FVectorNode::make_vec(const void *v) { + return FVector::create(ctx->metric, tref() + tref_len(), v, ctx->byte_len); +} + +FVectorNode::FVectorNode(MHNSW_Share *ctx_, const void *gref_) + : ctx(ctx_), stored(true), deleted(false) { + memcpy(gref(), gref_, gref_len()); +} + +FVectorNode::FVectorNode(MHNSW_Share *ctx_, const void *tref_, uint8_t layer, + const void *vec_) + : ctx(ctx_), stored(false), deleted(false) { + assert(tref_); + memset(gref(), 0xff, gref_len()); // important: larger than any real gref + memcpy(tref(), tref_, tref_len()); + vec = make_vec(vec_); + + alloc_neighborhood(layer); +} + +float FVectorNode::distance_to(const FVector *other) const { + return vec->distance_to(other, ctx->vec_len); +} + +int FVectorNode::alloc_neighborhood(uint8_t layer) { + if (neighbors) return 0; + max_layer = layer; + neighbors = (Neighborhood *)ctx->alloc_neighborhood(layer); + auto ptr = (FVectorNode **)(neighbors + (layer + 1)); + for (size_t i = 0; i <= layer; i++) + ptr = neighbors[i].init(ptr, ctx->max_neighbors(i)); + return 0; +} + +int FVectorNode::load(TABLE *graph) { + if (likely(vec)) return 0; + + assert(stored); + // trx: consider loading nodes from shared, when it makes sense + // for ann_benchmarks it does not + if (int err = graph->file->ha_rnd_pos(graph->record[0], gref())) return err; + return load_from_record(graph); +} + +int FVectorNode::load_from_record(TABLE *graph) { + assert(ctx->byte_len); + + uint ticket = ctx->lock_node(this); + Scope_guard guard{[this, ticket]() { ctx->unlock_node(ticket); }}; + + if (vec) return 0; + + String buf, *v = graph->field[FIELD_TREF]->val_str(&buf); + deleted = graph->field[FIELD_TREF]->is_null(); + if (!deleted) { + if (unlikely(v->length() != tref_len())) return HA_ERR_CRASHED; + memcpy(tref(), v->ptr(), v->length()); + } + + v = graph->field[FIELD_VEC]->val_str(&buf); + if (unlikely(!v)) return HA_ERR_CRASHED; + + if (v->length() != FVector::data_size(ctx->vec_len)) return HA_ERR_CRASHED; + FVector *vec_ptr = FVector::align_ptr(tref() + tref_len()); + memcpy(vec_ptr->data(), v->ptr(), v->length()); + vec_ptr->postprocess(ctx->vec_len); + + longlong layer = graph->field[FIELD_LAYER]->val_int(); + if (layer > 100) // 10e30 nodes at M=2, more at larger M's + return HA_ERR_CRASHED; + + if (int err = alloc_neighborhood(static_cast(layer))) return err; + + v = graph->field[FIELD_NEIGHBORS]->val_str(&buf); + if (unlikely(!v)) return HA_ERR_CRASHED; + + // ... ...etc... + uchar *ptr = (uchar *)v->ptr(), *end = ptr + v->length(); + for (size_t i = 0; i <= max_layer; i++) { + if (unlikely(ptr >= end)) return HA_ERR_CRASHED; + size_t grefs = *ptr++; + if (unlikely(ptr + grefs * gref_len() > end)) return HA_ERR_CRASHED; + neighbors[i].num = grefs; + for (size_t j = 0; j < grefs; j++, ptr += gref_len()) + neighbors[i].links[j] = ctx->get_node(ptr); + } + vec = vec_ptr; // must be done at the very end + return 0; +} + +void FVectorNode::push_neighbor(size_t layer, FVectorNode *other) { + assert(neighbors[layer].num < ctx->max_neighbors(layer)); + neighbors[layer].links[neighbors[layer].num++] = other; +} + +size_t FVectorNode::tref_len() const { return ctx->tref_len; } +size_t FVectorNode::gref_len() const { return ctx->gref_len; } +uchar *FVectorNode::gref() const { return (uchar *)(this + 1); } +uchar *FVectorNode::tref() const { return gref() + gref_len(); } + +const uchar *FVectorNode::get_key(const void *elem, size_t *key_len, my_bool) { + *key_len = static_cast(elem)->gref_len(); + return static_cast(elem)->gref(); +} + +int FVectorNode::save(TABLE *graph) { + assert(vec); + assert(neighbors); + + restore_record(graph, s->default_values); + graph->field[FIELD_LAYER]->store(max_layer, false); + if (deleted) + graph->field[FIELD_TREF]->set_null(); + else { + graph->field[FIELD_TREF]->set_notnull(); + graph->field[FIELD_TREF]->store((const char *)tref(), tref_len(), + &my_charset_bin); + } + graph->field[FIELD_VEC]->store((const char *)vec->data(), + FVector::data_size(ctx->vec_len), + &my_charset_bin); + + size_t total_size = 0; + for (size_t i = 0; i <= max_layer; i++) + total_size += 1 + gref_len() * neighbors[i].num; + + uchar *neighbor_blob = + static_cast(my_safe_alloca(total_size, MAX_ALLOCA_SZ)); + uchar *ptr = neighbor_blob; + for (size_t i = 0; i <= max_layer; i++) { + *ptr++ = (uchar)(neighbors[i].num); + for (size_t j = 0; j < neighbors[i].num; j++, ptr += gref_len()) + memcpy(ptr, neighbors[i].links[j]->gref(), gref_len()); + } + graph->field[FIELD_NEIGHBORS]->store((const char *)neighbor_blob, total_size, + &my_charset_bin); + + int err; + if (stored) { + if (!(err = graph->file->ha_rnd_pos(graph->record[1], gref()))) { + err = graph->file->ha_update_row(graph->record[1], graph->record[0]); + if (err == HA_ERR_RECORD_IS_THE_SAME) err = 0; + } + } else { + err = graph->file->ha_write_row(graph->record[0]); + graph->file->position(graph->record[0]); + memcpy(gref(), graph->file->ref, gref_len()); + stored = true; + ctx->cache_node(this); + } + my_safe_afree(neighbor_blob, total_size, MAX_ALLOCA_SZ); + return err; +} + +/* -------------------- External Functions Definition -------------------- */ +std::unique_ptr create_dd_table(THD *thd, const char *table_name, + KEY *key, dd::Table *dd_table, + TABLE *table, const char *db_name, + const uint tref_len) { + assert(key->flags & HA_VECTOR); + assert(key->user_defined_key_parts == 1); + + const dd::Schema *schema = nullptr; + if (thd->dd_client()->acquire(db_name, &schema)) { + return nullptr; // Error is reported by the dictionary subsystem. + } + + if (schema == nullptr) { + my_error(ER_BAD_DB_ERROR, MYF(0), db_name); + return nullptr; + } + + std::unique_ptr tab_obj(schema->create_table(thd)); + dd::Table *hlindex_dd = tab_obj.get(); + + /* Basic */ + hlindex_dd->set_name(table_name); + hlindex_dd->set_hidden(dd::Abstract_table::HT_HIDDEN_HLINDEX); + hlindex_dd->set_engine(dd_table->engine()); + hlindex_dd->set_comment( + dd::String_type(key->comment.str, key->comment.length)); + hlindex_dd->set_row_format(dd_table->row_format()); + hlindex_dd->set_partition_type(dd::Table::PT_NONE); + hlindex_dd->set_subpartition_type(dd::Table::ST_NONE); + /* some properties have been set in Schema_impl::create_table(). */ + + /* Options */ + dd::Properties *hlindex_options = &hlindex_dd->options(); + dd::Properties *table_options = &dd_table->options(); + + hlindex_options->set("pack_record", true); + hlindex_options->set("avg_row_length", 0); + hlindex_options->set("stats_sample_pages", 0); + hlindex_options->set("keys_disabled", 0); + hlindex_options->set("stats_auto_recalc", HA_STATS_AUTO_RECALC_DEFAULT); + + copy_option(hlindex_options, table_options, "compress"); + copy_option(hlindex_options, table_options, "encrypt_type"); + copy_option(hlindex_options, table_options, "storage"); + copy_option(hlindex_options, table_options, "key_block_size"); + + /* Options for vector */ + hlindex_options->set("__vector_m__", key->vector_m); + hlindex_options->set("__vector_distance__", key->vector_distance); + /* fieldnr is started from 0 during creating. but we record the number of + ordinal_position in dd::Index_element */ + hlindex_options->set("__vector_column__", key->key_part->fieldnr + 1); + + /* Columns */ + char type_tref[255]; + snprintf(type_tref, sizeof(type_tref), "varbinary(%d)", tref_len); + const char *name_layer = "layer"; + const char *name_tref = "tref"; + dd::Column *col_layer = fill_dd_add_columns( + hlindex_dd, name_layer, "tinyint", dd::enum_column_types::TINY, 4, false, + 3, &my_charset_utf8mb4_0900_ai_ci, false, false, 1, FIELD_LAYER); + dd::Column *col_tref = fill_dd_add_columns( + hlindex_dd, name_tref, type_tref, dd::enum_column_types::VARCHAR, + tref_len, true, 0, &my_charset_bin, true, true, 0, FIELD_TREF); + fill_dd_add_columns(hlindex_dd, "vec", "blob", dd::enum_column_types::BLOB, + 65535, false, 0, &my_charset_bin, true, true, + 2 + portable_sizeof_char_ptr, FIELD_VEC); + fill_dd_add_columns(hlindex_dd, "neighbors", "blob", + dd::enum_column_types::BLOB, 65535, false, 0, + &my_charset_bin, true, true, 2 + portable_sizeof_char_ptr, + FIELD_NEIGHBORS); + /* system columns like ROW_ID will be added in storage engine. */ + + /* Indexes */ + fill_dd_add_indexes(hlindex_dd, name_tref, true, IDX_TREF, col_tref, + tref_len); + fill_dd_add_indexes(hlindex_dd, name_layer, false, IDX_LAYER, col_layer, 1); + /* Primary key will be added in storage engine. */ + + /* Add the InnoDB system columns DB_ROW_ID, DB_TRX_ID, DB_ROLL_PTR. */ + if (table->file->get_extra_columns_and_keys(nullptr, nullptr, nullptr, 0, + hlindex_dd)) { + return nullptr; + } + + return tab_obj; +} + +int mhnsw_insert(TABLE *table, KEY *keyinfo) { + THD *thd = table->in_use; + TABLE *graph = table->hlindex; + /* For ASSERT_COLUMN_MARKED_FOR_READ in val_str() */ + my_bitmap_map *old_map = dbug_tmp_use_all_columns(table, table->read_set); + Field *vec_field = keyinfo->key_part->field; + String buf, *res = vec_field->val_str(&buf); + MHNSW_Share *ctx; + + /* metadata are checked on open */ + assert(graph); + assert(keyinfo->flags & HA_VECTOR); + assert(vec_field->binary()); + assert(vec_field->cmp_type() == STRING_RESULT); + assert(res); // ER_INDEX_CANNOT_HAVE_NULL + assert(table->file->ref_length <= graph->field[FIELD_TREF]->field_length); + assert(res->length() > 0 && res->length() % 4 == 0); + + table->file->position(table->record[0]); + + int err = MHNSW_Share::acquire(&ctx, table, true); + Scope_guard guard_ctx{[ctx, table]() { ctx->release(table); }}; + + DBUG_EXECUTE_IF("failed_before_vidx_dml", { + my_error(ER_VECTOR_INDEX_USAGE, MYF(0), "debug failed before vidx dml."); + err = 1; + }); + + if (err) { + if (err != HA_ERR_END_OF_FILE) { + func_end: + dbug_tmp_restore_column_map(table->read_set, old_map); + + return err; + } + + // First insert! + ctx->set_lengths(res->length()); + FVectorNode *target = new (ctx->alloc_node()) + FVectorNode(ctx, table->file->ref, 0, res->ptr()); + if (!((err = target->save(graph)))) ctx->start = target; + goto func_end; + } + + if (ctx->byte_len != res->length()) return HA_ERR_CRASHED; + + MEM_ROOT temp_root(key_memory_vidx_mem, MEM_ROOT_BLOCK_SIZE); + MEM_ROOT *saved_mem_root = thd->mem_root; + thd->mem_root = &temp_root; + + Scope_guard guard_mem{ + [thd, saved_mem_root]() { thd->mem_root = saved_mem_root; }}; + + const size_t max_found = ctx->max_neighbors(0); + Neighborhood candidates; + candidates.init(new (thd->mem_root) FVectorNode *[max_found + 7], max_found); + candidates.links[candidates.num++] = ctx->start; + + const double NORMALIZATION_FACTOR = 1 / std::log(ctx->M); + double log = -std::log(my_rnd(&thd->rand)) * NORMALIZATION_FACTOR; + const uint8_t max_layer = candidates.links[0]->max_layer; + uint8_t target_layer = + std::min(static_cast(std::floor(log)), max_layer + 1); + int cur_layer; + + FVectorNode *target = new (ctx->alloc_node()) + FVectorNode(ctx, table->file->ref, target_layer, res->ptr()); + + if ((err = graph->file->ha_rnd_init(0)) > 0) goto func_end; + Scope_guard guard_graph{[graph]() { graph->file->ha_rnd_end(); }}; + + for (cur_layer = max_layer; cur_layer > target_layer; cur_layer--) { + if ((err = search_layer(ctx, graph, target->vec, NEAREST, 1, cur_layer, + &candidates, false)) > 0) + goto func_end; + } + + for (; cur_layer >= 0; cur_layer--) { + uint max_neighbors = ctx->max_neighbors(cur_layer); + if ((err = search_layer(ctx, graph, target->vec, NEAREST, max_neighbors, + cur_layer, &candidates, true)) > 0) + goto func_end; + + if ((err = select_neighbors(ctx, graph, cur_layer, *target, candidates, 0, + max_neighbors)) > 0) + goto func_end; + } + + if ((err = target->save(graph)) > 0) goto func_end; + + if (target_layer > max_layer) ctx->start = target; + + for (cur_layer = target_layer; cur_layer >= 0; cur_layer--) { + if ((err = update_second_degree_neighbors(ctx, graph, cur_layer, target)) > + 0) + goto func_end; + } + + goto func_end; +} + +int mhnsw_read_first(TABLE *table, KEY *, Item *dist) { + THD *thd = table->in_use; + TABLE *graph = table->hlindex; + auto *fun = static_cast(dist->real_item()); + ulonglong limit = fun->get_limit(); + assert(fun); + + limit = std::min(limit, max_ef); + + String buf, *res = fun->get_const_arg()->val_str(&buf); + MHNSW_Share *ctx; + + /* removed into IndexScanIterator::Init() */ + /* if (int err = table->file->ha_rnd_init(0)) return err; */ + + int err = MHNSW_Share::acquire(&ctx, table, false); + Scope_guard guard{[ctx, table]() { ctx->release(table); }}; + if (err) return err; + + Neighborhood candidates; + candidates.init(new (thd->mem_root) FVectorNode *[limit + 7], limit); + + // one could put all max_layer nodes in candidates + // but it has no effect on the recall or speed + candidates.links[candidates.num++] = ctx->start; + + /* the length of const_arg has been checked in + Item_func_vec_distance::get_key() */ + assert(res != nullptr); + assert(ctx->byte_len == res->length()); + + const longlong max_layer = candidates.links[0]->max_layer; + auto target = FVector::create( + ctx->metric, thd_alloc(thd, FVector::alloc_size(ctx->vec_len)), + res->ptr(), res->length()); + + if ((err = graph->file->ha_rnd_init(0)) > 0) return err; + + for (size_t cur_layer = max_layer; cur_layer > 0; cur_layer--) { + if ((err = search_layer(ctx, graph, target, NEAREST, 1, cur_layer, + &candidates, false)) > 0) { + graph->file->ha_rnd_end(); + return err; + } + } + + if ((err = search_layer(ctx, graph, target, NEAREST, static_cast(limit), + 0, &candidates, false)) > 0) { + graph->file->ha_rnd_end(); + return err; + } + + auto result = new (thd->mem_root) Search_context(&candidates, ctx, target); + graph->context = result; + + return mhnsw_read_next(table); +} + +int mhnsw_read_next(TABLE *table) { + auto result = static_cast(table->hlindex->context); + if (result->pos < result->found.num) { + uchar *ref = result->found.links[result->pos++]->tref(); + return table->file->ha_rnd_pos(table->record[0], ref); + } + if (!result->found.num) return HA_ERR_END_OF_FILE; + + TABLE *graph = table->hlindex; + MHNSW_Share *ctx = result->ctx->dup(table->file->has_transactions()); + Scope_guard guard_ctx{[&ctx, table]() { ctx->release(table); }}; + + if (ctx->version != result->ctx_version) { + // oops, shared ctx was modified, need to switch to MHNSW_Trx + MHNSW_Share *trx; + graph->file->ha_rnd_end(); + int err = MHNSW_Share::acquire(&trx, table, true); + Scope_guard guard_trx{[&trx, table]() { trx->release(table); }}; + if (int err2 = graph->file->ha_rnd_init(0)) err = err ? err : err2; + if (err) return err; + for (size_t i = 0; i < result->found.num; i++) { + FVectorNode *node = trx->get_node(result->found.links[i]->gref()); + if (!node) return HA_ERR_OUT_OF_MEM; + if ((err = node->load(graph))) return err; + result->found.links[i] = node; + } + ctx->release(false, table->s); // release shared ctx + result->ctx = trx->dup(false); // replace it with trx + result->ctx_version = trx->version; + std::swap(trx, ctx); // free shared ctx in this scope, keep trx + } + + float new_threshold = + result->found.links[result->found.num - 1]->distance_to(result->target); + + if (int err = search_layer(ctx, graph, result->target, result->threshold, + static_cast(result->pos), 0, &result->found, + false)) + return err; + result->pos = 0; + result->threshold = new_threshold + FLT_EPSILON; + return mhnsw_read_next(table); +} + +int mhnsw_read_end(TABLE *table) { + auto result = static_cast(table->hlindex->context); + result->ctx->release(false, table->s); + table->hlindex->context = 0; + table->hlindex->file->ha_index_or_rnd_end(); + return 0; +} + +void mhnsw_free(TABLE_SHARE *share) { + TABLE_SHARE *graph_share = share->hlindex; + if (!graph_share->hlindex_data) return; + + static_cast(graph_share->hlindex_data)->~MHNSW_Share(); + graph_share->hlindex_data = nullptr; +} + +int mhnsw_invalidate(TABLE *table, const uchar *rec, + KEY *keyinfo [[maybe_unused]]) { + TABLE *graph = table->hlindex; + handler *h = table->file; + MHNSW_Share *ctx; + + int err = MHNSW_Share::acquire(&ctx, table, true); + Scope_guard guard_ctx{[ctx, table]() { ctx->release(table); }}; + + DBUG_EXECUTE_IF("failed_before_vidx_dml", { + my_error(ER_VECTOR_INDEX_USAGE, MYF(0), "debug failed before vidx dml."); + return 1; + }); + + if (err) return err; + + /* metadata are checked on open */ + assert(graph); + assert(keyinfo->flags & HA_VECTOR); + assert(h->ref_length <= graph->field[FIELD_TREF]->field_length); + + // target record: + h->position(rec); + graph->field[FIELD_TREF]->set_notnull(); + graph->field[FIELD_TREF]->store((const char *)h->ref, h->ref_length, + &my_charset_bin); + + uchar *key = (uchar *)alloca(graph->key_info[IDX_TREF].key_length); + key_copy(key, graph->record[0], &graph->key_info[IDX_TREF], + graph->key_info[IDX_TREF].key_length); + + if ((err = graph->file->ha_index_read_idx_map( + graph->record[1], IDX_TREF, key, HA_WHOLE_KEY, HA_READ_KEY_EXACT)) > + 0) + return err; + + restore_record(graph, record[1]); + graph->field[FIELD_TREF]->set_null(); + if ((err = graph->file->ha_update_row(graph->record[1], graph->record[0])) > + 0) + return err; + + graph->file->position(graph->record[0]); + FVectorNode *node = ctx->get_node(graph->file->ref); + node->deleted = true; + + return 0; +} + +int mhnsw_delete_all(TABLE *table, KEY *keyinfo [[maybe_unused]]) { + TABLE *graph = table->hlindex; + + /* metadata are checked on open */ + assert(graph); + assert(keyinfo->flags & HA_VECTOR); + + DBUG_EXECUTE_IF("failed_before_vidx_dml", { + my_error(ER_VECTOR_INDEX_USAGE, MYF(0), "debug failed before vidx dml."); + return 1; + }); + + if (int err = graph->file->delete_all_rows()) return err; + + MHNSW_Share *ctx; + if (!MHNSW_Share::acquire(&ctx, table, true)) { + ctx->reset(table->s); + } + + /* MDEV-36758: always release ctx in mhnsw_delete_all */ + ctx->release(table); + + return 0; +} +} // namespace hnsw +} // namespace vidx diff --git a/sql/vidx/vidx_index.cc b/sql/vidx/vidx_index.cc new file mode 100644 index 00000000000..636d1255ee6 --- /dev/null +++ b/sql/vidx/vidx_index.cc @@ -0,0 +1,1162 @@ +/* Copyright (c) 2025, 2025, Alibaba and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include +#include "mysql/plugin.h" // MYSQL_DAEMON_PLUGIN, ... +#include "scope_guard.h" // Scope_guard +#include "sql/dd/cache/dictionary_client.h" // dd::cache::Dictionary_client +#include "sql/dd/dd_table.h" // dd::table_exists, ... +#include "sql/dd/dictionary.h" // dd::release_mdl +#include "sql/dd/string_type.h" // dd::String_type +#include "sql/dd/types/index.h" // dd::Index +#include "sql/dd/types/index_element.h" // dd::Index_element +#include "sql/dd_table_share.h" // open_table_def +#include "sql/field.h" // Field +#include "sql/handler.h" // setup_transaction_participant +#include "sql/item_strfunc.h" // vidx::Item_func_vec_distance +#include "sql/join_optimizer/access_path.h" // AccessPath +#include "sql/mysqld.h" // reg_ext_length +#include "sql/sql_base.h" // EXTRA_RECORD +#include "sql/sql_class.h" // THD +#include "sql/sql_lex.h" // LEX +#include "sql/sql_plugin_ref.h" // st_plugin_int +#include "sql/sql_select.h" // JOIN_TAB +#include "sql/sql_table.h" // mysql_rename_table + +#include "vidx/vidx_hnsw.h" +#include "vidx/vidx_index.h" + +namespace vidx { +/* -------------------- Macros -------------------- */ +#define VIDX_NAME "vidx_%016lx_%02x" +static constexpr uint VIDX_NAME_LEN = 4 + 1 + 16 + 1 + 2 + 1; +static constexpr uint vidx_num = 0; + +#define TL_FIRST_WRITE TL_WRITE_ALLOW_WRITE + +static constexpr uint32_t SCAN_COST = 4; + +/* -------------------- External Vars Definition -------------------- */ +st_plugin_int *vidx_plugin; +bool feature_disabled = false; + +/* -------------------- Static Vars Definition -------------------- */ +static TYPELIB distances = {array_elements(distance_names) - 1, "", + distance_names, nullptr}; + +static MYSQL_SYSVAR_BOOL(disabled, feature_disabled, PLUGIN_VAR_RQCMDARG, + "Whether to enable vector index feature", nullptr, + nullptr, true); + +static MYSQL_THDVAR_ENUM(default_distance, PLUGIN_VAR_RQCMDARG, + "Distance function to build the vector index for", + nullptr, nullptr, EUCLIDEAN, &distances); + +static MYSQL_THDVAR_UINT( + hnsw_default_m, PLUGIN_VAR_RQCMDARG, + "Larger values mean slower SELECTs and INSERTs, larger index size " + "and higher memory consumption but more accurate results", + nullptr, nullptr, hnsw::M_DEF, hnsw::M_MIN, hnsw::M_MAX, 1); + +static MYSQL_THDVAR_UINT( + hnsw_ef_search, PLUGIN_VAR_RQCMDARG, + "Larger values mean slower SELECTs but more accurate results. " + "Defines the minimal number of result candidates to look for in the " + "vector index for ORDER BY ... LIMIT N queries. The search will never " + "search for less rows than that, even if LIMIT is smaller", + nullptr, nullptr, 20, 1, hnsw::max_ef, 1); + +static MYSQL_SYSVAR_ULONGLONG(hnsw_cache_size, hnsw::max_cache_size, + PLUGIN_VAR_RQCMDARG, + "Upper limit for one HNSW vector index cache", + nullptr, nullptr, hnsw::DEF_CACHE_SIZE, + 1024 * 1024, ULONG_LONG_MAX, 1); + +static SYS_VAR *sys_vars[] = { + MYSQL_SYSVAR(disabled), MYSQL_SYSVAR(default_distance), + MYSQL_SYSVAR(hnsw_default_m), MYSQL_SYSVAR(hnsw_ef_search), + MYSQL_SYSVAR(hnsw_cache_size), nullptr}; + +static struct st_mysql_storage_engine daemon = {MYSQL_DAEMON_INTERFACE_VERSION}; + +/* -------------------- Static Functions Definition -------------------- */ +static int plugin_init(void *p) { + vidx_plugin = (st_plugin_int *)p; + vidx_plugin->data = hnsw::trx_handler; + + if (setup_transaction_participant(vidx_plugin)) return 1; + + return 0; +} + +static int plugin_deinit(void *) { return 0; } + +/* Get the string value in the dd table's option "__hlindexes__". +@param[in] dd_table the dd table +@param[out] hlindexes the value of the option "__hlindexes__" */ +static inline void dd_table_get_hlindexes(const dd::Table *dd_table, + dd::String_type *hlindexes) { + assert(hlindexes != nullptr); + assert(dd_table_has_hlindexes(dd_table)); + + dd_table->options().get("__hlindexes__", hlindexes); + assert(!hlindexes->empty()); +} + +/* Set the string value in the dd table's option "__hlindexes__". +@param[in] dd_table the dd table +@param[in] hlindexes the value of the option "__hlindexes__" */ +static inline void dd_table_set_hlindexes(dd::Table *dd_table, + dd::String_type hlindexes) { + dd_table->options().set("__hlindexes__", hlindexes); +} + +static inline uint get_tref_len(TABLE *table) { + assert(table->s->keys == 0 || + strcmp(table->key_info[0].name, primary_key_name) != 0 || + table->key_info[0].flags & HA_NOSAME); + + return (table->s->keys == 0 || + strcmp(table->key_info[0].name, primary_key_name) != 0) + ? DATA_ROW_ID_LEN + : table->key_info[0].key_length; +} + +static const char *build_name(THD *thd, const uint64_t base, const uint num, + std::string &error_message) { + /* The length of vector index table name should be shorter than 64 because + the `name` of `tables` is varchar(64). See also Tables::Tables() in + sql/dd/impl/tables/tables.cc */ + static_assert(VIDX_NAME_LEN <= 64); + + char *name = reinterpret_cast(thd->mem_root->Alloc(VIDX_NAME_LEN)); + + if (name == nullptr) { + error_message = "Failed to allocate memory for table name."; + } else { + snprintf(name, VIDX_NAME_LEN, VIDX_NAME, base, num); + } + + return reinterpret_cast(name); +} + +static bool request_mdl_lock(THD *thd, const char *db_name, + const char *table_name, enum_mdl_type mdl_type, + enum_mdl_duration lock_duration, + std::string &error_message, + MDL_request *mdl_request = nullptr) { + if (mdl_request == nullptr) { + mdl_request = new (thd->mem_root) MDL_request; + + if (mdl_request == nullptr) { + error_message = "Failed to allocate memory for mdl_request."; + return true; + } + } + + MDL_REQUEST_INIT(mdl_request, MDL_key::TABLE, db_name, table_name, mdl_type, + lock_duration); + if (thd->mdl_context.acquire_lock(mdl_request, + thd->variables.lock_wait_timeout)) { + error_message = "Failed to acquire DML lock."; + return true; + } + + return false; +} + +static const dd::Table *open_hlindex_dd(THD *thd, const char *hlindex_name, + const char *db_name, + std::string &error_message) { + /* Acquire the dd table */ + const dd::Table *hlindex_dd = nullptr; + dd::cache::Dictionary_client::Auto_releaser releaser(thd->dd_client()); + + if (thd->dd_client()->acquire(db_name, hlindex_name, &hlindex_dd)) { + error_message = "Failed to acquire vector dd table."; + return nullptr; + } + + if (hlindex_dd == nullptr || !dd_table_is_hlindex(hlindex_dd)) { + assert(false); + + error_message = "Can't find vector table in dd."; + return nullptr; + } + + assert(hlindex_dd->hidden() == dd::Abstract_table::HT_HIDDEN_HLINDEX || + hlindex_dd->hidden() == dd::Abstract_table::HT_HIDDEN_DDL); + assert(dd_table_is_hlindex(hlindex_dd)); + + return hlindex_dd; +} + +static std::string sql_regex_replacement( + const std::string &sql, const std::regex &pattern, + std::string (*replacement)(const std::string &)) { + std::string result; + size_t last_pos = 0; + std::sregex_iterator end; + + for (std::sregex_iterator it(sql.begin(), sql.end(), pattern); it != end; + ++it) { + std::smatch match = *it; + if (!match[1].matched) break; + + result += sql.substr(last_pos, match.position() - last_pos); + result += replacement(match[1].str()); + last_pos = match.position() + match.length(); + } + + if (last_pos == 0) { + /* No match */ + return sql; + } + + return result + sql.substr(last_pos); +} + +static void rewrite_sql(THD *thd, const std::string &result) { + /* Reset the thd's query string */ + char *new_query = + strmake_root(thd->mem_root, result.c_str(), result.length()); + + if (new_query == nullptr) { + my_error(ER_DA_OOM, MYF(0)); + assert(false); + + return; + } + + thd->set_query(new_query, strlen(new_query)); +} + +static std::string replacement_vector(const std::string &catching) { + return std::string(RDS_COMMENT_VIDX_START "vector(") + catching + + std::string(")" RDS_COMMENT_VIDX_END " varbinary(") + + std::to_string(4 * stoi(catching)) + std::string(")"); +} + +// Rewrite the sql string. Replace vector(X) to +// /*!99999 vector(X) */ varbinary(4 * X) +// But avoid double replacement like: +// /*!99999 vector(X) */ varbinary(4 * X) => +// /*!99999 /*!99999 vector(X) */ varbinary(4 * X) */ varbinary(4 * X) +static void rewrite_sql_of_vector_column(THD *thd) { + /* Don't use '\b' after '\)' because it will not match space. */ + /* First check if the query already contains the processed format */ + std::string query_str = to_string(thd->query()); + + /* Pattern to match already processed vector declarations */ + std::regex processed_pattern( + R"(/\*!99999 vector\(\d+\) \*/ varbinary\(\d+\))"); + + /* If not contain processed format, process the query */ + if (!std::regex_search(query_str, processed_pattern)) { + rewrite_sql( + thd, sql_regex_replacement(query_str, + std::regex{R"(\bvector\s*\(\s*(\d+)\s*\))", + std::regex_constants::icase}, + replacement_vector)); + } + + thd->m_query_has_vector_column = false; + /* TODO: There is little possible that user use vector(x) as the name of + table or other objects. */ +} + +/* Check if in one ddl query, other operations is performed while alter a +vector index. there are 3 results: +0: alter vector index, and there are other operations performed in one query. + --> set my_error and return true. +1: alter vector index, and there are not other operations performed. + --> Replace the whole query string to be inside comment with version 99999. +2: not alter vector index, thus the keywords of vector column must be parsed. + --> rewrite the keywords of vector column in the query. +*/ +static uint check_alter_vector_ddl(THD *thd, Alter_info *alter_info, + const uint key_count, + const uint old_key_count, KEY *old_vidx, + KEY *new_vidx) { + assert(old_vidx == nullptr || old_vidx->flags & HA_VECTOR); + assert(new_vidx == nullptr || new_vidx->flags & HA_VECTOR); + + if (old_vidx == nullptr) { + assert(new_vidx != nullptr); + + if (alter_info->flags == Alter_info::ALTER_ADD_INDEX) { + /* ADD a vector index. */ + assert(!thd->m_query_has_vector_column); + assert(key_count > old_key_count); + + if ((key_count - old_key_count) == 1) { + /* There are not other operations performed. */ + return 1; + } + } + } else if (new_vidx == nullptr) { + assert(old_vidx != nullptr); + + if (alter_info->flags == Alter_info::ALTER_DROP_INDEX) { + /* DROP a vector index. */ + assert(!thd->m_query_has_vector_column); + assert(old_key_count > key_count); + + if ((old_key_count - key_count) == 1) { + /* There are not other operations performed. */ + return 1; + } + } + } else if (my_strcasecmp(system_charset_info, new_vidx->name, + old_vidx->name) != 0) { + /* RENAME a vector index. */ + if (alter_info->flags == Alter_info::ALTER_RENAME_INDEX && + alter_info->alter_rename_key_list.size() == 1) { + /* There are not other operations performed. */ + assert(!thd->m_query_has_vector_column); + assert(my_strcasecmp(system_charset_info, + alter_info->alter_rename_key_list[0]->old_name, + old_vidx->name) == 0); + assert(my_strcasecmp(system_charset_info, + alter_info->alter_rename_key_list[0]->new_name, + new_vidx->name) == 0); + + return 1; + } + } else { + /* The vector index is not modified. */ + /* Attention, ALTER_INDEX_VISIBILITY is not supported for the vector index. + */ + return 2; + } + + /* There are other operations performed with the vector index in the same + ddl query. */ + return 0; +} + +/* Return true if the vector field in the rec is NULL. +Otherwise, return false. */ +static bool check_vector_is_null(TABLE *table, const uchar *rec, KEY *vec_key) { + const ptrdiff_t offset = rec - table->record[0]; + Field *field = vec_key->key_part->field; + + assert(field->is_vector()); + + return field->is_real_null(offset); +} + +/* -------------------- External Functions Definition -------------------- */ +bool check_vector_ddl_and_rewrite_sql(THD *thd, Alter_info *alter_info, + KEY *key_info, const uint key_count, + TABLE *table) { + KEY *old_vidx = table->s->get_vec_key(); + KEY *new_vidx = nullptr; + + if (key_count > 0 && vidx::key_is_vector(key_info + key_count - 1)) { + /* key_info is already sorted in mysql_prepare_create_table() */ + new_vidx = key_info + key_count - 1; + } + + if (old_vidx == nullptr && new_vidx == nullptr) { + rewrite_vector_column: + if (thd->m_query_has_vector_column) { + rewrite_sql_of_vector_column(thd); + } + + return false; + } + + switch (check_alter_vector_ddl(thd, alter_info, key_count, + table->s->total_keys, old_vidx, new_vidx)) { + case 1: + /* The DDL query only alter the vector index. */ + rewrite_sql(thd, RDS_COMMENT_VIDX_START + to_string(thd->query()) + + RDS_COMMENT_VIDX_END); + return false; + + case 2: + /* The DDL query does not alter the vector index. */ + goto rewrite_vector_column; + + default: + assert(0); + [[fallthrough]]; + + case 0: + /* The DDL query not only alter the vector index, which is not + supported yet. */ + my_error(ER_NOT_SUPPORTED_YET, MYF(0), + "perform other operations while alter a vector index"); + return true; + } +} + +namespace hnsw { +uint get_ef_search(THD *thd) { return THDVAR(thd, hnsw_ef_search); } + +uint index_options_print(const uint distance, const uint m, char *buf, + uint buf_len) { + assert(validate_index_option_distance(distance)); + assert(validate_index_option_m(m)); + + uint len = snprintf(buf, buf_len, " M=%d DISTANCE=%s" RDS_COMMENT_VIDX_END, m, + distance_names[distance]); + + if (len >= buf_len) { + return buf_len - 1; + } + + return len; +} + +bool copy_index_option_m(THD *thd, uint *to, const uint from) { + if (from == UINT_MAX) { + /* distance is not set. */ + *to = THDVAR(thd, hnsw_default_m); + return false; + } + + if (!validate_index_option_m(from)) { + return true; + } + + *to = from; + return false; +} +} // namespace hnsw + +bool copy_index_option_distance(THD *thd, uint *to, const uint from) { + if (from == UINT_MAX) { + /* distance is not set. */ + *to = THDVAR(thd, default_distance); + return false; + } + + assert(validate_index_option_distance(from)); + + *to = from; + return false; +} + +bool create_table(THD *thd, KEY *key, dd::Table *dd_table, TABLE *table, + const char *db_name, const uint64_t old_table_id) { + assert(key_is_vector(key)); + assert(dd_table->engine() == "InnoDB"); + + std::string error_message; + + /* 1. Build table name and path */ + const char *hlindex_name = + build_name(thd, dd_table->se_private_id(), vidx_num, error_message); + if (hlindex_name == nullptr) { + error_end: + my_error(ER_VECTOR_INDEX_FAILED, MYF(0), + thd_sql_command(thd) == SQLCOM_TRUNCATE ? "Truncate" : "Create", + key->name, db_name, dd_table->name().c_str(), + (hlindex_name == nullptr ? "?" : hlindex_name), + error_message.c_str()); + return true; + } + + char path[FN_REFLEN + 1]; + bool was_truncated; + build_table_filename(path, sizeof(path) - 1 - reg_ext_length, db_name, + hlindex_name, "", 0, &was_truncated); + // Check truncation, will lead to overflow when adding extension + if (was_truncated) { + my_error(ER_IDENT_CAUSES_TOO_LONG_PATH, MYF(0), sizeof(path) - 1, path); + return true; + } + + DBUG_EXECUTE_IF("crash_before_vidx_ddl", DBUG_SUICIDE();); + DBUG_EXECUTE_IF("failed_before_vidx_ddl", { + error_message = "debug failed before vidx ddl."; + goto error_end; + }); + + /* 2. Request MDL X lock */ + if (request_mdl_lock(thd, db_name, hlindex_name, MDL_EXCLUSIVE, + MDL_TRANSACTION, error_message)) { + goto error_end; + } + + /* 3. Check if the hlindex name exists. */ + bool exists; + if (dd::table_exists(thd->dd_client(), db_name, hlindex_name, &exists)) { + return true; // Error is already reported. + } + + if (!exists && + ha_check_if_table_exists(thd, db_name, hlindex_name, &exists)) { + /* Table doesn't exist. Check if some engine can provide it. */ + my_printf_error(ER_OUT_OF_RESOURCES, + "Failed to open '%-.64s', error while " + "unpacking from engine", + MYF(0), hlindex_name); + return true; + } + + if (exists) { + error_message = "Vector table name exists."; + goto error_end; + } + + if (old_table_id == dd::INVALID_OBJECT_ID) { + /* CREATE TABLE */ + /* 4. Create dd table and store it */ + std::unique_ptr hlindex_dd_ptr = hnsw::create_dd_table( + thd, hlindex_name, key, dd_table, table, db_name, get_tref_len(table)); + + if (!hlindex_dd_ptr) { + return true; // Error is already reported. + } + + if (thd->dd_client()->store(hlindex_dd_ptr.get())) { + error_message = "Failed to store vector dd table."; + goto error_end; + } + } else { + /* TRUNCATE TABLE */ + assert(old_table_id != dd_table->se_private_id()); + + /* 4. Rename old table. */ + dd::cache::Dictionary_client::Auto_releaser releaser(thd->dd_client()); + const dd::Schema *schema = nullptr; + if (thd->dd_client()->acquire(db_name, &schema) || schema == nullptr) { + error_message = "Failed to acquire schema."; + goto error_end; + } + + const char *old_name = + build_name(thd, old_table_id, vidx_num, error_message); + if (old_name == nullptr || + request_mdl_lock(thd, db_name, old_name, MDL_EXCLUSIVE, MDL_TRANSACTION, + error_message)) { + goto error_end; + } + + if (mysql_rename_table(thd, table->file->ht, db_name, old_name, db_name, + old_name, *schema, db_name, hlindex_name, + NO_DD_COMMIT | VIDX_RENAME)) { + return true; + } + } + + /* 5. Get dd table. */ + dd::Table *hlindex_dd = nullptr; + + if (thd->dd_client()->acquire_for_modification(db_name, hlindex_name, + &hlindex_dd) || + hlindex_dd == nullptr) { + error_message = "Failed to acquire_for_modification vector dd table."; + goto error_end; + } + + if (!dd_table_is_hlindex(hlindex_dd)) { + error_message = "hlindex's name may be used by other tables."; + goto error_end; + } + + /* 6. Create table */ + HA_CREATE_INFO unused; + if (ha_create_table(thd, path, db_name, hlindex_name, &unused, true, false, + hlindex_dd) != 0) { + return true; // Error is already reported. + } + + /* 7. Update dd table */ + if (thd->dd_client()->update(hlindex_dd)) { + error_message = "Failed to update vector dd table."; + goto error_end; + } + + /* 8. Set hlindexes name of base dd table */ + vidx::dd_table_set_hlindexes(dd_table, key->name); + + return false; +} + +bool delete_table(THD *thd, const dd::Table *dd_table, const char *db_name) { + assert(dd_table_has_hlindexes(dd_table)); + assert(dd_table->engine() == "InnoDB"); + + std::string error_message; + + /* 1. Build name and path. */ + const char *hlindex_name = + build_name(thd, dd_table->se_private_id(), vidx_num, error_message); + if (hlindex_name == nullptr) { + error_end: + dd::String_type key_name; + dd_table_get_hlindexes(dd_table, &key_name); + + my_error(ER_VECTOR_INDEX_FAILED, MYF(0), "Drop", key_name.c_str(), db_name, + dd_table->name().c_str(), + (hlindex_name == nullptr ? "?" : hlindex_name), + error_message.c_str()); + return true; + } + + char path[FN_REFLEN + 1]; + bool was_truncated; + build_table_filename(path, sizeof(path) - 1 - reg_ext_length, db_name, + hlindex_name, "", 0, &was_truncated); + // Check truncation, will lead to overflow when adding extension + if (was_truncated) { + my_error(ER_IDENT_CAUSES_TOO_LONG_PATH, MYF(0), sizeof(path) - 1, path); + return true; + } + + DBUG_EXECUTE_IF("crash_before_vidx_ddl", DBUG_SUICIDE();); + DBUG_EXECUTE_IF("failed_before_vidx_ddl", { + error_message = "debug failed before vidx ddl."; + goto error_end; + }); + + /* 2. Acquire the dd table with X mdl. */ + if (request_mdl_lock(thd, db_name, hlindex_name, MDL_EXCLUSIVE, + MDL_TRANSACTION, error_message)) { + goto error_end; + } + + const dd::Table *hlindex_dd = + open_hlindex_dd(thd, hlindex_name, db_name, error_message); + + if (hlindex_dd == nullptr) { + goto error_end; + } + + /* 3. Drop table */ + handlerton *hton{nullptr}; + + if (dd::table_storage_engine(thd, hlindex_dd, &hton)) { + return true; + } + + if (ha_delete_table(thd, hton, path, db_name, hlindex_name, hlindex_dd, + false)) { + return true; + } + + /* 4. remove the "__hlindexes__" option in base dd table. The base dd + table may be used later in acquire_uncached_table() to build the base + table share */ + ((dd::Table *)dd_table)->options().remove("__hlindexes__"); + + /* 5. Drop dd table */ + return dd::drop_table(thd, db_name, hlindex_name, *hlindex_dd); +} + +bool rename_table(THD *thd, dd::Table *dd_table, handlerton *base, + const dd::Schema &new_schema, const char *old_db, + const char *new_db, uint flags) { + assert(dd_table_has_hlindexes(dd_table)); + assert(dd_table->engine() == "InnoDB"); + + std::string error_message; + + /* 1. Build table name */ + const char *hlindex_name = + build_name(thd, dd_table->se_private_id(), vidx_num, error_message); + + if (hlindex_name == nullptr) { + error_end: + dd::String_type key_name; + dd_table_get_hlindexes(dd_table, &key_name); + + my_error(ER_VECTOR_INDEX_FAILED, MYF(0), "Rename", key_name.c_str(), new_db, + dd_table->name().c_str(), + (hlindex_name == nullptr ? "?" : hlindex_name), + error_message.c_str()); + return true; + } + + DBUG_EXECUTE_IF("crash_before_vidx_ddl", DBUG_SUICIDE();); + DBUG_EXECUTE_IF("failed_before_vidx_ddl", { + error_message = "debug failed before vidx ddl."; + goto error_end; + }); + + /* 2. Request MDL X lock */ + if (request_mdl_lock(thd, new_db, hlindex_name, MDL_EXCLUSIVE, + MDL_TRANSACTION, error_message) || + request_mdl_lock(thd, old_db, hlindex_name, MDL_EXCLUSIVE, + MDL_TRANSACTION, error_message)) { + goto error_end; + } + + /* 3. Rename table */ + return mysql_rename_table(thd, base, old_db, hlindex_name, old_db, + hlindex_name, new_schema, new_db, hlindex_name, + flags | VIDX_RENAME); +} + +bool build_hlindex_key(THD *thd, TABLE_SHARE *table_share, + const dd::Table *dd_table, const uint nr) { + assert(dd_table_has_hlindexes(dd_table)); + assert(table_share->hlindex == nullptr); + assert(table_share->hlindex_data == nullptr); + assert(table_share->hlindexes() == 1); + assert(nr == table_share->keys); + + std::string error_message; + + /* 1. Build name. */ + dd::String_type key_name; + dd_table_get_hlindexes(dd_table, &key_name); + + const char *hlindex_name = + build_name(thd, dd_table->se_private_id(), vidx_num, error_message); + if (hlindex_name == nullptr) { + error_end: + my_error(ER_VECTOR_INDEX_FAILED, MYF(0), "Show", key_name.c_str(), + table_share->db.str, table_share->table_name.str, + (hlindex_name == nullptr ? "?" : hlindex_name), + error_message.c_str()); + return true; + } + + /* 2. Acquire the dd table with S mdl. */ + MDL_request mdl_request; + if (request_mdl_lock(thd, table_share->db.str, hlindex_name, MDL_SHARED, + MDL_EXPLICIT, error_message, &mdl_request)) { + goto error_end; + } + + Scope_guard guard{[thd, &mdl_request]() { + if (mdl_request.ticket != nullptr) dd::release_mdl(thd, mdl_request.ticket); + }}; + + const dd::Table *hlindex_dd = + open_hlindex_dd(thd, hlindex_name, table_share->db.str, error_message); + + if (hlindex_dd == nullptr) { + goto error_end; + } + + /* 3. Build the key info. Do fill_index_from_dd() and + fill_index_elements_from_dd(). */ + KEY *vec_key = &(table_share->key_info[nr]); + KEY_PART_INFO *key_part = vec_key->key_part; + MEM_ROOT *mem_root = &(table_share->mem_root); + + /* Don't assert table_share is not temp table, because the vector index may be + in a temp table during the copy ddl. */ + + vec_key->flags = HA_VECTOR; + vec_key->name = + strmake_root(mem_root, key_name.c_str(), key_name.length() + 1); + vec_key->algorithm = HA_KEY_ALG_BTREE; + vec_key->is_algorithm_explicit = false; + vec_key->is_visible = true; + vec_key->user_defined_key_parts = 1; + vec_key->parser = nullptr; + vec_key->engine_attribute.length = 0; + vec_key->engine_attribute.str = nullptr; + vec_key->secondary_engine_attribute.length = 0; + vec_key->secondary_engine_attribute.str = nullptr; + + dd::String_type comment = hlindex_dd->comment(); + if (comment.length() > 0) { + vec_key->comment.length = comment.length(); + vec_key->comment.str = + strmake_root(mem_root, comment.c_str(), comment.length() + 1); + vec_key->flags |= HA_USES_COMMENT; + } else { + vec_key->comment.length = 0; + } + + hlindex_dd->options().get("__vector_m__", &(vec_key->vector_m)); + hlindex_dd->options().get("__vector_distance__", &(vec_key->vector_distance)); + hlindex_dd->options().get("__vector_column__", &(key_part->fieldnr)); + Field *field = key_part->field = table_share->field[key_part->fieldnr - 1]; + key_part->key_part_flag = 0; + key_part->length = field->key_length(); + key_part->offset = field->offset(table_share->default_values); + key_part->type = field->key_type(); + key_part->bin_cmp = ((field->real_type() != MYSQL_TYPE_VARCHAR && + field->real_type() != MYSQL_TYPE_STRING) || + (field->charset()->state & MY_CS_BINSORT)); + + vec_key->key_length = key_part->length; + table_share->keynames.type_names[nr] = vec_key->name; + table_share->keys_in_use.set_bit(nr); + table_share->visible_indexes.set_bit(nr); + + return false; +} + +bool test_if_cheaper_vector_ordering(JOIN_TAB *tab, ORDER *order, ha_rows limit, + int *order_idx) { + if (order == nullptr || order->next != nullptr || + order->direction != ORDER_ASC || + !is_function_of_type(*order->item, Item_func::VECTOR_DISTANCE_FUNC)) { + return false; + } + + Item_func_vec_distance *item = + down_cast(*order->item); + int item_idx = item->get_key(); + ha_rows rows; + + if (item_idx == -1) { + /* args in function are not one vector column and one const value. */ + return false; + } + + assert(item_idx >= 0); + assert((uint)item_idx >= tab->table()->s->keys); + assert((uint)item_idx < tab->table()->s->total_keys); + + if (tab->table()->force_index_order) { + /* Handle the hint about force index. */ + if (tab->table()->keys_in_use_for_order_by.is_set(item_idx)) { + if (limit == HA_POS_ERROR || limit > tab->table()->file->stats.records) { + limit = tab->table()->file->stats.records; + } + + goto use_vector_index; + } else { + return false; + } + } + + if (limit == HA_POS_ERROR && limit >= tab->table()->file->stats.records) { + return false; + } + + switch (tab->type()) { + case JT_RANGE: + rows = tab->range_scan()->num_output_rows(); + break; + + case JT_ALL: + case JT_INDEX_SCAN: + rows = tab->table()->file->stats.records; + break; + + default: + return false; + } + + static_assert(SCAN_COST > 1); + + if (tab->index() == 0) { + /* PRIMARY index scanning vs vector index scanning */ + if (limit > rows / SCAN_COST) { + return false; + } + } else if (limit >= rows) { + /* Secondary index scanning vs vector index scanning */ + return false; + } + +use_vector_index: + assert(limit <= tab->table()->file->stats.records); + assert(limit != HA_POS_ERROR); + + *order_idx = item_idx; + item->set_limit(limit); + + tab->set_type(JT_INDEX_SCAN); + tab->ref().key = item_idx; + tab->ref().key_parts = 0; + tab->set_index(item_idx); + tab->set_vec_func(item); + + return true; +} +} // namespace vidx + +using namespace vidx; +using namespace vidx::hnsw; + +int TABLE::hlindex_open(uint nr) { + assert(s->hlindexes() == 1); + assert(nr == s->keys); + + if (in_use->tx_isolation != ISO_READ_COMMITTED) { + my_error(ER_NOT_SUPPORTED_YET, MYF(0), + "other transaction isolation levels except READ COMMITTED for the " + "vector index"); + return 1; + } + + if (hlindex == nullptr) { + std::string error_message; + KEY *vec_key = s->key_info + nr; + char path[FN_REFLEN + 1]; + + /* 1.Build name. */ + const char *hlindex_name = + build_name(in_use, s->m_se_private_id, vidx_num, error_message); + + if (hlindex_name == nullptr) { + error_end: + assert(0); + my_error(ER_VECTOR_INDEX_FAILED, MYF(0), "Open", vec_key->name, s->db.str, + s->table_name.str, + (hlindex_name == nullptr ? "?" : hlindex_name), + error_message.c_str()); + return 1; + } + + /* 2. Acquire the dd table with S mdl. */ + if (request_mdl_lock(in_use, s->db.str, hlindex_name, MDL_SHARED_READ, + MDL_TRANSACTION, error_message)) { + goto error_end; + } + + const dd::Table *hlindex_dd = + open_hlindex_dd(in_use, hlindex_name, s->db.str, error_message); + + if (hlindex_dd == nullptr) { + goto error_end; + } + + /* 3. Open the shared hlindex */ + s->lock_share(); + Scope_guard guard{[this]() { s->unlock_share(); }}; + + if (s->hlindex == nullptr) { + /* Build the table key. */ + MDL_key dml_key{MDL_key::TABLE, s->db.str, hlindex_name}; + size_t key_length = dml_key.length() - 1; + const char *key = (const char *)dml_key.ptr() + 1; + + /* Open the table hlindex */ + if (s->tmp_table != NO_TMP_TABLE) { + /* Base table is temp. */ + bool was_truncated; + build_table_filename(path, sizeof(path) - 1 - reg_ext_length, s->db.str, + hlindex_name, "", 0, &was_truncated); + + s->hlindex = reinterpret_cast( + in_use->mem_root->Alloc(sizeof(TABLE_SHARE))); + + init_tmp_table_share(in_use, s->hlindex, key, key_length, + strend(key) + 1, path, nullptr); + } else if ((s->hlindex = alloc_table_share(s->db.str, hlindex_name, key, + key_length, false)) == + nullptr) { + /* Base table is normal. */ + error_message = "Failed to alloc_table_share."; + goto error_end; + } + + if (open_table_def(in_use, s->hlindex, *hlindex_dd)) { + error_message = "Failed to open_table_def."; + goto error_end; + } + + s->hlindex->is_hlindex = true; + + assert(s->hlindex->hlindex_data == nullptr); + assert(s->hlindex->hlindex == nullptr); + } + + /* 4. Open a new hlindex */ + hlindex = + (TABLE *)my_malloc(key_memory_TABLE, sizeof(*hlindex), MYF(MY_WME)); + + if (hlindex == nullptr) { + error_message = "Failed to my_malloc hlindex table."; + goto error_end; + } + + if (s->hlindex->tmp_table == NO_TMP_TABLE) { + mysql_mutex_lock(&LOCK_open); + s->hlindex->increment_ref_count(); + mysql_mutex_unlock(&LOCK_open); + } + + int error = open_table_from_share(in_use, s->hlindex, hlindex_name, + (uint)(HA_OPEN_KEYFILE | HA_OPEN_RNDFILE | + HA_GET_INDEX | HA_TRY_READ_ONLY), + EXTRA_RECORD, in_use->open_options, + hlindex, false, hlindex_dd); + + if (error != 0 || hlindex == nullptr) { + error_message = "Failed to open_table_from_share."; + goto error_end; + } + + hlindex->in_use = nullptr; + } + + return 0; +} + +int TABLE::hlindex_lock(uint nr [[maybe_unused]]) { + assert(s->hlindexes() == 1); + assert(nr == s->keys); + assert(hlindex); + + if (hlindex->in_use != in_use) { + hlindex->file->rebind_psi(); + hlindex->file->ha_extra(HA_EXTRA_RESET_STATE); + + hlindex->reset(); + hlindex->set_created(); + hlindex->use_all_columns(); + + /* mark in use for this query */ + hlindex->in_use = in_use; + /* use the main table's lock_descriptor. */ + hlindex->pos_in_table_list = pos_in_table_list; + + assert(hlindex->file->lock_count() <= 1); + + return hlindex->file->ha_external_lock( + in_use, reginfo.lock_type < TL_WRITE_ALLOW_WRITE ? F_RDLCK : F_WRLCK); + } + + return 0; +} + +int TABLE::reset_hlindexes() { + if (hlindex && hlindex->in_use) { + hlindex->in_use = nullptr; + hlindex->pos_in_table_list = nullptr; + } + + return 0; +} + +int TABLE::hlindexes_on_insert() { + assert(s->hlindexes() == 1 || s->hlindexes() == 0); + + for (uint key = s->keys; key < s->total_keys; key++) { + if (vidx::check_vector_is_null(this, record[0], key_info + key)) { + continue; + } + + int err; + + if ((err = hlindex_open(key)) || (err = hlindex_lock(key)) || + (err = mhnsw_insert(this, key_info + key))) { + return err; + } + } + + return 0; +} + +int TABLE::hlindexes_on_update() { + assert(s->hlindexes() == 1 || s->hlindexes() == 0); + + for (uint key = s->keys; key < s->total_keys; key++) { + const bool old_is_null = + vidx::check_vector_is_null(this, record[1], key_info + key); + const bool new_is_null = + vidx::check_vector_is_null(this, record[0], key_info + key); + + if (old_is_null && new_is_null) { + continue; + } + + // TODO: if tref and vector are not changed, update should be all skipped. + + int err; + + if ((err = hlindex_open(key)) || (err = hlindex_lock(key)) || + (err = old_is_null + ? 0 + : mhnsw_invalidate(this, record[1], key_info + key)) || + (err = new_is_null ? 0 : mhnsw_insert(this, key_info + key))) { + return err; + } + } + + return 0; +} + +int TABLE::hlindexes_on_delete(const uchar *buf) { + assert(s->hlindexes() == 1 || s->hlindexes() == 0); + assert(buf == record[0] || buf == record[1]); // note: REPLACE + + for (uint key = s->keys; key < s->total_keys; key++) { + if (vidx::check_vector_is_null(this, buf, key_info + key)) { + continue; + } + + int err; + + if ((err = hlindex_open(key)) || (err = hlindex_lock(key)) || + (err = mhnsw_invalidate(this, buf, key_info + key))) { + return err; + } + } + + return 0; +} + +int TABLE::hlindexes_on_delete_all() { + assert(s->hlindexes() == 1 || s->hlindexes() == 0); + + for (uint key = s->keys; key < s->total_keys; key++) { + int err; + + if ((err = hlindex_open(key)) || (err = hlindex_lock(key)) || + (err = mhnsw_delete_all(this, key_info + key))) { + return err; + } + } + + return 0; +} + +int TABLE::hlindex_read_first(uint key, void *item) { + assert(s->hlindexes() == 1); + assert(key == s->keys); + + int err; + + if ((err = hlindex_open(key)) || (err = hlindex_lock(key)) || + (err = mhnsw_read_first(this, key_info + key, (Item *)item))) { + return err; + } + + return 0; +} + +int TABLE::hlindex_read_next() { return mhnsw_read_next(this); } + +int TABLE::hlindex_read_end() { return mhnsw_read_end(this); } + +mysql_declare_plugin(vidx){ + MYSQL_DAEMON_PLUGIN, + &vidx::daemon, + "vidx", + "AliCloud", + "A plugin for vector index algorithm", /* Plugin name */ + PLUGIN_LICENSE_GPL, + vidx::plugin_init, /* Plugin Init */ + nullptr, + vidx::plugin_deinit, /* Plugin Deinit */ + 0x0100, /* Plugin Version: major.minor */ + nullptr, /* status variables */ + vidx::sys_vars, /* system variables */ + nullptr, /* config options */ + 0, /* flags */ +} mysql_declare_plugin_end; diff --git a/storage/duckdb/delta_appender.cc b/storage/duckdb/delta_appender.cc index 2ffeedbbfc6..4f23df5eb40 100644 --- a/storage/duckdb/delta_appender.cc +++ b/storage/duckdb/delta_appender.cc @@ -490,12 +490,28 @@ int DeltaAppender::append_mysql_field(const Field *field, case MYSQL_TYPE_DATETIME2: { MYSQL_TIME tm; - static_cast(field)->get_date(&tm, TIME_FUZZY_DATE); - bool not_used; - longlong sec = my_tz_UTC->TIME_to_gmt_sec(&tm, ¬_used); - // write_batch_longlong(col_index, &value); + static_cast(field)->get_date(&tm, + TIME_FUZZY_DATE); + longlong ts_us; + if (tm.month == 0) { + /* + Zero date (e.g. 0000-00-00 00:00:00) or invalid date with month=0. + Bypass TIME_to_gmt_sec() which asserts month > 0. + Use calc_daynr() to compute timestamp directly, consistent with + MYSQL_TYPE_NEWDATE handling. + */ + long days = + calc_daynr(tm.year, tm.month, tm.day) - myduck::days_at_timestart; + ts_us = static_cast(days) * 86400LL * 1000000LL + + tm.hour * 3600LL * 1000000LL + tm.minute * 60LL * 1000000LL + + tm.second * 1000000LL + tm.second_part; + } else { + bool not_used; + longlong sec = my_tz_UTC->TIME_to_gmt_sec(&tm, ¬_used); + ts_us = sec * 1000000LL + tm.second_part; + } appender->Append( - static_cast(sec * 1000000 + tm.second_part)); + static_cast(ts_us)); break; } diff --git a/storage/innobase/dict/dict0dd.cc b/storage/innobase/dict/dict0dd.cc index b87e4963e51..f0a64836e95 100644 --- a/storage/innobase/dict/dict0dd.cc +++ b/storage/innobase/dict/dict0dd.cc @@ -80,6 +80,9 @@ Data dictionary interface */ #include "univ.i" // Using OS_PATH_SEPARATOR #endif /* !UNIV_HOTBACKUP */ +#include "vidx/vidx_index.h" +static_assert(DATA_ROW_ID_LEN == vidx::DATA_ROW_ID_LEN); + const char *DD_instant_col_val_coder::encode(const byte *stream, size_t in_len, size_t *out_len) { cleanup(); @@ -337,25 +340,26 @@ int acquire_uncached_table(THD *thd, dd::cache::Dictionary_client *client, TABLE_SHARE *ts, TABLE *td) { int error = 0; dd::Schema *schema; - const char *table_cache_key; - size_t table_cache_key_len; + const char *db_name_ptr; + char db_name[NAME_LEN + 1]; if (name != nullptr) { schema = nullptr; - table_cache_key = name; - table_cache_key_len = dict_get_db_name_len(name); + size_t db_name_len = dict_get_db_name_len(name); + strncpy(db_name, name, db_name_len); + db_name[db_name_len] = '\0'; + db_name_ptr = db_name; } else { error = client->acquire_uncached(dd_table->schema_id(), &schema); if (error != 0) { return (error); } - table_cache_key = schema->name().c_str(); - table_cache_key_len = schema->name().size(); + db_name_ptr = schema->name().c_str(); } - init_tmp_table_share(thd, ts, table_cache_key, table_cache_key_len, - dd_table->name().c_str(), "" /* file name */, nullptr); + init_tmp_table_share(thd, ts, db_name_ptr, 0, dd_table->name().c_str(), + "" /* file name */, nullptr); error = open_table_def_suppress_invalid_meta_data(thd, ts, dd_table->table()); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 8a8fbed02df..806bf18a060 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -10455,6 +10455,9 @@ int ha_innobase::change_active_index( active_index = keynr; + ut_ad(active_index == MAX_KEY || table_share->keys == 0 || + active_index < table_share->keys); + m_prebuilt->index = innobase_get_index(keynr); if (m_prebuilt->index == nullptr) { diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 919d2db1d7a..0d11fba2725 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1692,6 +1692,11 @@ static dberr_t row_insert_for_mysql_using_ins_graph(const byte *mysql_rec, with a latch. */ dict_table_n_rows_inc(table); + /* vidx: Save row id as gref or tref for hlindex table. */ + if (prebuilt->clust_index_was_generated) { + ut_memcpy(prebuilt->row_id, node->row_id_buf, DATA_ROW_ID_LEN); + } + row_update_statistics_if_needed(table); trx->op_info = ""; diff --git a/storage/ndb/plugin/ndb_dd_upgrade_table.cc b/storage/ndb/plugin/ndb_dd_upgrade_table.cc index 2392c627565..bad3834d450 100644 --- a/storage/ndb/plugin/ndb_dd_upgrade_table.cc +++ b/storage/ndb/plugin/ndb_dd_upgrade_table.cc @@ -42,6 +42,9 @@ #include "storage/ndb/plugin/ndb_table_guard.h" #include "storage/ndb/plugin/ndb_thd.h" // get_thd_ndb #include "storage/ndb/plugin/ndb_thd_ndb.h" // Thd_ndb +#ifndef NDEBUG +#include "vidx/vidx_index.h" +#endif /* !NDEBUG */ namespace dd { class Schema; @@ -465,7 +468,7 @@ bool migrate_table_to_dd(THD *thd, Ndb_dd_client *dd_client, thd, schema_name.c_str(), table_name.c_str(), &create_info, &alter_info, file, true, // NDB tables are auto-partitoned. &key_info_buffer, &key_count, &dummy_fk_key_info, &dummy_fk_key_count, - nullptr, 0, nullptr, 0, 0, false /* No FKs here. */)) { + nullptr, 0, nullptr, 0, 0, false /* No FKs or vector keys here. */)) { return false; } @@ -576,6 +579,9 @@ bool migrate_table_to_dd(THD *thd, Ndb_dd_client *dd_client, thd, *schema_def, to_table_name, &create_info, alter_info.create_list, key_info_buffer, key_count, Alter_info::ENABLE, nullptr, 0, nullptr, table.file); + + assert(!vidx::dd_table_has_hlindexes(table_def.get())); + if (!table_def) { thd_ndb->push_warning(ER_DD_ERROR_CREATING_ENTRY, "Error in Creating DD entry for %s.%s", diff --git a/strings/ctype-bin.cc b/strings/ctype-bin.cc index d79c9e4d7d5..810597277c3 100644 --- a/strings/ctype-bin.cc +++ b/strings/ctype-bin.cc @@ -276,9 +276,8 @@ static void my_hash_sort_8bit_bin(const CHARSET_INFO *cs [[maybe_unused]], *nr2 = tmp2; } -static void my_hash_sort_bin(const CHARSET_INFO *cs [[maybe_unused]], - const uchar *key, size_t len, uint64 *nr1, - uint64 *nr2) { +void my_hash_sort_bin(const CHARSET_INFO *cs [[maybe_unused]], const uchar *key, + size_t len, uint64 *nr1, uint64 *nr2) { const uchar *pos = key; uint64 tmp1; uint64 tmp2; diff --git a/wiki/duckdb/duckdb-en.md b/wiki/duckdb/duckdb-en.md new file mode 100644 index 00000000000..a00847ab5c6 --- /dev/null +++ b/wiki/duckdb/duckdb-en.md @@ -0,0 +1,140 @@ +# DuckDB in AliSQL +![MySQL with DuckDB](./pic/mysql_with_duckdb.png) + +[ [AliSQL DuckDB 引擎](./duckdb-zh.md) | [DuckDB in AliSQL](./duckdb-en.md) ] + +## What is DuckDB? + +[DuckDB](https://github.com/duckdb/duckdb) is an open-source embedded analytical database system (OLAP) designed for data analysis workloads. DuckDB is rapidly becoming a popular choice in data science, BI tools, and embedded analytics scenarios due to its key characteristics: + +- **Exceptional Query Performance**: Single-node DuckDB performance not only far exceeds InnoDB, but even surpasses ClickHouse and SelectDB +- **Excellent Compression**: DuckDB uses columnar storage and automatically selects appropriate compression algorithms based on data types, achieving very high compression ratios +- **Embedded Design**: DuckDB is an embedded database system, naturally suitable for integration into MySQL +- **Plugin Architecture**: DuckDB uses a plugin-based design, making it very convenient for third-party development and feature extensions +- **Friendly License**: DuckDB's license allows any form of use, including commercial purposes + + +## Why Integrate DuckDB with AliSQL? + +MySQL has long lacked an analytical query engine. While InnoDB is naturally designed for OLTP and excels in TP scenarios, its query efficiency is very low for analytical workloads. This integration enables: + +- **Hybrid Workloads**: Run both OLTP (MySQL/InnoDB) and OLAP (DuckDB) queries in a single database system +- **High-Performance Analytics**: Analytical query performance improves up to **200x** compared to InnoDB +- **Storage Cost Reduction**: DuckDB read replicas typically use only **20%** of the main instance's storage space due to high compression +- **100% MySQL Syntax Compatibility**: No learning curve - DuckDB is integrated as a storage engine, so users continue using MySQL syntax +- **Zero Additional Management Cost**: DuckDB instances are managed, operated, and monitored exactly like regular RDS MySQL instances +- **One-Click Deployment**: Create DuckDB read-only instances with automatic data conversion from InnoDB to DuckDB + +**AliSQL** integrates **DuckDB** as a native AP engine, empowering users with high-performance, lightweight analytical capabilities while maintaining a seamless, MySQL-compatible experience. + + +## Architecture +### MySQL's Pluggable Storage Engine Architecture +MySQL's pluggable storage engine architecture allows it to extend its capabilities through different storage engines: + +![MySQL Architecture](./pic/mysql_arch.png) + +The architecture consists of four main layers: +- **Runtime Layer**: Handles MySQL runtime tasks like communication, access control, system configuration, and monitoring +- **Binlog Layer**: Manages binlog generation, replication, and application +- **SQL Layer**: Handles SQL parsing, optimization, and execution +- **Storage Engine Layer**: Manages data storage and access + +### DuckDB Read-Only Instance Architecture + +![DuckDB Architecture](./pic/duckdb_arch.png) + +DuckDB analytical read-only instances use a read-write separation architecture: +- Analytical workloads are separated from the main instance, ensuring no mutual impact +- Data replication from the main instance via binlog mechanism (similar to regular read replicas) +- InnoDB stores only metadata and system information (accounts, configurations) +- All user data resides in the DuckDB engine + +### Query Path + +![Query Path](./pic/query_path.png) + +1. Users connect via MySQL client +2. MySQL parses the query and performs necessary processing +3. SQL is sent to DuckDB engine for execution +4. DuckDB returns results to server layer +5. Server layer converts results to MySQL format and returns to client + +**Compatibility**: +- Extended DuckDB's syntax parser to support MySQL-specific syntax +- Rewrote numerous DuckDB functions and added many MySQL functions +- Automated compatibility testing platform with ~170,000 SQL tests shows **[99% compatibility rate](https://www.alibabacloud.com/help/en/rds/apsaradb-rds-for-mysql/compatibility-of-duckdb-based-analytical-instances?spm=a2c63.p38356.help-menu-26090.d_3_4_2.6a97448exEuaFG)** + +### Binlog Replication Path + +![Binlog Replication](./pic/binlog_replication.png) + + +AliSQL allows DuckDB nodes to serve as replicas via Binlog synchronization. By re-engineering the transaction commit and replay processes, AliSQL overcomes the lack of 2PC support in DuckDB, ensuring full data and metadata consistency even after abnormal crashes. + +**Idempotent Replay**: +- Since DuckDB doesn't support two-phase commit, custom transaction commit and binlog replay processes ensure data consistency after instance crashes + +**DML Replay Optimization**: +- DuckDB favors large transactions; frequent small transactions cause severe replication lag +- Implemented batch replay mechanism achieving **300K rows/s** replay capability +- In Sysbench testing, achieves zero replication lag, even higher than InnoDB replay performance +- Batch-write optimization also applies to the primary node: with our DML optimizations, INSERT and DELETE may achieve excellent performance on the primary. +![Batch commit](./pic/batch_commit.png) + +### DDL Compatibility & Optimizations + +![DDL Compatibility](./pic/ddl_support.png) + +- Natively supported DDL uses Inplace/Instant execution +- For DDL operations DuckDB doesn't natively support (e.g., column reordering), implemented Copy DDL mechanism +- Convert from InnoDB to DuckDB using multi-threaded parallel execution. Execution time reduced by **7x** +![Copy DDL from InnoDB](./pic/parallel_copy_from_innodb.png) + + +## Performance Benchmarks +**Test Environment**: +- ECS Instance: 32 CPU, 128GB Memory, ESSD PL1 Cloud Disk 500GB +- Benchmark: TPC-H SF100 + +| Query ID | DuckDB | InnoDB | ClickHouse | +| --- | --- | --- | --- | +|q1|0.92|1134.25|3.47| +|q2|0.15|1800|1.52| +|q3|0.53|802.94|3.65| +|q4|0.46|1000.45|2.77| +|q5|0.5|1800|5.38| +|q6|0.22|566.73|0.73| +|q7|0.59|1800|6.06| +|q8|0.68|1800|6.99| +|q9|1.44|1800|13.29| +|q10|0.91|894.35|3.22| +|q11|0.11|79.63|1.1| +|q12|0.44|734.35|1.69| +|q13|1.59|454.15|5.85| +|q14|0.38|574.07|0.83| +|q15|0.31|568.43|1.53| +|q16|0.32|63.56|0.52| +|q17|0.89|1800|7.96| +|q18|1.59|1800|3.11| +|q19|0.8|1800|2.96| +|q20|0.51|1800|3.38| +|q21|1.64|1800|OOM| +|q22|0.33|361.4|4| +|total|15.31|25234.31|80.01 + +DuckDB demonstrates significant performance advantages over InnoDB in analytical query scenarios, with up to **200x improvement**. + +## Try It on Alibaba Cloud +You can experience RDS MySQL with DuckDB engine on Alibaba Cloud: + +https://help.aliyun.com/zh/rds/apsaradb-rds-for-mysql/duckdb-based-analytical-instance/ + + +## See also + +- [DuckDB Variables Reference](./duckdb_variables-en.md) +- [How to Setup DuckDB Node](./how-to-setup-duckdb-node-en.md) +- [DuckDB GitHub Repository](https://github.com/duckdb/duckdb) +- [Detailed Article (Chinese)](https://mp.weixin.qq.com/s/_YmlV3vPc9CksumXvXWBEw) +- [AliSQL](https://github.com/alibaba/AliSQL.git) diff --git a/wiki/duckdb/duckdb-zh.md b/wiki/duckdb/duckdb-zh.md index e2fb878745e..37d18fb659d 100644 --- a/wiki/duckdb/duckdb-zh.md +++ b/wiki/duckdb/duckdb-zh.md @@ -1,40 +1,100 @@ -# DuckDB in AliSQL -[DuckDB](https://github.com/duckdb/duckdb) 是一个开源的在线分析处理（OLAP）和数据分析工作负载而设计。因其轻量、高性能、零配置和易集成的特性，正在迅速成为数据科学、BI 工具和嵌入式分析场景中的热门选择。DuckDB主要有以下几个特点： -- 卓越的查询性能：单机DuckDB的性能不但远高于InnoDB，甚至比ClickHouse和SelectDB的性能更好。 -- 优秀的压缩比：DuckDB采用列式存储，根据类型自动选择合适的压缩算法，具有非常高的压缩率。 -- 嵌入式设计：DuckDB是一个嵌入式的数据库系统，天然的适合被集成到MySQL中。 -- 插件化设计：DuckDB采用了插件式的设计，非常方便进行第三方的开发和功能扩展。 -- 友好的License：DuckDB的License允许任何形式的使用DuckDB的源代码，包括商业行为。 +# AliSQL 中的 DuckDB +![MySQL with DuckDB](./pic/mysql_with_duckdb.png) -基于以上的几个原因，我们认为DuckDB非常适合成为MySQL的AP存储引擎。因此我们将DuckDB集成到了AliSQL中，将强大的分析能力带给 MySQL 用户，**用户可以像使用 MySQL 一样来操作 DuckDB**。目前，您可以通过 AliSQL 快速部署一个 DuckDB 服务节点，从而实现轻量级的分析能力。 +[ [DuckDB in AliSQL](./duckdb-en.md) | [AliSQL DuckDB 引擎](./duckdb-zh.md) ] -## DuckDB 引擎的实现 -### DuckDB 查询链路 -InnoDB仅用来保存元数据和系统信息，如账号、配置等。所有的用户数据都存在DuckDB引擎中，InnoDB仅用来保存元数据和系统信息，如账号、配置等。 +## 什么是 DuckDB？ -用户通过MySQL客户端连接到实例。查询到达后，MySQL首先进行解析和必要的处理。然后将SQL发送到DuckDB引擎执行。DuckDB执行完成后，将结果返回到Server层，server层将结果集转换成MySQL的结果集返回给客户。 +[DuckDB](https://github.com/duckdb/duckdb) 是一个开源的嵌入式分析型数据库系统（OLAP），面向数据分析工作负载。凭借以下关键特性，DuckDB 正在数据科学、BI 工具以及嵌入式分析等场景中快速流行： -查询链路最重要的工作就是兼容性的工作。DuckDB和MySQL的数据类型基本上是兼容的，但在语法和函数的支持上都和MySQL有比较大的差异，为此我们扩展了DuckDB的语法解析器，使其兼容MySQL特有的语法；重写了大量的DuckDB函数并新增了大量的MySQL函数，让常见的MySQL函数都可以准确运行。自动化兼容性测试平台大约17万SQL测试，显示兼容率达到99%。详细的兼容性情况见[《DuckDB 分析实例兼容性说明》](https://help.aliyun.com/zh/rds/apsaradb-rds-for-mysql/compatibility-of-duckdb-based-analytical-instances?spm=a2c4g.11186623.help-menu-26090.d_3_4_2.37e117e8adKlF2&scm=20140722.H_2964962._.OR_help-T_cn~zh-V_1) +- **卓越的查询性能**：单机 DuckDB 的性能不仅远超 InnoDB，甚至可超过 ClickHouse 和 SelectDB +- **优秀的压缩能力**：DuckDB 采用列式存储，并会根据数据类型自动选择合适的压缩算法，压缩率非常高 +- **嵌入式设计**：DuckDB 是嵌入式数据库系统，天然适合与 MySQL 集成 +- **插件化架构**：DuckDB 采用插件式设计，便于第三方开发和功能扩展 +- **友好的许可证**：DuckDB 的许可证允许任何形式的使用，包括商业用途 -### Binlog 幂等回放 -DuckDB 节点可以作为主节点的备节点，主节点通过 Binlog 将数据同步至 DuckDB 节点。由于DuckDB不支持两阶段提交，因此无法利用两阶段提交来保证 Binlog GTID 和数据之间的一致性，也无法保证DDL操作中InnoDB的元数据和DuckDB的一致性。因此我们对事务提交的过程和 Binlog 的回放过程进行了改造，从而保证实例异常宕机重启后的数据一致性。 +## 为什么在 AliSQL 中集成 DuckDB？ -### DML 优化 -由于DuckDB本身的实现上，有利于大事务的执行。频繁小事务的执行效率非常低迟。针对 DuckDB 引擎，我们专门设计了攒批(Batch)写入的方式，来提供更高的写入性能。 +MySQL 长期缺少分析型查询引擎。InnoDB 天然面向 OLTP，在 TP 场景表现优秀，但在分析型工作负载下查询效率较低。本次集成带来： -在 DuckDB 节点作为备库的场景下我们对 Binlog 回放做了优化，采用攒批的方式进行事务重放。优化后可以达到 30w rows/s的回放能力。在Sysbench压力测试中，能够做到没有复制延迟，比InnoDB的回放性能还高。 +- **混合负载**：在同一个数据库系统中同时运行 OLTP（MySQL/InnoDB）与 OLAP（DuckDB）查询 +- **高性能分析**：相较 InnoDB，分析查询性能最高可提升 **200x** +- **降低存储成本**：由于高压缩率，DuckDB 只读从库通常仅需主实例 **20%** 的存储空间 +- **100% MySQL 语法兼容**：无学习成本——DuckDB 以存储引擎方式集成，用户仍然使用 MySQL 语法 +- **零额外管理成本**：DuckDB 实例的管理、运维和监控方式与普通 RDS MySQL 实例完全一致 +- **一键部署**：可创建 DuckDB 只读实例，并支持将 InnoDB 数据自动转换为 DuckDB -### DDL优化 -DuckDB 作为独立的存储引擎接入到 ALiSQL 内核中，ALiSQL 实现了绝大部分 DuckDB 引擎的 DDL 操作。 +**AliSQL** 将 **DuckDB** 作为原生 AP 引擎集成，在保持 MySQL 兼容与无缝体验的同时，为用户提供高性能、轻量级的分析能力。 -对于 DuckDB 原生支持的 DDL 类型，ALiSQL 通过重新定义引擎层的 prepare_inplace_alter_table、inplace_alter_table 和 commit_inplace_alter_table 等接口 ，以 inplace/instant 的方式来实现；对于 DuckDB 原生无法支持的 DDL 类型，AliSQL 通过重新定义引擎层的 rnd_init、 rnd_next、rnd_end 等接口，以重建表的方式来实现。 +## 架构 -将 InnoDB 表转为 DuckDB 引擎的操作是以重建表的方式进行的。在 InnoDB 引擎转为 DuckDB 的过程中，AlISQL 优化了重建表的过程，采用多线程并行读 InnoDB、多线程并行攒批写 DuckDB 的方式实现了高效的数据转换。 +### MySQL 可插拔存储引擎架构 +MySQL 的可插拔存储引擎架构允许其通过不同的存储引擎扩展能力： +![MySQL Architecture](./pic/mysql_arch.png) -## DuckDB 引擎的性能 -在 TPC-H sf100场景下，DuckDB 引擎的性能远超 InnoDB 引擎，相比 ClickHouse 有明显优势， -| Query ID | DuckDB | InnoDB | InnoDB | +该架构主要由四层组成： +- **运行时层（Runtime Layer）**：处理通信、访问控制、系统配置、监控等 MySQL 运行时任务 +- **Binlog 层（Binlog Layer）**：负责 Binlog 生成、复制与应用 +- **SQL 层（SQL Layer）**：负责 SQL 解析、优化与执行 +- **存储引擎层（Storage Engine Layer）**：负责数据存储与访问 + +### DuckDB 只读实例架构 + +![DuckDB Architecture](./pic/duckdb_arch.png) + +DuckDB 分析型只读实例采用读写分离架构： +- 分析类负载与主实例隔离，互不影响 +- 通过 Binlog 机制从主实例复制数据（类似普通只读从库） +- InnoDB 仅存储元数据和系统信息（账号、配置等） +- 所有用户数据均存放在 DuckDB 引擎中 + +### 查询路径（Query Path） + +![Query Path](./pic/query_path.png) + +1. 用户通过 MySQL 客户端连接 +2. MySQL 解析查询并进行必要处理 +3. 将 SQL 发送到 DuckDB 引擎执行 +4. DuckDB 将结果返回给服务端层 +5. 服务端层将结果转换为 MySQL 格式并返回给客户端 + +**兼容性**： +- 扩展 DuckDB 的语法解析器以支持 MySQL 特有语法 +- 重写大量 DuckDB 函数并新增许多 MySQL 函数 +- 自动化兼容性测试平台包含约 170,000 条 SQL 测试，显示 **[99% 兼容率](https://www.alibabacloud.com/help/en/rds/apsaradb-rds-for-mysql/compatibility-of-duckdb-based-analytical-instances?spm=a2c63.p38356.help-menu-26090.d_3_4_2.6a97448exEuaFG)** + +### Binlog 复制路径（Replication Path） + +![Binlog Replication](./pic/binlog_replication.png) + +AliSQL 允许 DuckDB 节点通过 Binlog 同步作为从库使用。通过重新设计事务提交与回放流程，AliSQL 弥补了 DuckDB 不支持 2PC 的不足，即使发生异常宕机也能确保数据与元数据的完全一致。 + +**幂等回放（Idempotent Replay）**： +- 由于 DuckDB 不支持两阶段提交（2PC），通过定制化的事务提交与 Binlog 回放流程，保证实例异常崩溃后数据一致性 + +**DML 回放优化（DML Replay Optimization）**： +- DuckDB 更偏好大事务；频繁的小事务会导致严重的复制延迟 +- 实现批量回放机制，回放能力可达 **300K 行/秒** +- Sysbench 测试中可实现零复制延迟，甚至高于 InnoDB 的回放性能 +- 批量写入优化同样适用于主库：借助 DML 优化，主库上的 INSERT 与 DELETE 也可能获得优秀性能 +![Batch commit](./pic/batch_commit.png) + +### DDL 兼容性与优化 + +![DDL Compatibility](./pic/ddl_support.png) + +- 原生支持的 DDL 使用 Inplace/Instant 执行 +- 对于 DuckDB 原生不支持的 DDL（例如列重排），实现了 Copy DDL 机制 +- InnoDB 转 DuckDB 支持多线程并行转换，执行时间降低 **7x** +![Copy DDL from InnoDB](./pic/parallel_copy_from_innodb.png) + +## 性能基准测试（Performance Benchmarks） +**测试环境**： +- ECS 规格：32 CPU，128GB 内存，ESSD PL1 云盘 500GB +- Benchmark：TPC-H SF100 + +| Query ID | DuckDB | InnoDB | ClickHouse | | --- | --- | --- | --- | |q1|0.92|1134.25|3.47| |q2|0.15|1800|1.52| @@ -56,6 +116,21 @@ DuckDB 作为独立的存储引擎接入到 ALiSQL 内核中，ALiSQL 实现了 |q18|1.59|1800|3.11| |q19|0.8|1800|2.96| |q20|0.51|1800|3.38| -|q21|1.64|1800|内存不足| +|q21|1.64|1800|OOM| |q22|0.33|361.4|4| -|总计|15.31|25234.31|80.01 +|total|15.31|25234.31|80.01| + +DuckDB 在分析查询场景中展现出显著的性能优势，相比 InnoDB 最高可提升 **200x**。 + +## 在阿里云上体验 +你可以在阿里云上体验集成 DuckDB 引擎的 RDS MySQL： + +https://help.aliyun.com/zh/rds/apsaradb-rds-for-mysql/duckdb-based-analytical-instance/ + +## 相关链接（See also） + +- [DuckDB 参数参考](./duckdb_variables-zh.md) +- [如何搭建 DuckDB 节点](./how-to-setup-duckdb-node-zh.md) +- [DuckDB GitHub 仓库](https://github.com/duckdb/duckdb) +- [详细文章（中文）](https://mp.weixin.qq.com/s/_YmlV3vPc9CksumXvXWBEw) +- [AliSQL](https://github.com/alibaba/AliSQL.git) \ No newline at end of file diff --git a/wiki/duckdb/duckdb.md b/wiki/duckdb/duckdb.md index 6bb30e26c9e..a00847ab5c6 100644 --- a/wiki/duckdb/duckdb.md +++ b/wiki/duckdb/duckdb.md @@ -1,6 +1,8 @@ # DuckDB in AliSQL ![MySQL with DuckDB](./pic/mysql_with_duckdb.png) +[ [AliSQL DuckDB 引擎](./duckdb-zh.md) | [DuckDB in AliSQL](./duckdb-en.md) ] + ## What is DuckDB? [DuckDB](https://github.com/duckdb/duckdb) is an open-source embedded analytical database system (OLAP) designed for data analysis workloads. DuckDB is rapidly becoming a popular choice in data science, BI tools, and embedded analytics scenarios due to its key characteristics: @@ -94,8 +96,8 @@ AliSQL allows DuckDB nodes to serve as replicas via Binlog synchronization. By r **Test Environment**: - ECS Instance: 32 CPU, 128GB Memory, ESSD PL1 Cloud Disk 500GB - Benchmark: TPC-H SF100 -- -| Query ID | DuckDB | InnoDB | InnoDB | + +| Query ID | DuckDB | InnoDB | ClickHouse | | --- | --- | --- | --- | |q1|0.92|1134.25|3.47| |q2|0.15|1800|1.52| @@ -131,8 +133,8 @@ https://help.aliyun.com/zh/rds/apsaradb-rds-for-mysql/duckdb-based-analytical-in ## See also -- [DuckDB Variables Reference](./duckdb_variables-zh.md) -- [How to Setup DuckDB Node](./how-to-setup-duckdb-node-zh.md) +- [DuckDB Variables Reference](./duckdb_variables-en.md) +- [How to Setup DuckDB Node](./how-to-setup-duckdb-node-en.md) - [DuckDB GitHub Repository](https://github.com/duckdb/duckdb) - [Detailed Article (Chinese)](https://mp.weixin.qq.com/s/_YmlV3vPc9CksumXvXWBEw) - [AliSQL](https://github.com/alibaba/AliSQL.git) diff --git a/wiki/duckdb/duckdb_variables-en.md b/wiki/duckdb/duckdb_variables-en.md new file mode 100644 index 00000000000..53ab26d3c2e --- /dev/null +++ b/wiki/duckdb/duckdb_variables-en.md @@ -0,0 +1,375 @@ +# DuckDB Engine Variables in AliSQL +[ [AliSQL DuckDB 引擎参数](./duckdb_variables-zh.md) | [DuckDB Engine Variables in AliSQL](./duckdb_variables-en.md) ] + +## `duckdb_mode` +- **Scope**: Global +- **Change type**: Static (restart required) +- **Data type**: Enum +- **Default**: `NONE` +- **Valid values**: `NONE` \| `ON` +- **Description**: Controls whether the DuckDB storage engine is enabled. `ON` enables DuckDB; `NONE` disables it. This variable is read-only and can only be set at startup. + +--- + +## `duckdb_require_primary_key` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `ON` +- **Valid values**: `ON` \| `OFF` +- **Description**: Whether all DuckDB tables must define a primary key. If enabled, creating a table without a primary key will fail. + +> Notes: +> - DuckDB tables do not actually create indexes. Uniqueness for PRIMARY KEY / UNIQUE KEY must be guaranteed by the user. +> - When using a DuckDB node as a replica, you must enable this variable to ensure replication correctness. + +--- + +## `duckdb_memory_limit` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Integer (bytes) +- **Default**: `0` +- **Valid range**: `0` ~ `ULLONG_MAX` +- **Step**: 1024 bytes +- **Description**: Sets the maximum memory DuckDB is allowed to use. `0` means automatic (typically ~80% of physical memory). + +> Note: When DuckDB is enabled, it is recommended to reduce `innodb_buffer_pool_size` to free more memory for DuckDB. + +--- + +## `duckdb_temp_directory` +- **Scope**: Global +- **Change type**: Static +- **Data type**: String +- **Default**: (empty) +- **Description**: Directory path where DuckDB writes temporary files. This variable is read-only and can only be configured before startup. + +--- + +## `duckdb_max_temp_directory_size` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Integer (bytes) +- **Default**: `0` (uses 90% of available disk space) +- **Valid range**: `0` ~ `ULLONG_MAX` +- **Step**: 1024 bytes +- **Description**: Limits the maximum disk space DuckDB can use under `duckdb_temp_directory`. `0` means automatic (typically ~90% of free disk space). + +--- + +## `duckdb_threads` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Integer +- **Default**: `0` (auto) +- **Valid range**: `0` ~ `1048576` +- **Description**: Sets the total number of threads used by DuckDB. `0` lets the system choose based on CPU cores. + +--- + +## `duckdb_use_direct_io` +- **Scope**: Global +- **Change type**: Static +- **Data type**: Boolean +- **Default**: `OFF` +- **Valid values**: `ON` \| `OFF` +- **Description**: Whether to use Direct I/O to bypass the OS page cache for data reads/writes and improve large-file I/O performance. This variable is read-only and can only be set before startup. + +> Note: DuckDB Direct I/O is currently unstable and is not recommended. + +--- + +## `duckdb_scheduler_process_partial` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `ON` +- **Valid values**: `ON` \| `OFF` +- **Description**: Whether the scheduler partially processes tasks before rescheduling, which can improve fairness across concurrent queries. + +--- + +## `duckdb_merge_join_threshold` +- **Scope**: Session +- **Change type**: Dynamic +- **Data type**: Integer (rows) +- **Default**: `4611686018427387904` +- **Valid range**: `0` ~ `4611686018427387904` +- **Description**: If the row count of either table exceeds this threshold, DuckDB prefers Merge Join over Hash Join. + +--- + +## `duckdb_convert_all_at_startup` +- **Scope**: Global +- **Change type**: Static (read-only) +- **Data type**: Boolean +- **Default**: `OFF` +- **Valid values**: `ON` \| `OFF` +- **Description**: Whether to automatically convert all InnoDB tables to DuckDB tables during server startup. This variable is read-only and can only be configured before startup. + +--- + +## `duckdb_convert_all_at_startup_ignore_error` +- **Scope**: Global +- **Change type**: Static (read-only) +- **Data type**: Boolean +- **Default**: `OFF` +- **Valid values**: `ON` \| `OFF` +- **Description**: During startup conversion from InnoDB to DuckDB, whether to ignore conversion errors and continue. This variable is read-only and can only be configured before startup. + +--- + +## `duckdb_convert_all_at_startup_threads` +- **Scope**: Global +- **Change type**: Static +- **Data type**: Integer +- **Default**: `4` +- **Valid range**: `1` ~ `64` +- **Description**: Number of threads used to convert tables at startup to accelerate bulk migration. This variable is read-only and can only be configured before startup. + +--- + +## `duckdb_convert_all_skip_mtr_db` +- **Scope**: Global +- **Change type**: Static +- **Data type**: Boolean +- **Default**: `OFF` +- **Valid values**: `ON` \| `OFF` +- **Description**: Whether to skip a database named `mtr` during startup conversion. Typically used only for test purposes. This variable is read-only and can only be configured before startup. + +--- + +## `duckdb_force_no_collation` +- **Scope**: Session +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `OFF` +- **Valid values**: `ON` \| `OFF` +- **Description**: Disables collation pushdown optimization and forces binary comparison. If your queries do not care about collation/case order, setting this to `ON` may improve performance. + +--- + +## `duckdb_source_set_insert_only_to_binlog` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `OFF` +- **Valid values**: `ON` \| `OFF` +- **Description**: When a transaction contains only INSERT operations, whether to set an `insert_only` flag in the binlog to optimize replication performance. + +--- + +## `duckdb_explain_output` +- **Scope**: Session +- **Change type**: Dynamic +- **Data type**: Enum +- **Default**: `PHYSICAL_ONLY` +- **Valid values**: `ALL` \| `OPTIMIZED_ONLY` \| `PHYSICAL_ONLY` +- **Description**: Controls the default output format of DuckDB `EXPLAIN`: all plans, optimized plan only, or physical plan only. + +--- + +## `duckdb_multi_trx_in_batch` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `OFF` +- **Valid values**: `ON` \| `OFF` +- **Description**: Whether to merge multiple transactions from the relay log into a single batch commit to improve throughput. Effective only on replicas. + +--- + +## `duckdb_multi_trx_timeout` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Integer (ms) +- **Default**: `5000` ms +- **Valid range**: `0` ~ `100000` +- **Description**: Commit delay timeout (milliseconds) used to wait for more transactions to join the same batch. Effective only on replicas. + +--- + +## `duckdb_multi_trx_max_batch_length` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Integer (bytes) +- **Default**: `256MB` +- **Valid range**: `0` ~ `ULLONG_MAX` +- **Description**: Maximum batch size in bytes. Once reached, the batch is committed immediately. Effective only on replicas. + +--- + +## `duckdb_commit_multi_trx_due_to_reader` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `ON` +- **Valid values**: `ON` \| `OFF` +- **Description**: When the relay log is empty, whether to trigger a multi-transaction batch commit. Effective only on replicas. + +--- + +## `duckdb_commit_multi_trx_due_to_rotate` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `ON` +- **Valid values**: `ON` \| `OFF` +- **Description**: **Deprecated.** Whether to commit multiple transactions when a Rotate Event is received from the primary. Effective only on replicas. + +--- + +## `duckdb_commit_multi_trx_due_to_rotate_frequency` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Integer +- **Default**: `1` +- **Valid range**: `0` ~ `1048576` +- **Description**: When `duckdb_commit_multi_trx_due_to_rotate` is enabled, commit once per N binlog rotate events. `0` means never; `1` means every time. Effective only on replicas. + +--- + +## `duckdb_copy_ddl_threads` +- **Scope**: Session +- **Change type**: Dynamic +- **Data type**: Integer +- **Default**: `4` +- **Valid range**: `0` ~ `64` +- **Description**: Number of threads used during DDL conversion from InnoDB to DuckDB. The parallel conversion uses InnoDB parallel read infrastructure, but this thread count is not controlled by `innodb_parallel_read_threads`. + +--- + +## `duckdb_checkpoint_threshold` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Integer (bytes) +- **Default**: `268435456` (256MB) +- **Valid range**: `0` ~ `ULLONG_MAX` +- **Step**: 1024 bytes +- **Description**: Automatically triggers a checkpoint when DuckDB WAL reaches this size. + +--- + +## `duckdb_use_double_for_decimal` +- **Scope**: Global +- **Change type**: Static +- **Data type**: Boolean +- **Default**: `ON` +- **Valid values**: `ON` \| `OFF` +- **Description**: DuckDB does not support DECIMAL precision > 38. This variable controls whether to use DOUBLE instead for DECIMAL with precision > 38. This variable is read-only and can only be set before startup. + +> Note: This affects the actual column type and should not be changed after the instance is created. + +--- + +## `duckdb_disabled_optimizers` +- **Scope**: Session +- **Change type**: Dynamic +- **Data type**: Enum set +- **Default**: `0` (empty set) +- **Valid values**: + `EXPRESSION_REWRITER`, `FILTER_PULLUP`, `FILTER_PUSHDOWN`, `EMPTY_RESULT_PULLUP`, + `CTE_FILTER_PUSHER`, `REGEX_RANGE`, `IN_CLAUSE`, `JOIN_ORDER`, `DELIMINATOR`, + `UNNEST_REWRITER`, `UNUSED_COLUMNS`, `STATISTICS_PROPAGATION`, `COMMON_SUBEXPRESSIONS`, + `COMMON_AGGREGATE`, `COLUMN_LIFETIME`, `BUILD_SIDE_PROBE_SIDE`, `LIMIT_PUSHDOWN`, + `TOP_N`, `COMPRESSED_MATERIALIZATION`, `DUPLICATE_GROUPS`, `REORDER_FILTER`, + `SAMPLING_PUSHDOWN`, `JOIN_FILTER_PUSHDOWN`, `EXTENSION`, `MATERIALIZED_CTE`, + `SUM_REWRITER`, `LATE_MATERIALIZATION` +- **Description**: Disables the specified optimizer rules in DuckDB. + +--- + +## `duckdb_data_import_mode` +- **Scope**: Session +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `OFF` +- **Valid values**: `ON` \| `OFF` +- **Description**: Enables data import mode. In this mode, only DELETE and INSERT operations with constant equality predicates on the primary key are supported. + +> Notes: +> 1. Intended for bulk import: merges multiple INSERT/DELETE operations into a single batch to improve performance. +> 2. This variable cannot be changed inside a transaction. +> 3. When `ON`, the modified table must have a primary key. +> 4. When `ON`, UPDATE is not supported; rewrite UPDATE as DELETE + INSERT. +> 5. When `ON`, unsupported DML will raise an error. +> 6. This variable takes effect only when `duckdb_dml_in_batch` is enabled. + +--- + +## `duckdb_idempotent_data_import_enabled` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `OFF` +- **Valid values**: `ON` \| `OFF` +- **Description**: When `duckdb_data_import_mode=ON`, enables idempotent data import. If enabled, re-importing the same data (e.g., after restart/recovery) will not create duplicates. + +> Note: Enabling idempotent import may reduce import performance. + +--- + +## `duckdb_appender_allocator_flush_threshold` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Integer (bytes) +- **Default**: `64MB` +- **Valid range**: `0` ~ `ULLONG_MAX` +- **Step**: 1024 bytes +- **Description**: When DuckDB writes data in batches, if batch memory usage reaches this threshold, DuckDB proactively flushes to release memory and avoid OOM. + +--- + +## `duckdb_log_options` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Enum set +- **Default**: `0` (no logging) +- **Valid values**: + `DUCKDB_MULTI_TRX_BATCH_COMMIT`, `DUCKDB_MULTI_TRX_BATCH_DETAIL`, `DUCKDB_QUERY`, `DUCKDB_QUERY_RESULT` +- **Description**: Selects which DuckDB operations are logged for debugging and auditing. + +--- + +## `force_innodb_to_duckdb` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `OFF` +- **Valid values**: `ON` \| `OFF` +- **Description**: When creating tables or running DDL, whether to force-replace the InnoDB engine with DuckDB. Useful for testing or migration scenarios. + +--- + +## `duckdb_copy_ddl_in_batch` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `ON` +- **Valid values**: `ON` \| `OFF` +- **Description**: Whether to use batch inserts to accelerate DDL conversion from InnoDB to DuckDB. Enabling this can significantly improve conversion performance. + +--- + +## `duckdb_dml_in_batch` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `ON` +- **Valid values**: `ON` \| `OFF` +- **Description**: Enables batch mode to accelerate DML (INSERT/UPDATE/DELETE). When enabled, multiple changes can be merged into batches to improve throughput and reduce transaction overhead. + +> Notes: +> 1. When enabled on a DuckDB replica and the primary uses row-based binlog, DuckDB automatically batches DML during replay. +> 2. When enabled on a DuckDB primary, INSERT can be batched; whether DELETE can be batched depends on `duckdb_data_import_mode` and its constraints; UPDATE cannot be batched. + +--- + +## `update_modified_column_only` +- **Scope**: Global +- **Change type**: Dynamic +- **Data type**: Boolean +- **Default**: `ON` +- **Valid values**: `ON` \| `OFF` +- **Description**: During binlog replay, whether to update only the columns that actually changed. Enabling this reduces unnecessary writes and improves replication efficiency while lowering I/O and memory pressure. \ No newline at end of file diff --git a/wiki/duckdb/duckdb_variables-zh.md b/wiki/duckdb/duckdb_variables-zh.md index 18541462a5b..0914f07ae98 100644 --- a/wiki/duckdb/duckdb_variables-zh.md +++ b/wiki/duckdb/duckdb_variables-zh.md @@ -1,4 +1,7 @@ # AliSQL 中 DuckDB 引擎相关参数 + +[ [DuckDB Engine Variables in AliSQL](./duckdb_variables-en.md) | [AliSQL DuckDB 引擎参数](./duckdb_variables-zh.md) ] + ### `duckdb_mode` - **参数范围**: 全局参数 - **修改形式**: 静态修改（需重启生效） diff --git a/wiki/duckdb/how-to-setup-duckdb-node-en.md b/wiki/duckdb/how-to-setup-duckdb-node-en.md new file mode 100644 index 00000000000..a09a34a2bd5 --- /dev/null +++ b/wiki/duckdb/how-to-setup-duckdb-node-en.md @@ -0,0 +1,250 @@ +# How to setup DuckDB node in AliSQL + +[ [快速部署 AliSQL DuckDB 节点](./how-to-setup-duckdb-node-zh.md) | [How to setup DuckDB node in AliSQL](./how-to-setup-duckdb-node-en.md) ] + +## Overview + +AliSQL integrates DuckDB as an analytical storage engine. + +- **User data** is stored in **DuckDB** +- **System tables and metadata** remain in **InnoDB** (e.g., `mysql.*`, data dictionary) + +You can: +1) bootstrap a **brand-new instance** where new tables default to DuckDB +2) **convert an existing InnoDB instance** to DuckDB at startup +3) build a **DuckDB replica (HTAP)** to replay binlogs from an OLTP primary + +--- + +## 1. Bootstrap a Brand-New DuckDB Instance + +Use this when you are creating a fresh instance and want newly created tables to land in DuckDB by default. + +### 1.1 Install AliSQL 8.0.44 + +#### Option A: Build from source +```bash +git clone https://github.com/alibaba/AliSQL.git +cd AliSQL + +sh build.sh -t release -d /opt/alisql +make install +``` + +#### Option B: Install from RPM +```bash +# For example, use x86 el6 rpm for installation +wget https://github.com/alibaba/AliSQL/releases/download/AliSQL-8.0.44-1/alisql-8.0.44-1.el8.x86_64.rpm +rpm -ivh alisql-8.0.44-1.el8.x86_64.rpm +``` + +--- + +### 1.2 Directory bootstrap + config generator script + +```bash +cat > init_alisql_dir.sh <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +# Usage: +# ./init_alisql_dir.sh + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +DIR="$1" +if [[ "$DIR" != /* ]]; then + echo "Error: Argument must be an ABSOLUTE PATH (starting with /)." + exit 1 +fi + +DIR="${DIR%/}" + +mkdir -p \ + "$DIR/data/dbs" \ + "$DIR/data/mysql" \ + "$DIR/run" \ + "$DIR/log/mysql" \ + "$DIR/tmp" + +cat > "$DIR/alisql.cnf" < 本文提供的参数均为示例参数，请根据实际情况进行修改。参数的含义和默认值，请参考 [DuckDB in AliSQL 参数介绍](./duckdb_variables-zh.md)。 +你可以： +1) 从零初始化一个 **DuckDB 为默认引擎** 的新实例 +2) 将现有 **InnoDB 实例一键转换** 为 DuckDB +3) 构建 **DuckDB 从节点（HTAP/读写分离）**，从主库复制数据用于分析 --- -## 1. 构建一个独立的 DuckDB 节点 - -独立的 DuckDB 节点是您的主要分析数据存储。所有用户数据都将存入 DuckDB 引擎，InnoDB 仅用于保存系统表和元数据。您可以从一个全新的实例开始，也可以将现有的 InnoDB 实例快速转换为 DuckDB。 - -### 场景一: 从零开始构建一个全新的 DuckDB 实例 - -此场景适用于初始化一个全新的数据库实例，后续所有新建的表都将默认使用 DuckDB 引擎。 - -#### 步骤: - -1. **配置 `my.cnf`** - ```ini - [mysqld] - # 1. 启用 DuckDB 引擎。 - # 必选项，用以启用 DuckDB 存储引擎。 - duckdb_mode=ON - - # 2. 强制将 InnoDB 引擎重定向到 DuckDB。 - # 推荐项，开启后用户创建 InnoDB 表或对 InnoDB 表执行 DDL 操作时会自动触发 - # 到 DuckDB引擎的转换，这可以简化操作，确保了所有用户数据都落入 DuckDB。 - force_innodb_to_duckdb=ON - - # 3. 配置资源 - # 根据您的服务器硬件，合理分配内存、线程和临时目录。0 表示自动配置。 - duckdb_memory_limit=0 - duckdb_threads=0 - duckdb_temp_directory=/path/to/duckdb_temp_dir - ``` - -2. **启动实例** - 使用上述的配置初始化并启动 AliSQL 实例。 - ```shell - # 初始化数据目录 - mysqld --defaults-file=/etc/my.cnf --initialize-insecure - - # 启动实例 - mysqld_safe --defaults-file=/etc/my.cnf & - ``` - -3. **验证** - 实例启动后，您可以正常创建表和数据库。虽然您可能没有显式指定引擎，但所有新表都将由 DuckDB 负责存储和管理。您可以开始导入数据并执行分析查询。 - ```sql - -- 示例 - -- 1. 创建 DuckDB 表 - CREATE TABLE t (id INT PRIMARY KEY, name VARCHAR(255)) ENGINE = DuckDB; - - -- 2. 将 DuckDB 表转换成 InnoDB 表 - ALTER TABLE t ENGINE = InnoDB; - - -- 3. 将 InnoDB 表转换成 DuckDB 表 - ALTER TABLE t ENGINE = DuckDB; - - -- 4. DML - INSERT INTO t VALUES (1, 'John'); - SELECT * FROM t; - UPDATE t SET name = 'Jane' WHERE id = 1; - DELETE FROM t WHERE id = 1; - - -- 5.DDL - ALTER TABLE t ADD COLUMN age INT; - ALTER TABLE t RENAME TO t_new; - ``` - -### 场景二: 从现有 InnoDB 实例一键转换 - -此场景适用于将已有的 InnoDB 实例整体迁移到 DuckDB 以获得分析性能的提升。 - -#### 步骤: - -1. **停止现有实例** - 确保您的MySQL 数据库实例已安全关闭。 - -2. **配置 `my.cnf`** - 在配置文件中添加以下参数，以触发启动时自动转换。 - ```ini - [mysqld] - # 1. 启用 DuckDB 引擎。 - # 必选项，用以启用 DuckDB 存储引擎。 - duckdb_mode=ON - - # 2. 强制将 InnoDB 引擎重定向到 DuckDB。 - # 推荐项，开启后用户创建 InnoDB 表或对 InnoDB 表执行 DDL 操作时会自动触发 - # 到 DuckDB引擎的转换，这可以简化操作，确保了所有用户数据都落入 DuckDB。 - force_innodb_to_duckdb=ON - - # 3. 配置资源 - # 根据您的服务器硬件，合理分配内存、线程和临时目录。0 表示自动配置。 - duckdb_memory_limit=0 - duckdb_threads=0 - duckdb_temp_directory=/path/to/duckdb_temp_dir - - # 4. 开启启动时自动转换功能。 - # 推荐项，开启后，AliSQL 将在启动过程中，自动将所有 InnoDB 表转换成 DuckDB 表。 - # 若参数未开启，则需要在启动之后执行 ALTER TABLE ... ENGINE = DuckDB，手动将 - # InnoDB 表转换成 DuckDB 表。 - duckdb_convert_all_at_startup=ON - - # 5. 配置启动自动转换过程并行转表的数量。 - # 可选项，增加线程数可以显著加快大批量表的转换速度，可根据服务器硬件来配置。 - duckdb_convert_all_at_startup_threads=32 - - # 6. 配置启动自动转换过程忽略错误 - # 可选项，如果您担心某些表可能因兼容性问题转换失败而导致启动中断，可以开启此选项。 - duckdb_convert_all_at_startup_ignore_error=ON - ``` - -3. **启动实例** - 使用 AliSQL 启动数据库实例。 - -4. **验证** - 当实例成功启动并可以接受连接时，您可以查询实例转换引擎的状态。当所有目标数据均已迁移至 DuckDB 时，即可体验 DuckDB 的高性能分析查询。 - ```sql - -- 查询数据库实例转换引擎的状态。 - -- "EMPTY" - 未开始 - -- "INIT" - 初始化 - -- "CHECKING" - 检查元数据 - -- "CHECK_FAILED" - 检查元数据失败 - -- "CONVERTING" - 正在转换 - -- "CONVERT_FAILED" - 转换失败 - -- "FINISHED" - 转换完成 - SHOW GLOABL STATUS LIKE 'DuckDB_convert_stage_at_startup'; - - -- 查看数据库中的 DuckDB 表 - SELECT * FROM information_schema.tables WHERE ENGINE = 'DuckDB'; - ``` - -> **注意事项** -> 1. 当开启自动转换功能时，由于实例在后台执行全量数据的读取、转换和写入，启动过程会耗费硬件资源。您可以通过观察错误日志（error log）来监控转换进度和可能出现的错误。 -> 2. 数据转换过程会占用额外的磁盘空间，请确保磁盘空间 ≥ 原始数据大小。 -> 3. 转换前请对原 InnoDB 数据库进行备份，建议在备份集数据上进行 DuckDB 引擎的转换。 -> 4. 5.7及更低版本的 mysql 请先升级到 8.0。 -> 5. 非 8.0.44 的 MySQL 在启动后将自动升级至 AliSQL 8.0.44，建议在使用 AliSQL 8.0.44启动前确保数据库完成 clean shutdown。 +## 1. 从零开始构建 DuckDB 实例（推荐新建实例） + +适用于：初始化一个全新实例，后续新建表默认落入 DuckDB. + +### 1.1 安装 AliSQL 8.0.44 + +#### 方式 A：从源码编译安装 +```bash +git clone https://github.com/alibaba/AliSQL.git +cd AliSQL + +# Build (release) +sh build.sh -t release -d /opt/alisql + +# Install +make install +``` + +#### 方式 B：使用 RPM 安装包 +```bash +# 示例安装 x86 架构下 centos 8 的 AliSQL 8.0.44 +wget https://github.com/alibaba/AliSQL/releases/download/AliSQL-8.0.44-1/alisql-8.0.44-1.el8.x86_64.rpm +rpm -ivh alisql-8.0.44-1.el8.x86_64.rpm +``` + +--- + +### 1.2 初始化目录与生成配置文件脚本 + +> 说明：脚本要求传入**绝对路径**，并生成目录结构与 `alisql.cnf`。 + +```bash +cat > init_alisql_dir.sh <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +# Usage: +# ./init_alisql_dir.sh +# Example: +# ./init_alisql_dir.sh /root/alisql_8044 + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +DIR="$1" +if [[ "$DIR" != /* ]]; then + echo "Error: Argument must be an ABSOLUTE PATH (starting with /)." + echo "Current input: $DIR" + exit 1 +fi + +DIR="${DIR%/}" +echo "Initializing AliSQL directories at: $DIR" + +mkdir -p \ + "$DIR/data/dbs" \ + "$DIR/data/mysql" \ + "$DIR/run" \ + "$DIR/log/mysql" \ + "$DIR/tmp" + +cat > "$DIR/alisql.cnf" < 更多配置参数可参考： +> - MySQL 8.0 官方文档：https://dev.mysql.com/doc/refman/8.0/en/ +> - DuckDB 参数参考： [AliSQL DuckDB 参数](./duckdb_variables-zh.md) + +--- + +### 1.3 初始化并启动实例 + +```bash +# Example: store all data under $HOME/alisql_8044 +./init_alisql_dir.sh $HOME/alisql_8044 + +# Initialize +/opt/alisql/bin/mysqld --defaults-file=$HOME/alisql_8044/alisql.cnf --initialize-insecure + +# Start, the port is 3306 by default +/opt/alisql/bin/mysqld_safe --defaults-file=$HOME/alisql_8044/alisql.cnf & +``` + +--- + +### 1.4 验证与使用 + +> 即使不显式写 `ENGINE=DuckDB`，当 `force_innodb_to_duckdb=ON` 时，用户侧创建/变更 InnoDB 表将被自动转换为 DuckDB 引擎。 + +```sql +CREATE DATABASE test; +USE test; + +-- 创建 DuckDB 表 +CREATE TABLE t ( + id INT PRIMARY KEY, + name VARCHAR(255) +) ENGINE = DuckDB; + +-- 引擎转换 +ALTER TABLE t ENGINE = InnoDB; +ALTER TABLE t ENGINE = DuckDB; + +-- DML +INSERT INTO t VALUES (1, 'John'); +SELECT * FROM t; +UPDATE t SET name = 'Jane' WHERE id = 1; +DELETE FROM t WHERE id = 1; + +-- DDL +ALTER TABLE t ADD COLUMN age INT; +ALTER TABLE t RENAME TO t_new; +``` + +--- + +## 2. 从现有 InnoDB 实例一键转换为 DuckDB + +适用于：希望将历史数据整体迁移到 DuckDB 以提升分析性能。 + +### 步骤 + +1) **停止现有实例** +确保 MySQL 实例已 clean shutdown。 + +1) **在 `my.cnf` 增加参数** +```ini +[mysqld] +duckdb_mode=ON +force_innodb_to_duckdb=ON + +duckdb_memory_limit=2147483648 +duckdb_threads=0 +duckdb_temp_directory=/path/to/duckdb_temp_dir + +# Convert all InnoDB tables at startup +duckdb_convert_all_at_startup=ON +duckdb_convert_all_at_startup_threads=32 +duckdb_convert_all_at_startup_ignore_error=ON +``` + +3) **启动实例** + +4) **验证转换状态与结果** +```sql +-- 检查转换阶段 +SHOW GLOBAL STATUS LIKE 'DuckDB_convert_stage_at_startup'; + +-- 查询 DuckDB 表 +SELECT table_schema, table_name, engine +FROM information_schema.tables +WHERE engine = 'DuckDB'; +``` + +### 注意事项 + +1. 启动自动转换会大量读写数据并占用 CPU/IO；建议通过 error log 观察进度与报错。 +2. 转换过程中可能需要额外磁盘空间（建议预留 ≥ 原始数据大小）。 +3. 转换前务必备份（推荐使用备份集拉起 DuckDB 节点）。 +4. MySQL 5.7 及以下请先升级到 8.0。 +5. 非 8.0.44 的实例启动后可能触发升级流程，建议先确保 clean shutdown。 + --- -## 2. 构建一个 DuckDB 从节点（HTAP） - -将 DuckDB 节点作为主库（Source）的从库（Replica）是一个经典的读写分离和 AP/TP 混合负载场景。主库处理在线事务（OLTP），并将数据通过 Binlog 同步到 DuckDB 从节点，由从节点专门负责复杂的分析查询（OLAP）。 - -得益于专门的 DML 优化，DuckDB 从节点的回放性能极高，甚至可以超越 InnoDB，轻松实现无延迟复制。 - -#### 步骤: - -1. **准备主库** - 确保您的主库已开启 Binlog（格式为 `ROW`）和 GTID。这是所有主从复制的基础。 - -2. **配置从节点的 `my.cnf`** - 在从节点的配置文件中，除了启用 DuckDB，还需要开启一系列为复制和写入性能优化的参数。 - ```ini - [mysqld] - # 1. 启用 DuckDB 引擎。 - # 必选项，用以启用 DuckDB 存储引擎。 - duckdb_mode=ON - - # 2. 开启强制主键要求 - # 作为从库，为确保数据能正确按行应用，必须要求表有主键。 - duckdb_require_primary_key=ON - - # 3. 强制将 InnoDB 引擎重定向到 DuckDB。 - # 推荐项，开启后用户创建 InnoDB 表或对 InnoDB 表执行 DDL 操作时会自动触发 - # 到 DuckDB引擎的转换，这可以简化操作，确保了所有用户数据都落入 DuckDB。 - force_innodb_to_duckdb=ON - - # 4. 开启 DML 攒批处理 - # 推荐项，将多个 DML 操作合并提交，提升写入性能。 - dml_in_batch=ON - update_modified_column_only=ON - - # 5. 开启跨事务批量提交 - # 推荐项，将多个事务聚合成一个大批次提交，极大提升吞吐量。 - duckdb_multi_trx_in_batch=ON - - # 6. 调整批量提交的策略。 - # 可选项，用于控制攒批的大小。 - duckdb_multi_trx_timeout=5000 - duckdb_multi_trx_max_batch_length=268435456 - ``` - -3. **初始化 DuckDB 节点** - 在从节点上启动 AliSQL 实例。此操作可参考：从现有 InnoDB 实例一键转换 - -4. **建立主从关系** - 在从节点上，执行以下 SQL 命令： - ```sql - -- 指向主库的正确位置和 Binlog 信息 - CHANGE MASTER TO - MASTER_HOST='your-master-ip', - MASTER_USER='repl', - MASTER_PASSWORD='your-password', - MASTER_AUTO_POSITION=1; - - -- 启动复制 - START SLAVE; - ``` - -5. **完成** - 复制链路建立后，主库的变更会以极高的性能实时同步到 DuckDB 从节点。您现在可以将所有分析和报表类的复杂查询都路由到这个从节点上，而不会影响主库的事务处理性能。 +## 3. 构建 DuckDB 从节点（HTAP / 读写分离） + +适用于：主库（InnoDB/OLTP）负责事务，从库（DuckDB/AP）负责复杂分析；通过 binlog 复制同步数据。 + +### 步骤 + +1) **准备主库** +- 开启 binlog +- `binlog_format=ROW` +- 开启 GTID + +1) **配置从节点 `my.cnf`** +```ini +[mysqld] +duckdb_mode=ON +duckdb_require_primary_key=ON +force_innodb_to_duckdb=ON + +duckdb_dml_in_batch=ON +duckdb_update_modified_column_only=ON +duckdb_multi_trx_in_batch=ON +duckdb_multi_trx_timeout=5000 +duckdb_multi_trx_max_batch_length=268435456 +``` + +3) **初始化从节点实例** +参考第 2 节（可选启用启动转换）或按第 1 节新建实例。 + +4) **建立主从复制** +```sql +CHANGE MASTER TO + MASTER_HOST='your-master-ip', + MASTER_USER='repl', + MASTER_PASSWORD='your-password', + MASTER_AUTO_POSITION=1; + +START SLAVE; +``` + +5) **完成** +复制建立后，将分析/报表查询路由到 DuckDB 从节点。 diff --git a/wiki/vidx/pic/hnsw.png b/wiki/vidx/pic/hnsw.png new file mode 100644 index 00000000000..85e2315365c Binary files /dev/null and b/wiki/vidx/pic/hnsw.png differ diff --git a/wiki/vidx/pic/vidx_architecture.png b/wiki/vidx/pic/vidx_architecture.png new file mode 100644 index 00000000000..77b4be02960 Binary files /dev/null and b/wiki/vidx/pic/vidx_architecture.png differ diff --git a/wiki/vidx/pic/vidx_core_features.png b/wiki/vidx/pic/vidx_core_features.png new file mode 100644 index 00000000000..6c54c3a6a8f Binary files /dev/null and b/wiki/vidx/pic/vidx_core_features.png differ diff --git a/wiki/vidx/pic/vidx_core_features_zh.png b/wiki/vidx/pic/vidx_core_features_zh.png new file mode 100644 index 00000000000..efdb7b9a0c4 Binary files /dev/null and b/wiki/vidx/pic/vidx_core_features_zh.png differ diff --git a/wiki/vidx/vidx_readme.md b/wiki/vidx/vidx_readme.md new file mode 100644 index 00000000000..e04438103d3 --- /dev/null +++ b/wiki/vidx/vidx_readme.md @@ -0,0 +1,153 @@ +# AliSQL Vector Index (vidx) Feature Introduction + +[ [AliSQL](../../README.md) | [Vector Index](./vidx_readme.md) | [向量索引](./vidx_readme_zh.md) ] + +## Overview + +AliSQL natively supports storage and computation of up to 16,383 dimensional vector data, integrates mainstream vector operation functions such as cosine similarity (COSINE) and Euclidean distance (EUCLIDEAN), and builds efficient nearest neighbor search capabilities based on deeply optimized HNSW (Hierarchical Navigable Small World) algorithm, supporting indexing of full-dimensional vector columns. + +AliSQL's vector capabilities can provide out-of-the-box vectorized solutions for large-scale semantic retrieval, intelligent recommendation, multimodal analysis and other scenarios. Users can seamlessly achieve high-precision vector matching and complex business logic fusion computing through standard SQL interfaces. + +### Core Features + +
+vidx Core Features +
+ +- **High-Dimensional Support**: Supports up to 16,383 dimensional floating-point vector data storage +- **High Performance Retrieval**: Uses HNSW (Hierarchical Navigable Small World) graph algorithm to achieve high-performance similarity search +- **Multiple Distance Metrics**: Supports distance calculation methods such as EUCLIDEAN and COSINE +- **SIMD Hardware Acceleration**: Based on SIMD instruction set optimization to improve vector operation efficiency +- **Search Pruning Option**: Optimizes search process through techniques like Bloom filters +- **Configurable Parameters**: Supports adjusting index parameters to balance retrieval accuracy and performance +- **Hybrid Query**: Supports joint queries of vector data and scalar data + +## Usage + +### Vector Field Definition + +Vector fields use a special [Field_vector](../../include/vidx/vidx_func.h#L43-L65) type definition, inheriting from [Field_varstring](../../include/field.h#L793-L798), using binary character set to store floating-point arrays. + +```sql +CREATE TABLE table_name ( + id INT PRIMARY KEY, + vector_col VECTOR(128) -- 128-dimensional vector +); +``` + +### Creating Vector Index + +Vector indexes can be created using the following syntax: + +```sql +CREATE VECTOR INDEX vidx_name ON table_name (vector_col); -- Using default parameters +``` + +Or specify directly in table definition: + +```sql +CREATE TABLE table_name ( + id INT PRIMARY KEY, + vector_col VECTOR(128), + VECTOR INDEX vidx_name (vector_col) M=6 DISTANCE=COSINE -- Specifying parameters +); +``` + +### Function Support + +#### Vector Conversion Functions + +| Function Name | Meaning | +|---------------|---------| +| VEC_FROMTEXT, TO_VECTOR, STRING_TO_VECTOR | String to vector | +| VEC_TOTEXT, FROM_VECTOR, VECTOR_TO_STRING | Vector to string | + +#### Vector Calculation Functions + +| Function Name | Meaning | +|---------------|---------| +| VECTOR_DIM | Vector dimension | +| VEC_DISTANCE, VEC_DISTANCE_EUCLIDEAN, VEC_DISTANCE_COSINE | Calculate distance between two vectors
If one of the arguments is a column in the vector index, distance type does not need to be specified, the vector index distance type will be automatically recognized | + +Usage examples: + +```sql +-- Sort using vector distance +SELECT * FROM table_name ORDER BY VEC_DISTANCE(vector_col, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 10; + +-- Display distance value in results +SELECT id, VEC_DISTANCE(vector_col, VEC_FROMTEXT("[1,2,3,4,5]")) AS distance +FROM table_name ORDER BY distance LIMIT 10; +``` + +### Parameter Introduction + +#### System Variables + +| Variable Name | Description | Type | Default Value | Range | +|---------------|-------------|------|---------------|-------| +| vidx_disabled | Disable creation of vector columns and vector indexes | global | ON | ON, OFF | +| vidx_default_distance | Default vector distance type | session | EUCLIDEAN | EUCLIDEAN, COSINE | +| vidx_hnsw_default_m | HNSW algorithm default m | session | 6 | [3, 200] | +| vidx_hnsw_ef_search | HNSW algorithm default ef_search | session | 20 | [1, 10000] | +| vidx_hnsw_cache_size | HNSW algorithm default memory usage limit | global | 1024 * 1024 | [1048576,18446744073709551615] | + +#### Index Parameters + +- `M`: Controls the number of connections for each node in the graph, default value is 6, valid range is 3 to 200 +- `DISTANCE`: Distance type for building index, default value is EUCLIDEAN + +### Notes + +1. Current version only supports RC (READ COMMITTED) transaction isolation level +2. Only supports creating vector indexes on InnoDB engine tables. +3. Creating, modifying, and deleting vector indexes cannot use inplace syntax. +4. Vector indexes cannot be set to INVISIBLE. +5. Vector fields cannot be NULL. +6. Creating and maintaining vector indexes consumes additional storage space and computational resources + +### Error Handling + +- `ER_NOT_SUPPORTED_YET`: Unsupported transaction isolation level +- `ER_WRONG_ARGUMENTS`: Function argument error +- `ER_VECTOR_INDEX_USAGE`: Vector index usage error +- `ER_VECTOR_INDEX_FAILED`: Vector index operation failure + +## Technical Details + +### Overall Architecture of Vector Search + +As one of the most popular ANN algorithms, HNSW has gained widespread recognition and validation in institutional evaluations and engineering implementations. Currently, AliSQL prioritizes support for vector indexes based on the HNSW algorithm. The overall architecture of vector search is shown in the following diagram. + +
+vidx Overall Architecture +
+ +- ANN queries choose suitable indexes for searching after cost estimation, or can specify vector indexes for searching using hints such as FORCE INDEX. +- Logically, there is a complete HNSW graph. The information of this graph is organized as an auxiliary table and persistently stored on disk. Each row of data in this table represents a node in the HNSW graph. On this graph, the HNSW algorithm can be executed to achieve vector insertion and retrieval. +- Unlike ordinary indexes that directly access the storage engine, a vector index plugin is introduced. A node cache of the HNSW graph is maintained in memory to improve query efficiency. + +### HNSW Algorithm + +HNSW (Hierarchical Navigable Small World) is an efficient approximate nearest neighbor (ANN) search algorithm based on multi-layer graph structure. Its design can be summarized as: +- [Hierarchical Skip List] The 0th layer of the graph contains information of all points. From low to high, higher layers are abridged versions of lower layers containing only some points from lower layers. The top layer is used for quick jumps, and the bottom layer is used for precise searches. +- [Connecting Neighbors] Each layer is a proximity graph connected by vector distances, where each point records its closest few points as neighbors. + +
+hnsw +
+ +### Data Structure + +AliSQL introduces a public cache (MHNSW Share) and transaction cache (MHNSW Trx) for vector data to accelerate vector query performance and ensure transaction safety for vector updates, achieving a balance between resource isolation and performance optimization. Public and transaction caches are accessed by different operations and have different design goals: +- [Public Cache] MHNSW Share is accessed by read-only transactions and mounted on the auxiliary table's TABLE_SHARE. Its core goal is to reduce the overhead of repeatedly loading vector nodes through shared caching, thereby improving query efficiency. +- [Transaction Cache] MHNSW Trx inherits from MHNSW Share and is used by read-write transactions. Each read-write transaction creates an independent MHNSW Trx instance, caching the nodes it accesses including those it modifies, avoiding contamination of the public cache, and only updating the public cache upon commit. + +### Vector Computation Optimization + +- [Precomputation Strategy] During the node cache loading phase, the system precomputes vector distances and caches the results, thus avoiding repeated calculations for frequently accessed nodes. +- [SIMD Instruction Set Acceleration] At the computation optimization level, modern CPU SIMD instruction sets (such as AVX512) are utilized to accelerate vector distance calculations. Through Bloom filters, the system can batch-process multiple vectors, converting scalar operations that originally required multiple executions into parallelized vector operations. This optimization significantly reduces CPU instruction cycle consumption. + +## RDS MySQL Out-of-the-Box Vector Capabilities + +Welcome to visit [Alibaba Cloud RDS MySQL Vector Capabilities](https://help.aliyun.com/zh/rds/apsaradb-rds-for-mysql/vector-storage-1), open-source ecosystem, ready to use out-of-the-box. \ No newline at end of file diff --git a/wiki/vidx/vidx_readme_zh.md b/wiki/vidx/vidx_readme_zh.md new file mode 100644 index 00000000000..d06445d44ee --- /dev/null +++ b/wiki/vidx/vidx_readme_zh.md @@ -0,0 +1,154 @@ +# AliSQL Vector Index (vidx) 功能介绍 + +[ [AliSQL](../../README.md) | [Vector Index](./vidx_readme.md) | [向量索引](./vidx_readme_zh.md) ] + +## 概述 + +AliSQL 原生支持最高 16,383 维向量数据的存储及计算，集成余弦相似度（COSINE）、欧式距离（EUCLIDEAN）等主流向量运算函数，并基于深度优化的 HNSW（Hierarchical Navigable Small World）算法构建高效近邻搜索能力，支持对全维度向量列建立索引。 + +AliSQL 向量能力可为大规模语义检索、智能推荐、多模态分析等场景提供开箱即用的向量化解决方案，用户可通过标准SQL接口无缝实现高精度向量匹配与复杂业务逻辑的融合计算。 + +### 核心特性 + +
+vidx 核心特性 +
+ +- **高维向量支持**：支持最高 16,383 维度的浮点型向量数据存储 +- **高效向量检索**：采用 HNSW (Hierarchical Navigable Small World) 图算法实现高性能的相似性搜索 +- **多距离度量支持**：支持 EUCLIDEAN 和 COSINE 等距离计算方式 +- **SIMD 硬件加速**：基于 SIMD 指令集优化，提高向量运算效率 +- **搜索剪枝优化**：通过布隆过滤器等技术优化搜索过程 +- **可配置参数**：支持调整索引参数以平衡检索精度和性能 +- **混合查询支持**：支持向量数据与标量数据的联合查询 + +## 使用方法 + +### 向量字段定义 + +向量字段使用特殊的 [Field_vector](../../include/vidx/vidx_func.h#L43-L65) 类型定义，继承自 [Field_varstring](../../include/field.h#L793-L798)，使用二进制字符集存储浮点数数组。 + +```sql +CREATE TABLE table_name ( + id INT PRIMARY KEY, + vector_col VECTOR(128) -- 128维向量 +); +``` + +### 创建向量索引 + +向量索引可以通过以下语法创建： + +```sql +CREATE VECTOR INDEX vidx_name ON table_name (vector_col); -- 使用默认参数 +``` + +或者在表定义中直接指定： + +```sql +CREATE TABLE table_name ( + id INT PRIMARY KEY, + vector_col VECTOR(128), + VECTOR INDEX vidx_name (vector_col) M=6 DISTANCE=COSINE -- 指定参数 +); +``` + +### 函数支持 + +#### 向量转换函数 + +| 函数名 | 含义 | +|--------|------| +| VEC_FROMTEXT, TO_VECTOR, STRING_TO_VECTOR | 字符串转向量 | +| VEC_TOTEXT, FROM_VECTOR, VECTOR_TO_STRING | 向量转字符串 | + +#### 向量计算函数 + +| 函数名 | 含义 | +|--------|------| +| VECTOR_DIM | 向量维度 | +| VEC_DISTANCE, VEC_DISTANCE_EUCLIDEAN, VEC_DISTANCE_COSINE | 计算两向量间的距离
若参数之一是向量索引中的列，可以不指定距离类型，会自动识别向量索引的距离类型 | + +使用示例： + +```sql +-- 使用向量距离进行排序 +SELECT * FROM table_name ORDER BY VEC_DISTANCE(vector_col, VEC_FROMTEXT("[1,2,3,4,5]")) LIMIT 10; + +-- 在结果中显示距离值 +SELECT id, VEC_DISTANCE(vector_col, VEC_FROMTEXT("[1,2,3,4,5]")) AS distance +FROM table_name ORDER BY distance LIMIT 10; +``` + +### 参数介绍 + +#### 系统变量 + +| 变量名 | 描述 | 类型 | 默认值 | 范围 | +|--------|------|------|--------|------| +| vidx_disabled | 禁用向量列和向量索引的创建 | global | ON | ON, OFF | +| vidx_default_distance | 默认向量距离类型 | session | EUCLIDEAN | EUCLIDEAN,COSINE | +| vidx_hnsw_default_m | HNSW 算法默认 m | session | 6 | [3, 200] | +| vidx_hnsw_ef_search | HNSW 算法默认 ef_search | session | 20 | [1, 10000] | +| vidx_hnsw_cache_size | HNSW 算法默认使用内存限制 | global | 1024 * 1024 | [1048576,18446744073709551615] | + +#### 索引参数 + +- `M`: 控制图中每个节点的连接数，默认值为 6，有效范围是 3 到 200 +- `DISTANCE`: 构建索引的距离类型，默认值为 EUCLIDEAN + +### 注意事项 + +1. 当前版本只支持 RC (READ COMMITTED) 事务隔离级别 +2. 仅支持在 InnoDB 引擎表上创建向量索引。 +3. 创建、修改、删除向量索引无法使用 inplace 语法。 +4. 向量索引不能被设置为 INVISIBLE。 +5. 向量字段不能为 NULL。 +6. 向量索引的创建和维护会消耗额外的存储空间和计算资源 + +### 错误处理 + +- `ER_NOT_SUPPORTED_YET`: 不支持的事务隔离级别 +- `ER_WRONG_ARGUMENTS`: 函数参数错误 +- `ER_VECTOR_INDEX_USAGE`: 向量索引使用错误 +- `ER_VECTOR_INDEX_FAILED`: 向量索引操作失败 + +## 技术细节 + +### 向量搜索总体架构 + +HNSW 作为最为流行的 ANN 算法之一，在机构测评和工程实现上取得了广泛的认可和验证。目前 AliSQL 优先支持了基于 HNSW 算法的向量索引，向量搜索总体架构如下图所示。 + +
+vidx 总体架构 +
+ +- ANN 查询经过代价估计后选择合适的索引进行搜索，也可以用 FORCE INDEX 等 hint 指定向量索引进行搜索。 +- 逻辑上有一张完整的 HNSW 图，该图的信息被组织成一张辅助表持久化存储在磁盘中，表中每行数据代表 HNSW 图中的一个节点，在这张图上流转 HNSW 算法即可实现向量的插入和检索。 +- 不同于普通索引直接访问存储引擎，引入了向量索引插件，在内存中维护一个 HNSW 图的 Nodes Cache，提高查询效率。 + + +### HNSW 算法 + +HNSW（Hierarchical Navigable Small World）是一种基于多层图结构的高效近似最近邻（ANN）搜索算法。其设计可以概括为： +- 【分层跳表】第 0 层的图有全部点的信息，从低往高，高一层的图是低一层图的缩略图，只有低一层图的部分点，顶层用于快速跳转，底层用于精确搜索。 +- 【连接近邻】每一层都是根据向量距离连接的邻近图，每个点都记录它最近的几个点作为邻居。 + +
+hnsw +
+ +### 数据结构 + +AliSQL 引入了向量数据的公共缓存（MHNSW Share）和事务缓存（MHNSW Trx），用于加速向量查询性能并保证向量更新的事务安全，实现资源隔离与性能优化的平衡。公共缓存和事务缓存供不同的操作访问，有不同的设计目标： +- 【公共缓存】MHNSW Share 供只读事务访问，挂载于辅助表的 TABLE_SHARE 上。其核心目标是通过共享缓存减少重复加载向量节点的开销，提升查询效率。 +- 【事务缓存】MHNSW Trx 继承自 MHNSW Share，供读写事务使用，挂载于会话的 thd_set_ha_data。每个读写事务创建独立的 MHNSW Trx 实例，缓存其访问的节点包括其修改的节点，避免对公共缓存造成污染，仅在提交时去更新公共缓存。 + +### 向量计算优化 + +- 【预计算策略】在节点缓存加载阶段，系统会预先计算向量距离并缓存结果，从而避免对高频访问节点的重复计算。 +- 【SIMD 指令集加速】在计算优化层面，利用现代 CPU 的 SIMD 指令集（如 AVX512）对向量距离计算进行加速。通过布隆过滤器，系统能够批量处理多个向量，将原本需要多次执行的标量运算转换为并行化的向量操作。这一优化显著减少了 CPU 指令周期的消耗。 + +## RDS MySQL 开箱即用的向量能力 + +欢迎访问[阿里云 RDS MySQL 向量能力](https://help.aliyun.com/zh/rds/apsaradb-rds-for-mysql/vector-storage-1)，开源生态，开箱即用。