diff --git a/.github/workflows/merge-build-push.yml b/.github/workflows/merge-build-push.yml index 90cadad..1245236 100644 --- a/.github/workflows/merge-build-push.yml +++ b/.github/workflows/merge-build-push.yml @@ -4,17 +4,24 @@ on: pull_request_target: types: - closed + create: + workflow_dispatch: + inputs: + source_branch: + required: false + default: "" jobs: build-and-deploy: runs-on: ubuntu-latest - if: github.event.pull_request.merged == true + if: github.event_name == 'workflow_dispatch' || github.event_name == 'create' || github.event.pull_request.merged == true permissions: contents: write pull-requests: write steps: - name: PR was merged + if: github.event.pull_request.number != '' run: | echo "PR #${{ github.event.pull_request.number }} was merged into ${{ github.event.pull_request.base.ref }}." echo "Head commit was: ${{ github.event.pull_request.head.sha }}" @@ -23,6 +30,7 @@ jobs: uses: actions/checkout@v4 with: path: ivorysql_doc + ref: ${{ inputs.source_branch || github.ref }} - name: Checkout Doc Builder Repository (doc_builder) uses: actions/checkout@v4 @@ -82,6 +90,14 @@ jobs: else echo "WARNING: DETECTED_VERSION is empty. Skipping start_page update for $PLAYBOOK_FILE." fi + + # Add / support symbolic paths for the latest version + redirect:to + yq -i '.urls.latest_version_segment = "latest"' "$PLAYBOOK_FILE" + yq -i '.urls.latest_version_segment_strategy = "redirect:to"' "$PLAYBOOK_FILE" + + echo "Set urls.latest_version_segment = latest" + echo "Set urls.latest_version_segment_strategy = redirect:to" + echo "Modified content of $PLAYBOOK_FILE:" cat "$PLAYBOOK_FILE" echo "--- Finished modification for $PLAYBOOK_FILE ---" @@ -91,13 +107,68 @@ jobs: fi done + - name: Pin PDF assembler to merged PR branch + working-directory: ./ivory-doc-builder + env: + MERGED_BRANCH: ${{ inputs.source_branch || github.event.pull_request.base.ref || github.ref_name }} + COMPONENT_NAME: ivorysql-doc + run: | + if [[ -z "${MERGED_BRANCH}" ]]; then + echo "::error::Merged branch name is empty, cannot update PDF component version." + exit 1 + fi + + TARGET_COMPONENT_VERSION="${MERGED_BRANCH}@${COMPONENT_NAME}" + echo "Setting antora-assembler.yml component_versions to ${TARGET_COMPONENT_VERSION}" + yq -i ".component_versions = \"${TARGET_COMPONENT_VERSION}\"" antora-assembler.yml + echo "Updated antora-assembler.yml:" + cat antora-assembler.yml + - name: Checkout Web Repository (web) uses: actions/checkout@v4 with: repository: ${{ github.repository_owner }}/ivorysql_web path: www_publish_target token: ${{ secrets.WEB_TOKEN }} - + + - name: Ensure web index redirects to latest home + id: update_web_index + working-directory: ./www_publish_target + env: + LATEST_VERSION: ${{ steps.latest_version_step.outputs.version }} + MERGED_PR_BASE: ${{ github.event.pull_request.base.ref || github.ref_name }} + run: | + set -euo pipefail + + TARGET_BRANCH="v${LATEST_VERSION}" + EXPECTED_PATH="ivorysql-doc/v${LATEST_VERSION}/v${LATEST_VERSION}/welcome.html" + + if [[ "${MERGED_PR_BASE}" != "${TARGET_BRANCH}" ]]; then + echo "Base branch ${MERGED_PR_BASE} is not the latest version branch ${TARGET_BRANCH}, skip index redirect check." + echo "index_updated=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + UPDATE_NEEDED=false + for lang in cn en; do + FILE_PATH="docs/${lang}/index.html" + if [[ ! -f "${FILE_PATH}" ]]; then + echo "Missing ${FILE_PATH}, cannot update redirect." + continue + fi + + if grep -q "${EXPECTED_PATH}" "${FILE_PATH}"; then + echo "${FILE_PATH} already points to latest ${LATEST_VERSION}." + else + # Replace all version segments like vX.Y or vX.Y.Z in href/location/meta/script targets + sed -i -E "s@ivorysql-doc/v[0-9]+(\\.[0-9]+){1,2}/v[0-9]+(\\.[0-9]+){1,2}/welcome\\.html@${EXPECTED_PATH}@g" "${FILE_PATH}" + UPDATE_NEEDED=true + echo "Updated ${FILE_PATH} to latest ${LATEST_VERSION} redirect." + fi + done + + echo "index_updated=${UPDATE_NEEDED}" >> "$GITHUB_OUTPUT" + - name: Setup Ruby and Bundler uses: ruby/setup-ruby@v1 with: @@ -117,7 +188,7 @@ jobs: - name: Install Antora CLI run: | echo "Installing Antora packages local..." - npm install --global antora@3.1.7 @antora/lunr-extension @antora/pdf-extension @node-rs/jieba + npm install --global antora@3.1.7 @antora/lunr-extension @antora/pdf-extension@1.0.0-beta.20 @node-rs/jieba - name: Build English Documentation working-directory: ./ivory-doc-builder @@ -127,12 +198,66 @@ jobs: ls ../www_publish_target/ npx antora generate --stacktrace --to-dir ../www_publish_target/docs/en antora-playbook-EN.yml + - name: Copy English PDF export into web repo + working-directory: ./ivory-doc-builder + env: + MERGED_BRANCH: ${{ inputs.source_branch || github.event.pull_request.base.ref || github.ref_name }} + COMPONENT_NAME: ivorysql-doc + run: | + set -euo pipefail + + if [[ -z "${MERGED_BRANCH}" ]]; then + echo "::error::Merged branch name is empty, cannot locate PDF output." + exit 1 + fi + + SOURCE_PDF_EN="build/assembler-pdf/${COMPONENT_NAME}/${MERGED_BRANCH}/_exports/index.pdf" + DEST_EN="../www_publish_target/docs/en/${COMPONENT_NAME}/${MERGED_BRANCH}/ivorysql.pdf" + + if [[ ! -f "${SOURCE_PDF_EN}" ]]; then + echo "::error::English PDF not found at ${SOURCE_PDF_EN}" + exit 1 + fi + + echo "Copying English PDF from ${SOURCE_PDF_EN} to web repo target..." + mkdir -p "$(dirname "${DEST_EN}")" + cp "${SOURCE_PDF_EN}" "${DEST_EN}" + echo "English PDF copied to:" + echo " - ${DEST_EN}" + - name: Build Chinese Documentation working-directory: ./ivory-doc-builder run: | echo "Building Chinese site..." npx antora generate --stacktrace --to-dir ../www_publish_target/docs/cn antora-playbook-CN.yml + - name: Copy Chinese PDF export into web repo + working-directory: ./ivory-doc-builder + env: + MERGED_BRANCH: ${{ inputs.source_branch || github.event.pull_request.base.ref || github.ref_name }} + COMPONENT_NAME: ivorysql-doc + run: | + set -euo pipefail + + if [[ -z "${MERGED_BRANCH}" ]]; then + echo "::error::Merged branch name is empty, cannot locate PDF output." + exit 1 + fi + + SOURCE_PDF="build/assembler-pdf/${COMPONENT_NAME}/${MERGED_BRANCH}/_exports/index.pdf" + DEST_CN="../www_publish_target/docs/cn/${COMPONENT_NAME}/${MERGED_BRANCH}/ivorysql.pdf" + + if [[ ! -f "${SOURCE_PDF}" ]]; then + echo "::error::PDF not found at ${SOURCE_PDF}" + exit 1 + fi + + echo "Copying PDF from ${SOURCE_PDF} to web repo targets..." + mkdir -p "$(dirname "${DEST_CN}")" + cp "${SOURCE_PDF}" "${DEST_CN}" + echo "PDF copied to:" + echo " - ${DEST_CN}" + - name: Commit and Push to web Repository new branch , pull request id: commit_push_new_branch working-directory: ./www_publish_target diff --git a/.github/workflows/pr-preview.yml b/.github/workflows/pr-preview.yml index 8ff0446..70f8fb2 100644 --- a/.github/workflows/pr-preview.yml +++ b/.github/workflows/pr-preview.yml @@ -62,6 +62,8 @@ jobs: yq -i ".content.sources[0].url = \"$NEW_LOCAL_URL\"" "$PLAYBOOK_FILE" + yq -i ".ui.bundle.url = \"./entemplates.zip\"" "$PLAYBOOK_FILE" + yq -i ".content.sources[0].branches = [\"HEAD\"]" "$PLAYBOOK_FILE" yq -i ".content.sources[0].edit_url = false" "$PLAYBOOK_FILE" @@ -107,7 +109,7 @@ jobs: - name: Install Antora CLI run: | echo "Installing Antora packages local..." - npm install --global antora@3.1.7 @antora/lunr-extension @antora/pdf-extension @node-rs/jieba + npm install --global antora@3.1.7 @antora/lunr-extension @antora/pdf-extension@1.0.0-beta.20 @node-rs/jieba - name: Build English Documentation working-directory: ./ivory-doc-builder diff --git a/CN/modules/ROOT/nav.adoc b/CN/modules/ROOT/nav.adoc index deb089a..473ba5e 100644 --- a/CN/modules/ROOT/nav.adoc +++ b/CN/modules/ROOT/nav.adoc @@ -1,90 +1,123 @@ -* IvorySQL -** xref:master/welcome.adoc[欢迎] -** xref:master/1.adoc[发行说明] -** xref:master/2.adoc[关于IvorySQL] -** IvorySQL入门 -*** xref:master/3.1.adoc[快速开始] -*** xref:master/3.2.adoc[日常监控] -*** xref:master/3.3.adoc[日常维护] -** IvorySQL高级 -*** xref:master/4.1.adoc[安装指南] -*** xref:master/4.2.adoc[集群搭建] -*** xref:master/4.5.adoc[迁移指南] -*** xref:master/4.3.adoc[开发者指南] -*** 容器化指南 -**** xref:master/4.6.1.adoc[K8S部署] -**** xref:master/4.6.2.adoc[Operator部署] -**** xref:master/4.6.4.adoc[Docker & Podman部署] -**** xref:master/4.6.3.adoc[Docker Swarm & Docker Compose部署] -*** xref:master/4.4.adoc[运维管理指南] -*** 云服务平台指南 -**** xref:master/4.7.1.adoc[IvorySQL Cloud安装] -**** xref:master/4.7.2.adoc[IvorySQL Cloud使用] -** IvorySQL生态 -*** xref:master/cpu_arch_adp.adoc[芯片架构适配] -*** xref:master/os_arch_adp.adoc[操作系统适配] -*** 生态组件适配 -**** xref:master/5.0.adoc[概述] -**** xref:master/5.1.adoc[postgis] -**** xref:master/5.2.adoc[pgvector] -**** xref:master/5.3.adoc[pgddl(DDL Extractor)] -**** xref:master/5.4.adoc[pg_cron] -**** xref:master/5.5.adoc[pgsql-http] -**** xref:master/5.6.adoc[plpgsql_check] -**** xref:master/5.7.adoc[pgroonga] -**** xref:master/5.8.adoc[pgaudit] -**** xref:master/5.9.adoc[pgrouting] -**** xref:master/5.10.adoc[system_stats] -** IvorySQL架构设计 -*** 查询处理 -**** xref:master/6.1.1.adoc[双parser] -*** 兼容框架 -**** xref:master/7.1.adoc[框架设计] -**** xref:master/7.2.adoc[GUC框架] -**** xref:master/7.4.adoc[双模式设计] -**** xref:master/6.2.1.adoc[initdb过程] -*** 兼容特性 -**** xref:master/6.3.1.adoc[like] -**** xref:master/6.3.3.adoc[RowID] -**** xref:master/6.3.2.adoc[OUT 参数] -**** xref:master/6.3.4.adoc[%TYPE、%ROWTYPE] -**** xref:master/6.3.5.adoc[NLS 参数] -**** xref:master/6.3.6.adoc[函数与存储过程] -**** xref:master/6.3.7.adoc[嵌套子函数] -**** xref:master/6.3.8.adoc[Force View] -**** xref:master/6.3.9.adoc[大小写转换] -**** xref:master/6.3.10.adoc[sys_guid 函数] -**** xref:master/6.3.11.adoc[空字符串转null] -**** xref:master/6.3.12.adoc[CALL INTO] -*** 内置函数 -**** xref:master/6.4.1.adoc[sys_context] -**** xref:master/6.4.2.adoc[userenv] -*** xref:master/6.5.adoc[国标GB18030] -** Oracle兼容功能列表 -*** xref:master/7.3.adoc[1、大小写转换] -*** xref:master/7.5.adoc[2、LIKE操作符] -*** xref:master/7.6.adoc[3、匿名块] -*** xref:master/7.7.adoc[4、函数与存储过程] -*** xref:master/7.8.adoc[5、内置数据类型与内置函数] -*** xref:master/7.9.adoc[6、端口与IP] -*** xref:master/7.10.adoc[7、XML函数] -*** xref:master/7.11.adoc[8、sequence] -*** xref:master/7.12.adoc[9、包] -*** xref:master/7.13.adoc[10、不可见列] -*** xref:master/7.14.adoc[11、RowID] -*** xref:master/7.15.adoc[12、OUT 参数] -*** xref:master/7.16.adoc[13、%TYPE、%ROWTYPE] -*** xref:master/7.17.adoc[14、NLS 参数] -*** xref:master/7.18.adoc[15、Force View] -*** xref:master/7.19.adoc[16、嵌套子函数] -*** xref:master/7.20.adoc[17、sys_guid 函数] -*** xref:master/7.21.adoc[18、空字符串转null] -*** xref:master/7.22.adoc[19、CALL INTO] -** IvorySQL贡献指南 -*** xref:master/8.1.adoc[社区贡献指南] -*** xref:master/8.2.adoc[asciidoc语法快速参考] -** xref:master/9.adoc[工具参考] -** xref:master/10.adoc[FAQ] -* PostgreSQL -** xref:master/100.adoc[PG参数参考手册] -** xref:master/110.adoc[PG函数参考手册] +* xref:master/about_ivorysql.adoc[关于IvorySQL] +** xref:master/welcome.adoc[欢迎] +** xref:master/release_notes.adoc[发行说明] +* 快速上手 +** xref:master/getting-started/quick_start.adoc[快速开始] +* 安装部署 +** xref:master/installation_guide.adoc[安装指南] +** xref:master/cluster_setup.adoc[集群搭建] +* Oracle兼容功能 +** xref:master/oracle_compatibility/compat_case_conversion.adoc[1、大小写转换] +** xref:master/oracle_compatibility/compat_like_operator.adoc[2、LIKE操作符] +** xref:master/oracle_compatibility/anonymous_block.adoc[3、匿名块] +** xref:master/oracle_compatibility/compat_function_procedure.adoc[4、函数与存储过程] +** xref:master/oracle_compatibility/builtin_types_functions.adoc[5、内置数据类型与内置函数] +** xref:master/oracle_compatibility/port_ip.adoc[6、端口与IP] +** xref:master/oracle_compatibility/xml_functions.adoc[7、XML函数] +** xref:master/oracle_compatibility/sequence.adoc[8、sequence] +** xref:master/oracle_compatibility/package.adoc[9、包] +** xref:master/oracle_compatibility/invisible_column.adoc[10、不可见列] +** xref:master/oracle_compatibility/compat_rowid.adoc[11、RowID] +** xref:master/oracle_compatibility/compat_out_parameter.adoc[12、OUT 参数] +** xref:master/oracle_compatibility/compat_type_rowtype.adoc[13、%TYPE、%ROWTYPE] +** xref:master/oracle_compatibility/compat_nls_parameter.adoc[14、NLS 参数] +** xref:master/oracle_compatibility/compat_force_view.adoc[15、Force View] +** xref:master/oracle_compatibility/compat_nested_function.adoc[16、嵌套子函数] +** xref:master/oracle_compatibility/compat_sys_guid.adoc[17、sys_guid 函数] +** xref:master/oracle_compatibility/compat_empty_string_to_null.adoc[18、空字符串转null] +** xref:master/oracle_compatibility/compat_call_into.adoc[19、CALL INTO] +** xref:master/oracle_compatibility/compat_read_only_view.adoc[20、视图只读] +** xref:master/oracle_compatibility/with_function_procedure.adoc[21、WITH FUNCTION/PROCEDURE] +** xref:master/oracle_compatibility/compat_create_index_online.adoc[22、索引 ONLINE 参数] +** xref:master/oracle_compatibility/compat_stragg.adoc[23、STRAGG 函数] +* 容器化与云服务 +** 容器化指南 +*** xref:master/containerization/k8s_deployment.adoc[K8S部署] +*** xref:master/containerization/operator_deployment.adoc[Operator部署] +*** xref:master/containerization/docker_podman_deployment.adoc[Docker & Podman部署] +*** xref:master/containerization/docker_swarm_compose_deployment.adoc[Docker Swarm & Docker Compose部署] +** 云服务平台指南 +*** xref:master/cloud_platform/ivorysql_cloud_installation.adoc[IvorySQL Cloud安装] +*** xref:master/cloud_platform/ivorysql_cloud_usage.adoc[IvorySQL Cloud使用] +* IvorySQL生态 +** xref:master/cpu_os_adaptation/cpu_architecture_adaptation.adoc[芯片架构适配] +** xref:master/cpu_os_adaptation/os_architecture_adaptation.adoc[操作系统适配] +** 生态组件适配 +*** xref:master/ecosystem_components/ecosystem_overview.adoc[概述] +*** xref:master/ecosystem_components/postgis.adoc[postgis] +*** xref:master/ecosystem_components/pgvector.adoc[pgvector] +*** xref:master/ecosystem_components/pgddl.adoc[pgddl(DDL Extractor)] +*** xref:master/ecosystem_components/pg_cron.adoc[pg_cron] +*** xref:master/ecosystem_components/pgsql_http.adoc[pgsql-http] +*** xref:master/ecosystem_components/plpgsql_check.adoc[plpgsql_check] +*** xref:master/ecosystem_components/pgroonga.adoc[pgroonga] +*** xref:master/ecosystem_components/pgaudit.adoc[pgaudit] +*** xref:master/ecosystem_components/pgrouting.adoc[pgrouting] +*** xref:master/ecosystem_components/system_stats.adoc[system_stats] +*** xref:master/ecosystem_components/wal2json.adoc[wal2json] +*** xref:master/ecosystem_components/pg_stat_monitor.adoc[pg_stat_monitor] +*** xref:master/ecosystem_components/pg_ai_query.adoc[pg_ai_query] +*** xref:master/ecosystem_components/pg_partman.adoc[pg_partman] +*** xref:master/ecosystem_components/pgbouncer.adoc[pgbouncer] +*** xref:master/ecosystem_components/age.adoc[age] +*** xref:master/ecosystem_components/pg_curl.adoc[pg_curl] +*** xref:master/ecosystem_components/pg_textsearch.adoc[pg_textsearch] +*** xref:master/ecosystem_components/pg_hint_plan.adoc[pg_hint_plan] +*** xref:master/ecosystem_components/redis_fdw.adoc[redis_fdw] +*** xref:master/ecosystem_components/pg_show_plans.adoc[pg_show_plans] +*** xref:master/ecosystem_components/pg_bulkload.adoc[pg_bulkload] +*** xref:master/ecosystem_components/pg_bigm.adoc[pg_bigm] +*** xref:master/ecosystem_components/pg_profile.adoc[pg_profile] +*** xref:master/ecosystem_components/pg_repack.adoc[pg_repack] +*** xref:master/ecosystem_components/pgdog.adoc[PgDog] +*** xref:master/ecosystem_components/pg_readonly.adoc[pg_readonly] +*** xref:master/ecosystem_components/zhparser.adoc[zhparser] +*** xref:master/ecosystem_components/pgbackrest.adoc[pgBackRest] +*** xref:master/ecosystem_components/set_user.adoc[set_user] +* 监控运维 +** xref:master/getting-started/daily_monitoring.adoc[日常监控] +** xref:master/getting-started/daily_maintenance.adoc[日常维护] +** xref:master/operation_guide.adoc[运维管理指南] +* 数据迁移 +** xref:master/migration_guide.adoc[迁移指南] +* IvorySQL开发者 +** xref:master/contribution/community_contribution_guide.adoc[社区贡献指南] +** xref:master/developer_guide.adoc[开发者指南] +** IvorySQL架构设计 +*** 查询处理 +**** xref:master/architecture/dual_parser.adoc[双parser] +*** 兼容框架 +**** xref:master/architecture/framework_design.adoc[框架设计] +**** xref:master/architecture/guc_framework.adoc[GUC框架] +**** xref:master/architecture/dual_mode_design.adoc[双模式设计] +**** xref:master/architecture/initdb_process.adoc[initdb过程] +*** 兼容特性 +**** xref:master/compatibility_features_design/like_operator.adoc[like] +**** xref:master/compatibility_features_design/rowid.adoc[RowID] +**** xref:master/compatibility_features_design/out_parameter.adoc[OUT 参数] +**** xref:master/compatibility_features_design/type_rowtype.adoc[%TYPE、%ROWTYPE] +**** xref:master/compatibility_features_design/nls_parameter.adoc[NLS 参数] +**** xref:master/compatibility_features_design/function_procedure.adoc[函数与存储过程] +**** xref:master/compatibility_features_design/nested_function.adoc[嵌套子函数] +**** xref:master/compatibility_features_design/force_view.adoc[Force View] +**** xref:master/compatibility_features_design/case_conversion.adoc[大小写转换] +**** xref:master/compatibility_features_design/sys_guid_function.adoc[sys_guid 函数] +**** xref:master/compatibility_features_design/empty_string_to_null.adoc[空字符串转null] +**** xref:master/compatibility_features_design/call_into.adoc[CALL INTO] +**** xref:master/compatibility_features_design/read_only_view.adoc[视图只读] +**** xref:master/compatibility_features_design/with_function_procedure_impl.adoc[WITH FUNCTION/PROCEDURE] +**** xref:master/compatibility_features_design/create_index_online.adoc[索引 ONLINE 参数] +*** 内置函数 +**** xref:master/oracle_builtin_functions/sys_context.adoc[sys_context] +**** xref:master/oracle_builtin_functions/userenv.adoc[userenv] +**** xref:master/oracle_builtin_functions/rawtohex.adoc[rawtohex] +**** xref:master/oracle_builtin_functions/stragg.adoc[stragg] +*** xref:master/gb18030.adoc[国标GB18030] +* 参考指南 +** xref:master/tools_reference.adoc[工具参考] +** xref:master/contribution/asciidoc_syntax_reference.adoc[asciidoc语法快速参考] +** xref:master/pg_reference/pg_parameters_reference.adoc[PG参数参考手册] +** xref:master/pg_reference/pg_functions_reference.adoc[PG函数参考手册] +* 常见问题解答 +** xref:master/contribution/faq.adoc[FAQ] + diff --git a/CN/modules/ROOT/pages/master/4.7.adoc b/CN/modules/ROOT/pages/master/4.7.adoc deleted file mode 100644 index e69de29..0000000 diff --git a/CN/modules/ROOT/pages/master/5.0.adoc b/CN/modules/ROOT/pages/master/5.0.adoc deleted file mode 100644 index 4fe9f8b..0000000 --- a/CN/modules/ROOT/pages/master/5.0.adoc +++ /dev/null @@ -1,28 +0,0 @@ -:sectnums: -:sectnumlevels: 5 - - -[discrete] -== IvorySQL生态插件适配列表 - -IvorySQL 作为一款兼容 Oracle 且基于 PostgreSQL 的高级开源数据库，具备强大的扩展能力，支持丰富的生态系统插件。这些插件可以帮助用户在不同场景下增强数据库功能，包括地理信息处理、向量检索、全文搜索、数据定义提取和路径规划等。以下是当前 IvorySQL 官方兼容和支持的主要插件列表： - - -[cols="1,2,1,3,3"] -|==== -|*序号*|*插件名称*|*版本*|*功能描述*|*适用场景* -| 1 | xref:master/5.1.adoc[postgis] | 3.5.4 | 为 IvorySQL 提供地理空间数据支持，包括空间索引、空间函数和地理对象存储 | 地理信息系统（GIS）、地图服务、位置数据分析 -| 2 | xref:master/5.2.adoc[pgvector] | 0.8.1 | 支持向量相似性搜索，可用于存储和检索高维向量数据| AI 应用、图像检索、推荐系统、语义搜索 -| 3 | xref:master/5.3.adoc[pgddl (DDL Extractor)] | 0.31 | 提取数据库中的 DDL（数据定义语言）语句，便于版本管理和迁移 | 数据库版本控制、CI/CD 集成、结构比对与同步 -| 4 | xref:master/5.4.adoc[pg_cron]​ | 1.6.0 | 提供数据库内部的定时任务调度功能，支持定期执行SQL语句 | 数据清理、定期统计、自动化维护任务 -| 5 | xref:master/5.5.adoc[pgsql-http]​ | 1.7.0 | 允许在SQL中发起HTTP请求，与外部Web服务进行交互 | 数据采集、API集成、微服务调用 -| 6 | xref:master/5.6.adoc[plpgsql_check] | 2.8 | 提供PL/pgSQL代码的静态分析功能，可在开发阶段发现潜在错误 | 存储过程开发、代码质量检查、调试优化 -| 7 | xref:master/5.7.adoc[pgroonga] | 4.0.4 | 提供​非英语语言全文搜索功能，满足高性能应用的需求 | 中日韩等语言的全文搜索功能 -| 8 | xref:master/5.8.adoc[pgaudit] | 18.0 | 提供细粒度的审计功能，记录数据库操作日志，便于安全审计和合规性检查 | 数据库安全审计、合规性检查、审计报告生成 -| 9 | xref:master/5.9.adoc[pgrouting] | 3.8.0 | 提供地理空间数据的路由计算功能，支持多种算法和数据格式 | 地理空间分析、路径规划、物流优化 -| 10 | xref:master/5.10.adoc[system_stats] | 3.2 | 提供用于访问系统级统计信息的函数 | 系统监控 -|==== - -这些插件均经过 IvorySQL 团队的测试和适配，确保在 IvorySQL 环境下稳定运行。用户可以根据业务需求选择合适的插件，进一步提升数据库系统的能力和灵活性。 - -我们也将持续扩展和丰富 IvorySQL 的插件生态，欢迎社区开发者提交新的插件适配建议或代码贡献。如需了解更多每个插件的详细使用方法和最新兼容版本，请参阅各插件对应的文档章节。 diff --git a/CN/modules/ROOT/pages/master/2.adoc b/CN/modules/ROOT/pages/master/about_ivorysql.adoc similarity index 98% rename from CN/modules/ROOT/pages/master/2.adoc rename to CN/modules/ROOT/pages/master/about_ivorysql.adoc index 5688a4c..8d99c9c 100644 --- a/CN/modules/ROOT/pages/master/2.adoc +++ b/CN/modules/ROOT/pages/master/about_ivorysql.adoc @@ -87,4 +87,6 @@ IvorySQL是一个功能强大的开源对象关系数据库管理系统(ORDBMS) * 嵌套子函数 * sys_guid 函数 * 空字符串转null -* CALL INTO \ No newline at end of file +* CALL INTO +* 视图只读 +* WITH FUNCTION/PROCEDURE diff --git a/CN/modules/ROOT/pages/master/7.4.adoc b/CN/modules/ROOT/pages/master/architecture/dual_mode_design.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.4.adoc rename to CN/modules/ROOT/pages/master/architecture/dual_mode_design.adoc diff --git a/CN/modules/ROOT/pages/master/6.1.1.adoc b/CN/modules/ROOT/pages/master/architecture/dual_parser.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.1.1.adoc rename to CN/modules/ROOT/pages/master/architecture/dual_parser.adoc diff --git a/CN/modules/ROOT/pages/master/7.1.adoc b/CN/modules/ROOT/pages/master/architecture/framework_design.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.1.adoc rename to CN/modules/ROOT/pages/master/architecture/framework_design.adoc diff --git a/CN/modules/ROOT/pages/master/7.2.adoc b/CN/modules/ROOT/pages/master/architecture/guc_framework.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.2.adoc rename to CN/modules/ROOT/pages/master/architecture/guc_framework.adoc diff --git a/CN/modules/ROOT/pages/master/6.2.1.adoc b/CN/modules/ROOT/pages/master/architecture/initdb_process.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.2.1.adoc rename to CN/modules/ROOT/pages/master/architecture/initdb_process.adoc diff --git a/CN/modules/ROOT/pages/master/4.7.1.adoc b/CN/modules/ROOT/pages/master/cloud_platform/ivorysql_cloud_installation.adoc similarity index 98% rename from CN/modules/ROOT/pages/master/4.7.1.adoc rename to CN/modules/ROOT/pages/master/cloud_platform/ivorysql_cloud_installation.adoc index 3c8e2b5..8ea02e3 100644 --- a/CN/modules/ROOT/pages/master/4.7.1.adoc +++ b/CN/modules/ROOT/pages/master/cloud_platform/ivorysql_cloud_installation.adoc @@ -364,6 +364,8 @@ https://github.com/IvorySQL/ivory-operator/tree/IVYO_REL_5_STABLE[https://github 如果服务器可以直接访问到docker hub,可以跳过该章节。否则需要在所有的K8S集群节点提前load 如下docker镜像 +TIP: 国内用户也可使用 IvorySQL 社区的 Harbor 镜像服务拉取以下镜像，将 `docker.io/` 替换为 `registry.highgo.com/` 即可。 + [literal] ---- docker.io/ivorysql/pgadmin:ubi8-9.9-5.0-1 diff --git a/CN/modules/ROOT/pages/master/4.7.2.adoc b/CN/modules/ROOT/pages/master/cloud_platform/ivorysql_cloud_usage.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/4.7.2.adoc rename to CN/modules/ROOT/pages/master/cloud_platform/ivorysql_cloud_usage.adoc diff --git a/CN/modules/ROOT/pages/master/4.2.adoc b/CN/modules/ROOT/pages/master/cluster_setup.adoc similarity index 92% rename from CN/modules/ROOT/pages/master/4.2.adoc rename to CN/modules/ROOT/pages/master/cluster_setup.adoc index 67a2ec8..11d8421 100644 --- a/CN/modules/ROOT/pages/master/4.2.adoc +++ b/CN/modules/ROOT/pages/master/cluster_setup.adoc @@ -8,9 +8,9 @@ == 主节点 === 安装并启动数据库 -yum源快速安装数据库，请参考xref:master/3.1.adoc#从yum源安装ivorysql数据库[从yum源安装ivorysql数据库]。 +yum源快速安装数据库，请参考xref:master/getting-started/quick_start.adoc#从yum源安装ivorysql数据库[从yum源安装ivorysql数据库]。 -想要获取更多安装方式，请参考xref:master/4.1.adoc[安装指南]。 +想要获取更多安装方式，请参考xref:master/installation_guide.adoc[安装指南]。 [NOTE] 主节点数据库需要安装并**启动** @@ -56,9 +56,9 @@ $ pg_ctl restart == 备节点 === 安装数据库 -Yum源快速安装数据库，请参考xref:master/3.1.adoc#从yum源安装ivorysql数据库[从yum源安装ivorysql数据库]。 +Yum源快速安装数据库，请参考xref:master/getting-started/quick_start.adoc#从yum源安装ivorysql数据库[从yum源安装ivorysql数据库]。 -想要获取更多安装方式，请参考xref:master/4.1.adoc[安装指南]。 +想要获取更多安装方式，请参考xref:master/installation_guide.adoc[安装指南]。 [NOTE] 备节点数据库只需要安装，**不需要启动** diff --git a/CN/modules/ROOT/pages/master/6.3.12.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/call_into.adoc similarity index 99% rename from CN/modules/ROOT/pages/master/6.3.12.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/call_into.adoc index 96d405b..a0d55c9 100644 --- a/CN/modules/ROOT/pages/master/6.3.12.adoc +++ b/CN/modules/ROOT/pages/master/compatibility_features_design/call_into.adoc @@ -129,7 +129,7 @@ typedef enum IvyStmtType { IVY_STMT_UNKNOW, IVY_STMT_DO, - IVY_STMT_DOFROMCALL, /* new statementt type */ + IVY_STMT_DOFROMCALL, /* new statement type */ IVY_STMT_DOHANDLED, IVY_STMT_OTHERS } IvyStmtType; diff --git a/CN/modules/ROOT/pages/master/6.3.9.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/case_conversion.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.9.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/case_conversion.adoc diff --git a/CN/modules/ROOT/pages/master/compatibility_features_design/create_index_online.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/create_index_online.adoc new file mode 100644 index 0000000..a952e28 --- /dev/null +++ b/CN/modules/ROOT/pages/master/compatibility_features_design/create_index_online.adoc @@ -0,0 +1,142 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += 索引 ONLINE 参数 + +== 目的 + +IvorySQL 数据库支持在创建索引时使用 `ONLINE` 参数，用于在线创建索引，同时不阻塞DML操作。 + +== 实现说明 + +=== 数据结构扩展 + +`IndexStmt` 新增 `online_keyword` 字段。 + +[source,c] +---- +bool transformed; /* true when transformIndexStmt is finished */ +bool concurrent; /* should this be a concurrent index build? */ +bool online_keyword; /* was ONLINE keyword used (as opposed to CONCURRENTLY)? */ +---- + +=== 语法与解析 + +==== 语法规则扩展 + +在 `ora_gram.y` 文件中引入一个弹性选项列表 `create_index_opt_list` / `create_index_opt`，类似现有的 `rebuild_index_opt_list`： + +[source,yacc] +---- +create_index_opt_list: + create_index_opt_list create_index_opt { $$ = lappend($1, $2); } + | /* EMPTY */ { $$ = NIL; } +; + +create_index_opt: + ONLINE + { $$ = makeDefElem("online", (Node *) makeBoolean(true), @1); } + | TABLESPACE name + { $$ = makeDefElem("tablespace", (Node *) makeString($2), @1); } +; +---- + +修改 `IndexStmt` 的两个产生式，将 `OptTableSpace` 替换为 `create_index_opt_list`，并在 action 中从选项列表中提取 `online` 和 `tablespace`。 +[source,yacc] +---- +IndexStmt: CREATE opt_unique INDEX opt_concurrently opt_single_name + ON relation_expr access_method_clause '(' index_params ')' + opt_include opt_unique_null_treatment opt_reloptions + create_index_opt_list where_clause + { + IndexStmt *n = makeNode(IndexStmt); + bool online = false; + bool online_seen = false; + char *tablespace = NULL; + ListCell *lc; + + /* 解析 create_index_opt_list，提取 online 和 tablespace */ + foreach(lc, $15) + { + DefElem *opt = (DefElem *) lfirst(lc); + + if (strcmp(opt->defname, "online") == 0) + { + if (online_seen) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ONLINE specified multiple times"), + parser_errposition(opt->location))); + online = defGetBoolean(opt); + online_seen = true; + } + else if (strcmp(opt->defname, "tablespace") == 0) + { + if (tablespace != NULL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("TABLESPACE specified multiple times"), + parser_errposition(opt->location))); + tablespace = defGetString(opt); + } + } + + /* CONCURRENTLY 与 ONLINE 互斥 */ + if ($4 && online) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot use both CONCURRENTLY and ONLINE"), + parser_errposition(@4))); + + n->unique = $2; + n->concurrent = $4 || online; + n->online_keyword = online; + n->idxname = $5; + n->relation = $7; + n->accessMethod = $8; + n->indexParams = $10; + n->indexIncludingParams = $12; + n->nulls_not_distinct = !$13; + n->options = $14; /* opt_reloptions (WITH clause) */ + n->tableSpace = tablespace; + n->whereClause = $16; + n->excludeOpNames = NIL; + n->idxcomment = NULL; + n->indexOid = InvalidOid; + n->oldNumber = InvalidRelFileNumber; + n->oldCreateSubid = InvalidSubTransactionId; + n->oldFirstRelfilelocatorSubid = InvalidSubTransactionId; + n->primary = false; + n->isconstraint = false; + n->deferrable = false; + n->initdeferred = false; + n->transformed = false; + n->if_not_exists = false; + n->reset_default_tblspc = false; + $$ = (Node *) n; + } +---- + +==== 执行层修改（`indexcmds.c`） + +`DefineIndex()` 中已有临时表降级逻辑（`concurrent = false` when temp table），需在相同位置补充分区表的降级逻辑： + +[source,c] +---- +/* 已有：临时表降级 */ +if (stmt->concurrent && get_rel_persistence(tableId) != RELPERSISTENCE_TEMP) + concurrent = true; +else + concurrent = false; + +/* 新增：分区表降级 + * Oracle 对分区表 CREATE INDEX ONLINE 返回成功（全局非分区索引）。 + * PostgreSQL 的 concurrent + partitioned 路径会报错，因此在 Oracle 解析器 + * 模式下（stmt->online_keyword = true）对分区表静默降级为普通构建。 + */ +if (concurrent && partitioned && stmt->online_keyword) + concurrent = false; +---- + diff --git a/CN/modules/ROOT/pages/master/6.3.11.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/empty_string_to_null.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.11.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/empty_string_to_null.adoc diff --git a/CN/modules/ROOT/pages/master/6.3.8.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/force_view.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.8.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/force_view.adoc diff --git a/CN/modules/ROOT/pages/master/6.3.6.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/function_procedure.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.6.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/function_procedure.adoc diff --git a/CN/modules/ROOT/pages/master/6.3.1.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/like_operator.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.1.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/like_operator.adoc diff --git a/CN/modules/ROOT/pages/master/6.3.7.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/nested_function.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.7.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/nested_function.adoc diff --git a/CN/modules/ROOT/pages/master/6.3.5.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/nls_parameter.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.5.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/nls_parameter.adoc diff --git a/CN/modules/ROOT/pages/master/6.3.2.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/out_parameter.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.2.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/out_parameter.adoc diff --git a/CN/modules/ROOT/pages/master/compatibility_features_design/read_only_view.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/read_only_view.adoc new file mode 100644 index 0000000..959a58e --- /dev/null +++ b/CN/modules/ROOT/pages/master/compatibility_features_design/read_only_view.adoc @@ -0,0 +1,235 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += 视图只读属性 + +== 目的 + +Oracle 数据库支持在创建视图时使用 `WITH READ ONLY` 子句，将视图设置为只读状态，防止对视图执行任何数据修改操作（INSERT、UPDATE、DELETE、MERGE）。为了兼容 Oracle 的这一特性，IvorySQL 实现了视图只读属性功能。当视图被标记为只读后，任何试图修改视图数据的 DML 操作都会被拒绝，从而保证 Oracle 应用程序在 IvorySQL 上的行为一致性。 + +== 实现说明 + +=== 语法与解析 + +==== 语法规则扩展 + +在 `ora_gram.y` 文件中添加了 `READ_ONLY_OPTION` 枚举值，并扩展了 `opt_check_option` 语法规则： + +[source,yacc] +---- +opt_check_option: + WITH CHECK OPTION { $$ = CASCADED_CHECK_OPTION; } + | WITH CASCADED CHECK OPTION { $$ = CASCADED_CHECK_OPTION; } + | WITH LOCAL CHECK OPTION { $$ = LOCAL_CHECK_OPTION; } + | WITH READ ONLY { $$ = READ_ONLY_OPTION; } /* 新增 */ + | /* EMPTY */ { $$ = NO_CHECK_OPTION; } + ; +---- + +==== ViewStmt 结构体扩展 + +在 `parsenodes.h` 文件中，`ViewCheckOption` 枚举新增 `READ_ONLY_OPTION` 选项，`ViewStmt` 结构体新增 `readOnly` 字段： + +[source,c] +---- +typedef enum ViewCheckOption { + NO_CHECK_OPTION, + LOCAL_CHECK_OPTION, + CASCADED_CHECK_OPTION, + READ_ONLY_OPTION, /* WITH READ ONLY (Oracle compat) */ +} ViewCheckOption; + +typedef struct ViewStmt { + // ... 其他字段 + bool readOnly; /* WITH READ ONLY (Oracle compat) */ +} ViewStmt; +---- + +解析时设置 `readOnly` 标志： + +[source,c] +---- +n->readOnly = ($10 == READ_ONLY_OPTION); +n->withCheckOption = n->readOnly ? NO_CHECK_OPTION : $10; +---- + +需要注意的是，`WITH READ ONLY` 与 `WITH CHECK OPTION` 是互斥的，不能同时使用。 + +=== 关系选项系统 + +==== read_only 关系选项 + +在 `reloptions.c` 文件中定义了 `read_only` 视图关系选项： + +[source,c] +---- +/* reloptions.c */ +{ + {"read_only", + "Prevents INSERT, UPDATE, and DELETE on this view (Oracle compatibility)", + RELOPT_KIND_VIEW, + AccessExclusiveLock}, + false +}, + +/* view_reloptions() 函数中 */ +{"read_only", RELOPT_TYPE_BOOL, + offsetof(ViewOptions, read_only)}, +---- + +==== ViewOptions 结构体扩展 + +在 `rel.h` 文件中，`ViewOptions` 结构体新增 `read_only` 字段： + +[source,c] +---- +typedef struct ViewOptions { + // ... 其他字段 + bool read_only; /* WITH READ ONLY (Oracle compat) */ +} ViewOptions; +---- + +==== RelationIsReadOnlyView 宏 + +通过 `RelationIsReadOnlyView` 宏可以快速判断视图是否为只读： + +[source,c] +---- +#define RelationIsReadOnlyView(relation) \ + (AssertMacro(relation->rd_rel->relkind == RELKIND_VIEW), \ + (relation)->rd_options && \ + ((ViewOptions *) (relation)->rd_options)->read_only) +---- + +=== 视图创建处理 + +在 `view.c` 文件的 `DefineView` 函数中处理 `WITH READ ONLY` 选项： + +[source,c] +---- +/* DefineView() 中处理 WITH READ ONLY */ +if (stmt->readOnly) +{ + /* 设置 read_only 关系选项 */ + options = transformViewOptions(RelationGetRelid(newRelationView), + stmt->options, true); + setRelOptions(newRelationView, options, true, AccessExclusiveLock); +} +---- + +同时在 `compile_force_view_internal` 函数中也支持 `WITH READ ONLY`，确保 FORCE VIEW 可以正常使用只读属性。 + +=== DML 执行拦截 + +==== 执行层拦截 + +在 `execMain.c` 文件的 `CheckValidResultRel()` 函数中，对只读视图的 DML 操作进行拦截： + +[source,c] +---- +/* execMain.c */ +if (RelationIsReadOnlyView(resultRel)) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot modify view \"%s\"", + RelationGetRelationName(resultRel)), + errhint("The view is defined as read-only."))); +---- + +当执行 INSERT、UPDATE、DELETE 或 MERGE 语句时，如果目标视图被标记为只读，将抛出错误。 + +==== 重写层拦截 + +在 `rewriteHandler.c` 文件的 `rewriteTargetView()` 函数中也添加了相同的检查： + +[source,c] +---- +/* rewriteHandler.c */ +if (RelationIsReadOnlyView(view)) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot modify view \"%s\"", + RelationGetRelationName(view)), + errhint("The view is defined as read-only."))); +---- + +这确保了在查询重写层面也能阻止对只读视图的修改操作。 + +=== pg_dump 支持 + +在 `pg_dump.c` 文件中添加了对 `WITH READ ONLY` 属性的导出和恢复支持： + +[source,c] +---- +/* pg_dump.h - TableInfo 结构体新增字段 */ +typedef struct _tableInfo { + // ... 其他字段 + bool readOnly; /* WITH READ ONLY (Oracle compat) */ + // ... 其他字段 +} _tableInfo; + +/* pg_dump.c - 提取 read_only 选项 */ +if (strcmp(options[i].defname, "read_only") == 0) + info->readOnly = defGetBoolean(options[i].def); +---- + +导出时，`dumpTableSchema()` 和 `dumpRule()` 函数会在适当位置输出 `WITH READ ONLY` 子句，确保只读属性在数据库迁移过程中得以保留。 + +== 使用示例 + +=== 创建只读视图 + +[source,sql] +---- +-- 创建带 WITH READ ONLY 的视图 +CREATE VIEW emp_view AS +SELECT * FROM employees +WITH READ ONLY; + +-- 在只读视图上执行 DML 将被拒绝 +INSERT INTO emp_view VALUES (1, 'John', 5000); +-- ERROR: cannot modify view "emp_view" +-- HINT: The view is defined as read-only. + +UPDATE emp_view SET salary = 6000 WHERE id = 1; +-- ERROR: cannot modify view "emp_view" +-- HINT: The view is defined as read-only. + +DELETE FROM emp_view WHERE id = 1; +-- ERROR: cannot modify view "emp_view" +-- HINT: The view is defined as read-only. +---- + +=== 创建或替换只读视图 + +[source,sql] +---- +-- 使用 OR REPLACE 保留只读属性 +CREATE OR REPLACE VIEW emp_view AS +SELECT id, name, department FROM employees +WITH READ ONLY; +---- + +=== FORCE VIEW 与只读属性结合 + +[source,sql] +---- +-- 创建 FORCE VIEW 并设置为只读 +CREATE OR REPLACE FORCE VIEW emp_force_view AS +SELECT * FROM employees +WITH READ ONLY; +---- + +=== 与 WITH CHECK OPTION 的互斥性 + +[source,sql] +---- +-- 错误: WITH READ ONLY 和 WITH CHECK OPTION 不能同时使用 +CREATE VIEW emp_view AS +SELECT * FROM employees +WITH CHECK OPTION +WITH READ ONLY; +-- ERROR: READ ONLY and CHECK OPTION are mutually exclusive +---- diff --git a/CN/modules/ROOT/pages/master/6.3.3.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/rowid.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.3.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/rowid.adoc diff --git a/CN/modules/ROOT/pages/master/6.3.10.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/sys_guid_function.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.10.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/sys_guid_function.adoc diff --git a/CN/modules/ROOT/pages/master/6.3.4.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/type_rowtype.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.3.4.adoc rename to CN/modules/ROOT/pages/master/compatibility_features_design/type_rowtype.adoc diff --git a/CN/modules/ROOT/pages/master/compatibility_features_design/with_function_procedure_impl.adoc b/CN/modules/ROOT/pages/master/compatibility_features_design/with_function_procedure_impl.adoc new file mode 100644 index 0000000..9ddf5df --- /dev/null +++ b/CN/modules/ROOT/pages/master/compatibility_features_design/with_function_procedure_impl.adoc @@ -0,0 +1,676 @@ +:sectnums: +:sectnumlevels: 5 + += WITH FUNCTION/PROCEDURE 实现说明 + +== 目的 + +本文档详细说明 IvorySQL 中 `WITH FUNCTION` 和 `WITH PROCEDURE` 功能的实现原理。该功能允许在 SQL 的 WITH 子句（公共表表达式，CTE）中直接定义 PL/SQL 函数和过程，实现 Oracle 的 Subquery Factoring with PL/SQL Declarations 特性。 + +== 实现说明 + +=== 系统分层架构 + +WITH 函数/过程的实现贯穿四个层次： + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Layer 1: Oracle 解析器 (ora_gram.y + liboracle_parser.c) │ +│ ─ 扩展 with_clause 语法，允许 plsql_declarations │ +│ ─ 复用 OraBody_FUNC 词法机制将函数体捕获为 Sconst │ +│ ─ 输出: WithClause { plsql_defs: [InlineFunctionDef...], ctes: [...]}│ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ Layer 2: 语义分析 (parse_cte.c + parse_func.c) │ +│ ─ transformWithClause() 处理 InlineFunctionDef 节点 │ +│ ─ 解析函数签名，注册到 ParseState.p_with_func_list │ +│ ─ p_subprocfunc_hook 拦截对 WITH 函数的调用解析 │ +│ ─ FuncExpr.function_from = FUNC_FROM_WITH_CLAUSE ('w') │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ Layer 3: 规划器 (planner.c / createplan.c) │ +│ ─ 将 Query.withFuncDefs 复制到 PlannedStmt.withFuncDefs │ +│ ─ 无额外代价模型变更 │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ Layer 4: 执行器 (execExpr.c + pl_handler.c) │ +│ ─ ExecInitFunc() 识别 FUNC_FROM_WITH_CLAUSE │ +│ ─ 按需编译 WITH 函数体 │ +│ ─ 编译结果缓存在 EState.es_with_func_container │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +=== 语法与解析 + +==== 语法规则扩展 + +在 `ora_gram.y` 文件中扩展 `with_clause` 语法规则： + +[source,yacc] +---- +with_clause: + WITH plsql_declarations cte_list + { + WithClause *n = makeNode(WithClause); + n->plsql_defs = $2; + n->ctes = $3; + n->recursive = false; + n->location = @1; + $$ = n; + } + | WITH plsql_declarations + { + WithClause *n = makeNode(WithClause); + n->plsql_defs = $2; + n->ctes = NIL; + n->recursive = false; + n->location = @1; + $$ = n; + } + | WITH cte_list /* 现有语法保持不变 */ + | WITH_LA cte_list /* 现有 */ + | WITH RECURSIVE cte_list /* 现有 */ + ; + +plsql_declarations: + plsql_declaration + { $$ = list_make1($1); } + | plsql_declarations plsql_declaration + { $$ = lappend($1, $2); } + ; + +plsql_declaration: + FUNCTION ora_func_name opt_ora_func_args_with_defaults + RETURN func_return + ora_func_is_or_as Sconst ';' + { + InlineFunctionDef *n = makeNode(InlineFunctionDef); + n->funcname = strVal(llast($2)); + n->args = $3; + n->rettype = $5; + n->is_proc = false; + n->src = $7; + n->location = @2; + $$ = (Node *) n; + } + | PROCEDURE ora_func_name opt_procedure_args_with_defaults + ora_func_is_or_as Sconst ';' + { + InlineFunctionDef *n = makeNode(InlineFunctionDef); + n->funcname = strVal(llast($2)); + n->args = $3; + n->rettype = NULL; + n->is_proc = true; + n->src = $5; + n->location = @2; + $$ = (Node *) n; + } + ; +---- + +==== OraBody_FUNC 机制复用 + +Oracle 解析器已有 `set_oracle_plsql_body(OraBody_FUNC)` 机制，能将 IS/AS 后的 PL/SQL 体完整捕获为 Sconst 字符串。WITH FUNCTION 直接复用此机制，无需改动词法分析器。 + +[source,c] +---- +/* ora_gram.y */ +ora_func_is_or_as: + IS { set_oracle_plsql_body(yyscanner, OraBody_FUNC); } + | AS { set_oracle_plsql_body(yyscanner, OraBody_FUNC); } + ; +---- + +词法分析器（`liboracle_parser.c:439-652`）进入体捕获模式，跟踪 BEGIN/END/FUNCTION/PROCEDURE 嵌套深度，直到最外层 END 处停止，将全部文本作为单个 SCONST 返回。 + +==== 语法歧义处理 + +`FUNCTION` 和 `PROCEDURE` 都是 `unreserved_keyword`，可作 CTE 名。LALR(1) 状态表基于"第二个 token"自动消歧： + +| `WITH FUNCTION|PROCEDURE` 后第一个 token | LALR(1) 动作 | 走向 | +|--------------------------------|-------------|------| +| **IDENT** / 普通标识符 | shift（进入 `ora_func_name`） | plsql_declaration | +| **`AS`** | reduce | CTE 无列名列表 | +| **`(`** | reduce | CTE 有列名列表 | + +=== AST 节点设计 + +==== InlineFunctionDef 节点 + +新增 AST 节点表示 WITH 子句中的内嵌函数/过程定义： + +[source,c] +---- +/* parsenodes.h */ +typedef struct InlineFunctionDef +{ + NodeTag type; /* T_InlineFunctionDef */ + char *funcname; /* 函数/过程名（非限定名） */ + List *args; /* FunctionParameter 节点列表 */ + TypeName *rettype; /* 返回类型（过程为 NULL） */ + bool is_proc; /* true = 过程，false = 函数 */ + char *src; /* 函数体原始文本（IS/AS...END 全文） */ + ParseLoc location; /* 在原始 SQL 中的位置 */ +} InlineFunctionDef; +---- + +==== WithClause 扩展 + +[source,c] +---- +/* parsenodes.h */ +typedef struct WithClause +{ + NodeTag type; + List *plsql_defs; /* 新增：InlineFunctionDef 节点列表（ORA 模式） */ + List *ctes; /* 现有：CommonTableExpr 节点列表 */ + bool recursive; /* true = WITH RECURSIVE */ + ParseLoc location; +} WithClause; +---- + +==== FuncExpr 扩展 + +在 `primnodes.h` 中新增函数来源标识： + +[source,c] +---- +/* primnodes.h */ +#define FUNC_FROM_WITH_CLAUSE 'w' /* WITH 子句内嵌函数 */ + +/* 更新宏，将 'w' 纳入非 pg_proc 函数 */ +#define FUNC_EXPR_FROM_PG_PROC(function_from) \ + (function_from != FUNC_FROM_SUBPROCFUNC && \ + function_from != FUNC_FROM_PACKAGE && \ + function_from != FUNC_FROM_PACKGE_INITBODY && \ + function_from != FUNC_FROM_WITH_CLAUSE) +---- + +==== WithFuncEntry 结构体 + +ParseState 中新增轻量结构体用于存储函数签名： + +[source,c] +---- +/* parse_node.h */ +typedef struct WithFuncEntry +{ + char *funcname; + List *argtypes; /* Oid 列表 */ + Oid rettype; + bool is_proc; + int funcindex; /* 对应 FuncExpr.funcid */ + InlineFunctionDef *def; /* 指向原始定义节点 */ +} WithFuncEntry; +---- + +==== WithFuncContainer 运行时容器 + +[source,c] +---- +/* pl_handler.h */ +typedef struct WithFuncContainer +{ + int nfuncs; /* WITH 函数数量 */ + PLiSQL_subproc_function **funcs; /* 编译后的函数数组 */ + MemoryContext mcxt; /* 本容器的内存上下文 */ +} WithFuncContainer; +---- + +=== 语义分析 + +==== transformWithClause 扩展 + +在 `parse_cte.c` 中新增 `transformWithFuncDefs()` 函数： + +[source,c] +---- +/* parse_with_plsql.c */ +static void +transformWithFuncDefs(ParseState *pstate, List *plsql_defs) +{ + int funcindex = 0; + ListCell *lc; + + foreach(lc, plsql_defs) + { + InlineFunctionDef *ifd = (InlineFunctionDef *) lfirst(lc); + WithFuncEntry *entry = palloc(sizeof(WithFuncEntry)); + + /* 解析参数类型 */ + entry->argtypes = resolveWithFuncArgTypes(pstate, ifd->args); + + /* 解析返回类型 */ + entry->rettype = ifd->rettype ? + typenameTypeId(pstate, ifd->rettype) : InvalidOid; + + entry->funcname = ifd->funcname; + entry->is_proc = ifd->is_proc; + entry->funcindex = funcindex++; + entry->def = ifd; + + /* 检查重复定义 */ + checkDuplicateWithFunc(pstate->p_with_func_list, entry); + + pstate->p_with_func_list = lappend(pstate->p_with_func_list, entry); + } + + /* 安装函数查找钩子 */ + if (pstate->p_subprocfunc_hook == NULL) + pstate->p_subprocfunc_hook = withFuncLookupHook; +} +---- + +==== 函数调用解析钩子 + +`withFuncLookupHook` 拦截对 WITH 子句内嵌函数的调用： + +[source,c] +---- +/* parse_with_plsql.c */ +static FuncDetailCode +withFuncLookupHook(ParseState *pstate, List *funcname, + List **fargs, List *fargnames, int nargs, + Oid *argtypes, bool expand_variadic, bool expand_defaults, + bool proc_call, Oid *funcid, Oid *rettype, bool *retset, + int *nvargs, Oid *vatype, Oid **true_typeids, + List **argdefaults, void **pfunc) +{ + char *fname; + ListCell *lc; + + if (list_length(funcname) != 1) + return FUNCDETAIL_NOTFOUND; + + fname = strVal(linitial(funcname)); + + foreach(lc, pstate->p_with_func_list) + { + WithFuncEntry *entry = (WithFuncEntry *) lfirst(lc); + + if (strcmp(entry->funcname, fname) != 0) + continue; + + /* 检查参数数量和类型匹配 */ + if (!matchWithFuncArgs(entry, nargs, argtypes, fargnames, + true_typeids, argdefaults)) + continue; + + *funcid = (Oid) entry->funcindex; + *rettype = entry->rettype; + *retset = false; + *nvargs = 0; + *vatype = InvalidOid; + *pfunc = NULL; + + return entry->is_proc ? FUNCDETAIL_PROCEDURE : FUNCDETAIL_NORMAL; + } + + return FUNCDETAIL_NOTFOUND; +} +---- + +=== 执行器设计 + +==== ExecInitFunc 扩展 + +在 `execExpr.c` 中识别 `FUNC_FROM_WITH_CLAUSE`： + +[source,c] +---- +/* execExpr.c */ +if (funcexpr->function_from == FUNC_FROM_WITH_CLAUSE) +{ + scratch->d.func.finfo = palloc0(sizeof(FmgrInfo)); + scratch->d.func.fcinfo_data = palloc0(SizeForFunctionCallInfo(nargs)); + flinfo = scratch->d.func.finfo; + fcinfo = scratch->d.func.fcinfo_data; + + /* 使用专用调度函数 */ + flinfo->fn_addr = plisql_with_func_call_handler; + flinfo->fn_oid = funcid; + + /* fn_extra 存储 EState 指针 */ + flinfo->fn_extra = state->parent->state; + + fmgr_info_set_expr((Node *) node, flinfo); + InitFunctionCallInfoData(*fcinfo, flinfo, nargs, inputcollid, NULL, NULL); + scratch->d.func.fn_addr = flinfo->fn_addr; + scratch->d.func.nargs = nargs; + return; +} +---- + +==== 运行时调度函数 + +[source,c] +---- +/* pl_handler.c */ +Datum +plisql_with_func_call_handler(PG_FUNCTION_ARGS) +{ + EState *estate = (EState *) fcinfo->flinfo->fn_extra; + int funcindex = (int) fcinfo->flinfo->fn_oid; + WithFuncContainer *container; + + /* 懒加载：首次调用时编译所有 WITH 函数 */ + if (estate->es_with_func_container == NULL) + estate->es_with_func_container = buildWithFuncContainer(estate); + + container = estate->es_with_func_container; + + Assert(funcindex >= 0 && funcindex < container->nfuncs); + subprocfunc = container->funcs[funcindex]; + + return execWithFunction(subprocfunc, fcinfo); +} +---- + +==== WithFuncContainer 编译 + +[source,c] +---- +/* pl_handler.c */ +WithFuncContainer * +buildWithFuncContainer(EState *estate) +{ + PlannedStmt *pstmt = estate->es_plannedstmt; + MemoryContext oldcxt = MemoryContextSwitchTo(estate->es_query_cxt); + WithFuncContainer *container; + int nfuncs, i; + ListCell *lc; + + nfuncs = list_length(pstmt->withFuncDefs); + container = palloc0(sizeof(WithFuncContainer)); + container->nfuncs = nfuncs; + container->funcs = palloc0(nfuncs * sizeof(PLiSQL_subproc_function *)); + container->mcxt = CurrentMemoryContext; + + /* 编译阶段：建立共享编译上下文，以支持函数间互调 */ + plisql_push_subproc_func(); + plisql_start_subproc_func(); + + /* Pass 1：注册所有函数签名 */ + i = 0; + foreach(lc, pstmt->withFuncDefs) + { + InlineFunctionDef *ifd = (InlineFunctionDef *) lfirst(lc); + List *argitems = buildArgItemsFromFuncParams(ifd->args); + PLiSQL_type *rettype = ifd->rettype ? + buildPLiSQLType(ifd->rettype) : NULL; + + PLiSQL_subproc_function *subprocfunc = + plisql_build_subproc_function(ifd->funcname, argitems, + rettype, ifd->location); + subprocfunc->is_proc = ifd->is_proc; + subprocfunc->src = ifd->src; + + container->funcs[i++] = subprocfunc; + } + + /* Pass 2：编译每个函数体 */ + i = 0; + foreach(lc, pstmt->withFuncDefs) + { + InlineFunctionDef *ifd = (InlineFunctionDef *) lfirst(lc); + PLiSQL_subproc_function *subprocfunc = container->funcs[i++]; + + PLiSQL_stmt_block *action = + compileWithFuncBody(ifd->src, subprocfunc); + + plisql_set_subprocfunc_action(subprocfunc, action); + } + + plisql_pop_subproc_func(); + + MemoryContextSwitchTo(oldcxt); + return container; +} +---- + +=== 内存生命周期 + +[source] +---- +SQL 文本解析阶段 + ├── InlineFunctionDef 节点 + │ 生命周期：与解析树相同（ParseState.p_mem_cxt） + └── WithFuncEntry 列表 + 生命周期：与 ParseState 相同 + +查询分析阶段 + └── Query.withFuncDefs（InlineFunctionDef 列表，复制指针） + 生命周期：与 Query 相同 + +规划阶段 + └── PlannedStmt.withFuncDefs（同上） + 生命周期：与 PlannedStmt 相同（可被 plancache 持有） + +执行阶段 + ├── EState.es_with_func_container（WithFuncContainer） + │ 内存上下文：estate->es_query_cxt + │ 生命周期：绑定到 EState，ExecutorEnd() 时释放 + ├── PLiSQL_subproc_function 数组 + │ 生命周期：与 WithFuncContainer 相同 + └── PLiSQL_function（编译结果） + 生命周期：查询执行结束时自动释放 +---- + +=== 关键设计原则 + +1. **复用 OraBody_FUNC**：Oracle 解析器已有 `set_oracle_plsql_body(OraBody_FUNC)` 机制，WITH FUNCTION 直接复用，无需改动词法分析器。 + +2. **不写入系统目录**：WITH 函数的编译结果存储在 EState 本地，不写入 `pg_proc`，不产生 WAL，事务结束自动释放。 + +3. **延迟编译**：函数体在执行器初始化阶段（`ExecInitFunc`）才真正编译，解析和规划阶段只处理签名，避免了在 Plan 节点中存储指针的生命周期问题。 + +4. **与 PG_PARSER 隔离**：所有新逻辑受 `compatible_db == ORA_PARSER` 守卫，PostgreSQL 原有解析器路径不受影响。 + +5. **两阶段编译**：通过 two-pass 设计支持函数间互调（Pass 1 注册签名，Pass 2 编译函数体）。 + +=== 函数间互调 + +得益于 two-pass 编译设计，**WITH 子句内函数互调对声明顺序没有限制**：A 可以调用在 A 之后定义的 B，反之亦然。无需 Oracle PL/SQL 风格的显式前向声明。 + +[source,sql] +---- +-- 函数互调示例 +WITH + FUNCTION mul2(n NUMBER) RETURN NUMBER AS BEGIN RETURN n*2; END; + FUNCTION add1(n NUMBER) RETURN NUMBER AS BEGIN RETURN n+1; END; +SELECT mul2(add1(3)) FROM dual; -- add1 先定义但 mul2 可以调用它 +---- + +=== EXPLAIN 输出 + +==== 基本输出 + +[source,sql] +---- +EXPLAIN WITH + FUNCTION add_one(n NUMBER) RETURN NUMBER AS BEGIN RETURN n + 1; END; +SELECT add_one(5) FROM dual; +---- +输出包含：`WITH Function: add_one(number) RETURN number` + +==== VERBOSE 模式 + +[source,sql] +---- +EXPLAIN (VERBOSE ON) WITH + FUNCTION double(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n * 2; END; +SELECT double(3) FROM dual; +---- +输出包含：`Body: BEGIN RETURN n * 2; END` + +=== 错误处理 + +==== 重复定义检查 + +[source,sql] +---- +WITH + FUNCTION dup(n NUMBER) RETURN NUMBER AS BEGIN RETURN n; END; + FUNCTION dup(n NUMBER) RETURN NUMBER AS BEGIN RETURN n * 2; END; +SELECT dup(1) FROM dual; +-- ERROR: WITH clause function "dup" is defined more than once +---- + +==== PG_PARSER 模式拒绝 + +[source,sql] +---- +SET compatible_db = PG_PARSER; +WITH FUNCTION foo(n NUMBER) RETURN NUMBER AS BEGIN RETURN n; END; +SELECT foo(1); +-- ERROR: syntax error at or near "FUNCTION" +---- + +==== 函数体编译错误 + +[source,sql] +---- +WITH + FUNCTION broken(n NUMBER) RETURN NUMBER AS + BEGIN + RETRUN n; -- 拼写错误 + END; +SELECT broken(1) FROM dual; +-- ERROR: syntax error at or near "RETRUN" +---- +错误上下文：`while compiling WITH FUNCTION "broken_body"` + +==== 表函数用法拒绝 + +[source,sql] +---- +WITH + FUNCTION get_rows(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n; END; +SELECT * FROM get_rows(5); +-- ERROR: WITH clause function cannot be used as a table function +---- + +==== 限定名拒绝 + +[source,sql] +---- +WITH + FUNCTION public.qual_func(n NUMBER) RETURN NUMBER IS + BEGIN RETURN n; END; +SELECT qual_func(1) FROM dual; +-- ERROR: qualified name is not allowed in WITH FUNCTION declaration +---- + +=== 扩展的文件清单 + +==== 新增文件 + +| 文件 | 说明 | +|------|------| +| `src/backend/oracle_parser/ora_with_function.c` | WITH 函数运行时逻辑 | +| `src/backend/parser/parse_with_plsql.c` | `transformWithFuncDefs`、`withFuncLookupHook` | +| `src/include/oracle_parser/ora_with_function.h` | 头文件：`WithFuncEntry`、`WithFuncContainer` | +| `src/oracle_test/regress/sql/with_function.sql` | 回归测试（32 用例） | + +==== 修改现有文件 + +| 文件 | 修改内容 | +|------|----------| +| `src/include/nodes/parsenodes.h` | 添加 `InlineFunctionDef`；扩展 `WithClause.plsql_defs` 和 `Query.withFuncDefs` | +| `src/include/nodes/plannodes.h` | 扩展 `PlannedStmt.withFuncDefs` | +| `src/include/nodes/primnodes.h` | 添加 `FUNC_FROM_WITH_CLAUSE = 'w'` | +| `src/include/nodes/execnodes.h` | 扩展 `EState.es_with_func_container` | +| `src/include/parser/parse_node.h` | 扩展 `ParseState.p_with_func_list` | +| `src/backend/oracle_parser/ora_gram.y` | 扩展 `with_clause`、新增 `plsql_declarations`/`plsql_declaration` | +| `src/backend/parser/parse_cte.c` | `transformWithClause()` 调用 `transformWithFuncDefs` | +| `src/backend/parser/parse_func.c` | FuncExpr 标记逻辑 | +| `src/backend/parser/analyze.c` | 传递 `withFuncDefs` 到 Query 节点 | +| `src/backend/optimizer/plan/planner.c` | 传递 `withFuncDefs` 到 PlannedStmt | +| `src/backend/executor/execExpr.c` | `ExecInitFunc()` 处理 WITH 函数 | +| `src/pl/plisql/src/pl_handler.c` | `buildWithFuncContainer`、`plisql_with_func_call_handler` | +| `src/pl/plisql/src/pl_comp.c` | `plisql_parser_setup` 根据 flag gate `p_with_func_list` | +| `src/backend/commands/explain.c` | EXPLAIN 输出 `WITH Function:` / `WITH Procedure:` | +| `src/oracle_fe_utils/ora_psqlscan.l` | psql 客户端扫描器识别 `WITH FUNCTION/PROCEDURE` | + +=== 节点函数自动生成 + +PostgreSQL 16+ 引入了节点基础设施代码生成器（`src/backend/nodes/gen_node_support.pl`）。新增 `InlineFunctionDef` 节点后，构建系统自动重跑生成器，产出： + +| 自动生成的文件 | 包含内容 | +|--------------|---------| +| `copyfuncs.funcs.c` / `copyfuncs.switch.c` | `_copyInlineFunctionDef` | +| `equalfuncs.funcs.c` / `equalfuncs.switch.c` | `_equalInlineFunctionDef` | +| `outfuncs.funcs.c` / `outfuncs.switch.c` | `_outInlineFunctionDef` | +| `readfuncs.funcs.c` / `readfuncs.switch.c` | `_readInlineFunctionDef` | +| `nodetags.h` | `T_InlineFunctionDef = 498` | +| `queryjumblefuncs.funcs.c` | `_jumbleInlineFunctionDef` | + +== 使用示例 + +=== 最简单的内嵌函数 + +[source,sql] +---- +WITH + FUNCTION double_it(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n * 2; END; +SELECT double_it(5) FROM dual; +-- 输出：10 +---- + +=== 函数与 CTE 混合 + +[source,sql] +---- +WITH + FUNCTION tax(amt NUMBER) RETURN NUMBER AS + BEGIN RETURN amt * 0.1; END; + orders AS (SELECT 100 AS amount) +SELECT amount, tax(amount) FROM orders; +-- 输出：100 | 10 +---- + +=== 多个内嵌函数 + +[source,sql] +---- +WITH + FUNCTION add1(n NUMBER) RETURN NUMBER AS BEGIN RETURN n+1; END; + FUNCTION mul2(n NUMBER) RETURN NUMBER AS BEGIN RETURN n*2; END; +SELECT mul2(add1(3)) FROM dual; +-- 输出：8 +---- + +=== 递归函数 + +[source,sql] +---- +WITH + FUNCTION factorial(n NUMBER) RETURN NUMBER AS + BEGIN + IF n <= 1 THEN RETURN 1; END IF; + RETURN n * factorial(n-1); + END; +SELECT factorial(5) FROM dual; +-- 输出：120 +---- + +=== 与 DML 集成 + +[source,sql] +---- +-- INSERT 中使用内嵌函数 +WITH + FUNCTION get_bonus(sal NUMBER) RETURN NUMBER AS + BEGIN RETURN sal * 1.2; END; +INSERT INTO emp_bonus (empno, bonus) +SELECT empno, get_bonus(sal) FROM emp WHERE deptno = 10; + +---- + +NOTE: Oracle 不允许 WITH FUNCTION 位于 UPDATE、DELETE、MERGE 之前；IvorySQL 遵循相同限制，此类用法报 `ERRCODE_FEATURE_NOT_SUPPORTED` 错误。 diff --git a/CN/modules/ROOT/pages/master/4.6.4.adoc b/CN/modules/ROOT/pages/master/containerization/docker_podman_deployment.adoc similarity index 90% rename from CN/modules/ROOT/pages/master/4.6.4.adoc rename to CN/modules/ROOT/pages/master/containerization/docker_podman_deployment.adoc index 2cbbd0b..57d618f 100644 --- a/CN/modules/ROOT/pages/master/4.6.4.adoc +++ b/CN/modules/ROOT/pages/master/containerization/docker_podman_deployment.adoc @@ -4,6 +4,11 @@ = Docker & Podman 部署IvorySQL +[TIP] +==== +除 Docker Hub 外，IvorySQL 社区还为国内用户提供 Harbor 镜像服务。将本章节中的镜像地址前缀替换为 `registry.highgo.com/` 即可使用，例如 `registry.highgo.com/ivorysql/ivorysql:5.0-ubi8`。 +==== + == docker方式运行 ** 从Docker Hub上获取IvorySQL镜像 diff --git a/CN/modules/ROOT/pages/master/4.6.3.adoc b/CN/modules/ROOT/pages/master/containerization/docker_swarm_compose_deployment.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/4.6.3.adoc rename to CN/modules/ROOT/pages/master/containerization/docker_swarm_compose_deployment.adoc diff --git a/CN/modules/ROOT/pages/master/4.6.1.adoc b/CN/modules/ROOT/pages/master/containerization/k8s_deployment.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/4.6.1.adoc rename to CN/modules/ROOT/pages/master/containerization/k8s_deployment.adoc diff --git a/CN/modules/ROOT/pages/master/4.6.2.adoc b/CN/modules/ROOT/pages/master/containerization/operator_deployment.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/4.6.2.adoc rename to CN/modules/ROOT/pages/master/containerization/operator_deployment.adoc diff --git a/CN/modules/ROOT/pages/master/8.2.adoc b/CN/modules/ROOT/pages/master/contribution/asciidoc_syntax_reference.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/8.2.adoc rename to CN/modules/ROOT/pages/master/contribution/asciidoc_syntax_reference.adoc diff --git a/CN/modules/ROOT/pages/master/8.1.adoc b/CN/modules/ROOT/pages/master/contribution/community_contribution_guide.adoc similarity index 90% rename from CN/modules/ROOT/pages/master/8.1.adoc rename to CN/modules/ROOT/pages/master/contribution/community_contribution_guide.adoc index 7e8c285..fadfeb7 100644 --- a/CN/modules/ROOT/pages/master/8.1.adoc +++ b/CN/modules/ROOT/pages/master/contribution/community_contribution_guide.adoc @@ -79,7 +79,7 @@ IvorySQL源码托管在github: 。 * 有新的功能建议或使用体验改进？ -如果您准备向社区上报 Bug 或者提交需求，请在 IvorySQL 社区对应的仓库上提交 Issue，并参考Issue xref:./33.adoc[提交指南]。 +如果您准备向社区上报 Bug 或者提交需求，请在 IvorySQL 社区对应的仓库上提交 Issue，并参考Issue xref:master/contribution/issue_submission_guide.adoc[提交指南]。 ==== 参与社区讨论 @@ -93,6 +93,10 @@ IvorySQL源码托管在github: 。 我们欢迎代码、文档、测试等各类贡献。 +==== 文档贡献 + +如果您新增或更新某个插件适配文档，请同步更新 xref:master/ecosystem_components/ecosystem_overview.adoc[ecosystem_overview.adoc] 概述页面中的插件表格，至少补充或更新插件名称、版本、功能描述、适用场景以及详情页链接，确保概述页与插件详情页信息一致。 + ==== 签署CLA 在提交代码或文档贡献之前，为了确保代码合法合规，个人或企业贡献者需要签署贡献者许可协议(CLA)。签署CLA是IvorySQL社区接受贡献的必要条件，以确保您的贡献被合法分发。请根据下列链接下载CLA进行签署并将签署后的CLA发送至 cla@ivorysql.org。 @@ -125,6 +129,8 @@ IvorySQL源码托管在github: 。 ==== 开发与提交Pull Request 对于提交一个PR应该保持一个功能，或者一个bug提交一次。禁止多个功能一次提交。 +提交 GitHub PR 之前，请先创建或认领对应的 Issue。所有 PR 都必须有至少一个对应的 Issue，用于关联需求、缺陷、讨论、评审和合并结果；没有对应 Issue 的 PR 不符合社区贡献流程。 + ===== Fork仓库 前往项目主页，点击Fork按钮，将IvorySQL项目Fork到您自己的GitHub账户中。 @@ -154,6 +160,8 @@ leave a comment 对该提交功能进行比较详细的描述 ``` +请在 PR 标题或描述中显式关联对应 Issue 编号，例如 `Fixes #123` 或 `Refs #123`。 + 点击Create pull request 按钮即可提交。 === 维护者 diff --git a/CN/modules/ROOT/pages/master/10.adoc b/CN/modules/ROOT/pages/master/contribution/faq.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/10.adoc rename to CN/modules/ROOT/pages/master/contribution/faq.adoc diff --git a/CN/modules/ROOT/pages/master/23.adoc b/CN/modules/ROOT/pages/master/contribution/issue_submission_guide.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/23.adoc rename to CN/modules/ROOT/pages/master/contribution/issue_submission_guide.adoc diff --git a/CN/modules/ROOT/pages/master/cpu_arch_adp.adoc b/CN/modules/ROOT/pages/master/cpu_os_adaptation/cpu_architecture_adaptation.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/cpu_arch_adp.adoc rename to CN/modules/ROOT/pages/master/cpu_os_adaptation/cpu_architecture_adaptation.adoc diff --git a/CN/modules/ROOT/pages/master/os_arch_adp.adoc b/CN/modules/ROOT/pages/master/cpu_os_adaptation/os_architecture_adaptation.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/os_arch_adp.adoc rename to CN/modules/ROOT/pages/master/cpu_os_adaptation/os_architecture_adaptation.adoc diff --git a/CN/modules/ROOT/pages/master/4.3.adoc b/CN/modules/ROOT/pages/master/developer_guide.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/4.3.adoc rename to CN/modules/ROOT/pages/master/developer_guide.adoc diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/age.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/age.adoc new file mode 100644 index 0000000..e9c34e6 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/age.adoc @@ -0,0 +1,189 @@ +:sectnums: +:sectnumlevels: 5 + += Apache AGE + +== 概述 + +Apache AGE 是一个 PostgreSQL 扩展，为关系型数据库提供图数据库处理能力。AGE 代表 *Adaptive Graph Engine*（自适应图引擎），它将图数据库的功能引入 PostgreSQL，允许用户在同一数据库中同时使用关系模型和图模型。 + +Apache AGE 是 Apache 软件基金会的顶级项目，完全兼容 openCypher 查询语言（Neo4j 使用的图查询语言）。 + +*核心特性：* + +[cols="1,2"] +|=== +| 特性 | 描述 + +| openCypher 支持 +| 完整支持 openCypher 查询语言，行业标准的图查询语言 + +| 混合数据库 +| 在同一数据库中同时使用关系型和图数据模型 + +| ACID 事务 +| 继承 PostgreSQL 的完整 ACID 事务支持 + +| SQL 集成 +| 可将 Cypher 图查询与 SQL 查询无缝集成 + +| 属性图模型 +| 支持带有属性的顶点和边的属性图模型 + +| 图遍历 +| 高效的图遍历和模式匹配能力 + +| 免费开源 +| Apache 2.0 许可证，完全开源 +|=== + +== 应用场景 + +* 社交网络分析（好友关系、关注关系、影响力分析） +* 知识图谱构建与推理 +* 欺诈检测（金融交易网络分析） +* 推荐系统（基于关系链的推荐） +* 网络与 IT 基础设施管理 +* 路径规划与物流优化 +* 访问控制与权限管理 + +== 安装 + +[TIP] +源码测试安装环境为 Ubuntu 24.04。 + +=== 依赖 + +[literal] +---- +# Ubuntu / Debian +sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libxslt-dev libssl-dev libxml2-utils xsltproc ccache libssl-dev pkg-config + +# 安装 AGE 依赖 +sudo apt install python3 python3-pip python3-dev +pip3 install antlr4-runtime4 +---- + +=== 从源码安装 + +[literal] +---- +# 下载 Apache AGE 1.7.0 源码包 +wget https://github.com/apache/age/releases/download/PG18%2Fv1.7.0-rc0/apache-age-1.7.0-src.tar.gz + +# 解压 +tar -xzf apache-age-1.7.0-src.tar.gz +cd apache-age-1.7.0-src + +# 编译安装 +make install + +# 或指定 IvorySQL 安装路径 +make install PG_CONFIG=/usr/ivory-5/bin/pg_config +---- + +=== 验证安装 + +[literal] +---- +# 检查 AGE 扩展 +ls /usr/ivory-5/lib/age--*.so +---- + +== 配置 + +=== 修改 IvorySQL 配置 + +编辑 `postgresql.conf` 或 `ivorysql.conf`： + +[literal] +---- +# 预加载 AGE 扩展（推荐） +shared_preload_libraries = 'age' + +# 或者在数据库级别加载（不需要重启） +# shared_preload_libraries = '' +---- + +[literal] +---- +# 重启 IvorySQL 使配置生效 +# 使用 systemd +sudo systemctl restart ivorysql-5 + +# 或手动重启 +pg_ctl restart -D /usr/ivory-5/data +---- + +=== 创建扩展 + +连接到 IvorySQL 并创建 AGE 扩展： + +[literal] +---- +# 连接到数据库 +psql -U postgres -d postgres + +# 创建 AGE 扩展 +CREATE EXTENSION age; + +# 验证安装 +SELECT * FROM pg_extension WHERE extname = 'age'; +---- + +== 使用 + +[literal] +---- +要创建图，使用位于 ag_catalog 命名空间中的 create_graph 函数。 + +SELECT create_graph('graph_name'); + +要创建带有标签和属性的单个顶点，使用 CREATE 子句。 + +SELECT * +FROM cypher('graph_name', $$ + CREATE (:label {property:"Node A"}) +$$) as (v agtype); + +SELECT * +FROM cypher('graph_name', $$ + CREATE (:label {property:"Node B"}) +$$) as (v agtype); + +要在两个节点之间创建边并设置其属性： + +SELECT * +FROM cypher('graph_name', $$ + MATCH (a:label), (b:label) + WHERE a.property = 'Node A' AND b.property = 'Node B' + CREATE (a)-[e:RELTYPE {property:a.property + '<->' + b.property}]->(b) + RETURN e +$$) as (e agtype); + +查询连接的节点： + +SELECT * from cypher('graph_name', $$ + MATCH (V)-[R]-(V2) + RETURN V,R,V2 +$$) as (V agtype, R agtype, V2 agtype); + +---- + +== 管理命令 + +[literal] +---- +-- 列出所有图 +SELECT * FROM ag_graph; + +-- 删除图（删除所有相关顶点和边） +SELECT drop_graph('graph_name', true); + +-- 查看图统计信息 +SELECT + graph_name, + (SELECT count(*) FROM ag_vertex WHERE graph_id = ag_graph.graph_id) AS vertex_count, + (SELECT count(*) FROM ag_edge WHERE graph_id = ag_graph.graph_id) AS edge_count +FROM ag_graph; +---- diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc new file mode 100644 index 0000000..074ce67 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc @@ -0,0 +1,48 @@ +:sectnums: +:sectnumlevels: 5 + + +[discrete] +== IvorySQL生态插件适配列表 + +IvorySQL 作为一款兼容 Oracle 且基于 PostgreSQL 的高级开源数据库，具备强大的扩展能力，支持丰富的生态系统插件。这些插件可以帮助用户在不同场景下增强数据库功能，包括地理信息处理、向量检索、全文搜索、数据定义提取和路径规划等。以下是当前 IvorySQL 官方兼容和支持的主要插件列表： + + +[cols="1,2,1,3,3"] +|==== +|*序号*|*插件名称*|*版本*|*功能描述*|*适用场景* +| 1 | xref:master/ecosystem_components/postgis.adoc[postgis] | 3.5.4 | 为 IvorySQL 提供地理空间数据支持，包括空间索引、空间函数和地理对象存储 | 地理信息系统（GIS）、地图服务、位置数据分析 +| 2 | xref:master/ecosystem_components/pgvector.adoc[pgvector] | 0.8.1 | 支持向量相似性搜索，可用于存储和检索高维向量数据| AI 应用、图像检索、推荐系统、语义搜索 +| 3 | xref:master/ecosystem_components/pgddl.adoc[pgddl (DDL Extractor)] | 0.31 | 提取数据库中的 DDL（数据定义语言）语句，便于版本管理和迁移 | 数据库版本控制、CI/CD 集成、结构比对与同步 +| 4 | xref:master/ecosystem_components/pg_cron.adoc[pg_cron]​ | 1.6.0 | 提供数据库内部的定时任务调度功能，支持定期执行SQL语句 | 数据清理、定期统计、自动化维护任务 +| 5 | xref:master/ecosystem_components/pgsql_http.adoc[pgsql-http]​ | 1.7.0 | 允许在SQL中发起HTTP请求，与外部Web服务进行交互 | 数据采集、API集成、微服务调用 +| 6 | xref:master/ecosystem_components/plpgsql_check.adoc[plpgsql_check] | 2.8 | 提供PL/pgSQL代码的静态分析功能，可在开发阶段发现潜在错误 | 存储过程开发、代码质量检查、调试优化 +| 7 | xref:master/ecosystem_components/pgroonga.adoc[pgroonga] | 4.0.4 | 提供​非英语语言全文搜索功能，满足高性能应用的需求 | 中日韩等语言的全文搜索功能 +| 8 | xref:master/ecosystem_components/pgaudit.adoc[pgaudit] | 18.0 | 提供细粒度的审计功能，记录数据库操作日志，便于安全审计和合规性检查 | 数据库安全审计、合规性检查、审计报告生成 +| 9 | xref:master/ecosystem_components/pgrouting.adoc[pgrouting] | 3.8.0 | 提供地理空间数据的路由计算功能，支持多种算法和数据格式 | 地理空间分析、路径规划、物流优化 +| 10 | xref:master/ecosystem_components/system_stats.adoc[system_stats] | 3.2 | 提供用于访问系统级统计信息的函数 | 系统监控 +| 11 | xref:master/ecosystem_components/wal2json.adoc[wal2json] | 2.6 | 用于 PostgreSQL 逻辑解码的输出插件，为每个事务生成一个JSON对象 | 变更数据捕获（CDC）、实时数据同步、事件驱动微服务、缓存刷新及跨库数据同步等 +| 12 | xref:master/ecosystem_components/pg_stat_monitor.adoc[pg_stat_monitor] | 2.3.1 | 收集性能统计数据，并通过统一视图和直方图形式直观展示查询性能指标。 | 性能监控 +| 13 | xref:master/ecosystem_components/pg_ai_query.adoc[pg_ai_query] | 0.1.1 | AI驱动的自然语言转SQL扩展，支持多种大语言模型 | AI辅助查询、自然语言数据库交互 +| 14 | xref:master/ecosystem_components/pg_partman.adoc[pg_partman] | 5.2 | 辅助管理原生分区表，自动创建、维护、清理分区子表 | 海量数据存储管理 +| 15 | xref:master/ecosystem_components/pgbouncer.adoc[pgbouncer] | 1.25.1 | PostgreSQL 轻量级连接池，提供连接复用和高效连接管理 | 高并发连接管理、连接复用、减少数据库连接开销 +| 16 | xref:master/ecosystem_components/pg_curl.adoc[pg_curl] | 2.4 | 基于 libcurl 的网络传输扩展，支持 HTTP/HTTPS、FTP、SMTP、IMAP 等二十余种协议，可在 SQL 中完成各类网络数据传输操作 | REST API 集成、邮件发送、文件传输、外部系统通知 +| 17 | xref:master/ecosystem_components/pg_textsearch.adoc[pg_textsearch] | 0.6.1 | 提供全文检索能力，支持文本分词、索引构建与高效全文查询 | 文档检索与内容搜索 +| 18 | xref:master/ecosystem_components/pg_hint_plan.adoc[pg_hint_plan] | PG18 | 通过SQL注释中的hints控制执行计划，在不修改SQL逻辑的情况下优化查询性能 | 查询性能优化、执行计划调优、数据库性能分析 +| 19 | xref:master/ecosystem_components/redis_fdw.adoc[redis_fdw] | PG18 | 将 Redis 数据映射为 PostgreSQL 外部表，支持通过标准 SELECT/INSERT/UPDATE/DELETE 语句读写 Redis | 统一 SQL 查询、轻量级数据同步、透明化缓存读写及跨库数据分析 +| 20 | xref:master/ecosystem_components/age.adoc[Apache AGE] | 1.7.0 | 为 IvorySQL 提供图数据库处理能力，支持 openCypher 查询语言，实现关系型与图数据库的混合使用 | 社交网络分析、知识图谱、欺诈检测、推荐系统、路径规划 +| 21 | xref:master/ecosystem_components/pg_show_plans.adoc[pg_show_plans] | 2.1 | 显示当前所有正在运行的 SQL 语句的执行计划，支持 TEXT、JSON、YAML 等多种输出格式 | 查询性能诊断、实时执行计划监控、慢查询分析 +| 22 | xref:master/ecosystem_components/pg_bulkload.adoc[pg_bulkload] | 3.1.23 | 为IvorySQL提供高速数据载入工具，可以跳过PG的共享缓存直接将数据导入表中 | 海量数据初始加载，历史数据归档，跨库迁移 +| 23 | xref:master/ecosystem_components/pg_bigm.adoc[pg_bigm] | 1.2 | 为 IvorySQL 提供二元分词全文检索能力，适配中日韩文本，快速实现模糊检索与相似度查询 | 中日韩文内容、商品、地址类文字搜索场景 +| 24 | xref:master/ecosystem_components/pg_profile.adoc[pg_profile] | 4.11 | 收集数据库资源密集型活动的统计，基于两次采样点的差量分析生成历史负载报告，帮助定位性能瓶颈 | 数据库性能分析 +| 25 | xref:master/ecosystem_components/pg_repack.adoc[pg_repack] | 1.5.3 | 在几乎不阻塞业务读写的情况下在线重整表和索引、消除存储膨胀，效果类似 VACUUM FULL | 表与索引在线重建、存储空间回收 +| 26 | xref:master/ecosystem_components/pgdog.adoc[PgDog] | 0.1.45 | 专为 PostgreSQL 设计的高性能、开源集群中间件（代理工具），采用 Rust 语言编写 | 自动分片、连接池和负载均衡功能 +| 27 | xref:master/ecosystem_components/pg_readonly.adoc[pg_readonly] | 1.0.5 | 可将 PostgreSQL 数据库集群设置为只读 | 系统调试、灾难恢复 +| 28 | xref:master/ecosystem_components/zhparser.adoc[zhparser] | master branch | 用于中文全文搜索的PostgreSQL插件，基于SCWS（即：简易中文分词系统）实现了一个中文解析器 | 搜索引擎、关键字提取 +| 29 | xref:master/ecosystem_components/pgbackrest.adoc[pgBackRest] | 2.58.0 | 可靠的 PostgreSQL 备份和恢复解决方案 | 容灾备份、大库备份、异地/多层容灾 +| 30 | xref:master/ecosystem_components/set_user.adoc[set_user] | REL4_2_0 | PostgreSQL 安全审计扩展，可控角色切换，支持白名单、强制审计、拦截高危操作 | 可控角色切换、权限管理、审计日志 +|==== + +这些插件均经过 IvorySQL 团队的测试和适配，确保在 IvorySQL 环境下稳定运行。用户可以根据业务需求选择合适的插件，进一步提升数据库系统的能力和灵活性。 + +我们也将持续扩展和丰富 IvorySQL 的插件生态，欢迎社区开发者提交新的插件适配建议或代码贡献。如需了解更多每个插件的详细使用方法和最新兼容版本，请参阅各插件对应的文档章节。 diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_ai_query.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_ai_query.adoc new file mode 100644 index 0000000..0ea5eb2 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_ai_query.adoc @@ -0,0 +1,320 @@ +:sectnums: +:sectnumlevels: 5 + += pg_ai_query + +== 概述 + +pg_ai_query 是一个用于 IvorySQL/PostgreSQL 的 AI 驱动自然语言转 SQL 扩展。它利用大语言模型（LLM）将用户的自然语言描述直接转换为可执行的 SQL 查询语句，支持 OpenAI、Anthropic Claude 和 Google Gemini 等多种 AI 模型。 + +项目地址： + +开源协议：Apache-2.0 + +主要特性： + +* **自然语言转 SQL**：将文本语言描述转换为有效的 SQL 查询 +* **多模型支持**：支持 gpt-4o-mini、gpt-4o、gpt-5、claude-3-haiku-20240307、claude-sonnet-4-5-20250929、claude-4.5-opus 等大模型 +* **安全保护**：阻止访问 `information_schema` 和 `pg_catalog` 系统表 +* **作用域限制**：仅对用户表进行操作 +* **可配置限制**：内置行数限制强制执行 +* **API 密钥安全**：安全处理 API 凭证 + +== 快速开始 +下面以v0.1.1为例进行演示。 + +=== 安装 + +*依赖要求* + +* PostgreSQL 14+ with development headers +* CMake 3.16+ +* C++20 compatible compiler +* API key from OpenAI, Anthropic, or Google (Gemini) + +*安装依赖* + +[source,bash] +---- +sudo apt-get install libcurl4-openssl-dev +---- + +*编译安装 IvorySQL* + +如需从源码编译 IvorySQL，可参考以下配置： + +[source,bash] +---- +./configure \ +--prefix=$PWD/inst \ +--enable-cassert \ +--enable-debug \ +--enable-tap-tests \ +--enable-rpath \ +--enable-nls \ +--enable-injection-points \ +--with-tcl \ +--with-python \ +--with-gssapi \ +--with-pam \ +--with-ldap \ +--with-openssl \ +--with-libedit-preferred \ +--with-uuid=e2fs \ +--with-ossp-uuid \ +--with-libxml \ +--with-libxslt \ +--with-perl \ +--with-icu \ +--with-libnuma +---- + +*编译安装 pg_ai_query* + +[source,bash] +---- +git clone --recurse-submodules https://github.com/benodiwal/pg_ai_query.git +cd pg_ai_query +mkdir build && cd build +export PATH="$HOME/works/repo/ivorysql/IvorySQL/inst/bin:$PATH" +cmake .. -DCMAKE_INSTALL_PREFIX=$HOME/works/repo/ivorysql/IvorySQL/inst +make && sudo make install +---- + +*创建扩展* + +[source,sql] +---- +CREATE EXTENSION pg_ai_query; +---- + +=== 配置 + +在 home 目录下创建 `~/.pg_ai.config` 配置文件： + +[source,ini] +---- +[general] +log_level = "INFO" +enable_logging = false + +[query] +enforce_limit = true +default_limit = 1000 + +[response] +show_explanation = true +show_warnings = true +show_suggested_visualization = false +use_formatted_response = false + +[anthropic] +# Your Anthropic API key (if using Claude) +api_key = "******" + +# Default model to use (options: claude-sonnet-4-5-20250929) +default_model = "claude-sonnet-4-5-20250929" + +# Custom API endpoint (optional) - for Anthropic-compatible APIs +api_endpoint = "https://open.bigmodel.cn/api/anthropic" + +[prompts] +# Use file paths to read custom prompts +system_prompt = /home/highgo/.pg_ai.prompts +explain_system_prompt = /home/highgo/.pg_ai.explain.prompts +---- + +更多示例请参考： + +== 使用示例 + +=== 基本用法 + +[source,sql] +---- +SELECT generate_query('找出所有的用户'); +---- + +输出示例： + +---- +[INFO] Text generation successful - model: claude-sonnet-4-5-20250929, response_id: msg_20260209135507cc16362d5d324ccd + + generate_query +-------------------------------------------------------- + SELECT * FROM public.users LIMIT 1000; ++ + -- Explanation: + -- Retrieves all columns and rows from the users table. ++ + -- Warning: INFO: Applied LIMIT 1000 to prevent large result sets. Remove LIMIT if you need all data. ++ + -- Note: Row limit was automatically applied to this query for safety +(1 row) +---- + +执行查询： + +[source,sql] +---- +SELECT * FROM public.users LIMIT 1000; +---- + +输出： + +---- + id | name | email | age | created_at | city +----+---------------+-------------------+-----+----------------------------+--------------- + 1 | Alice Johnson | alice@example.com | 28 | 2026-02-04 15:47:55.208111 | New York + 2 | Bob Smith | bob@example.com | 35 | 2026-02-04 15:47:55.208111 | San Francisco + 3 | Carol Davis | carol@example.com | 31 | 2026-02-04 15:47:55.208111 | Chicago + 4 | David Wilson | david@example.com | 27 | 2026-02-04 15:47:55.208111 | Seattle + 5 | Eva Brown | eva@example.com | 33 | 2026-02-04 15:47:55.208111 | Boston +(5 rows) +---- + +=== generate_query 示例 + +*生成测试数据* + +[source,sql] +---- +SELECT generate_query('生成100条user数据，插入到users'); +---- + +输出： + +---- +[INFO] Text generation successful - model: claude-sonnet-4-5-20250929, response_id: msg_2026021114092101601c5650864a2d + + generate_query +-------------------------------------------------------------------------------------------------------- + INSERT INTO public.users (name, email, age, city, status) + SELECT 'User_' || generate_series AS name, + 'user' || generate_series || '@example.com' AS email, + (18 + (generate_series % 50)) AS age, + (ARRAY['Beijing','Shanghai','Guangzhou','Shenzhen','Hangzhou'])[1 + (generate_series % 5)] AS city, + 'active' AS status + FROM generate_series(1, 100); ++ + -- Explanation: + -- 生成100条模拟用户数据并插入到users表中。数据包括自动生成的姓名、唯一邮箱、随机年龄(18-67岁)、随机城市和默认状态。 ++ + -- Warnings: + -- 1. INFO: 依赖users表的id列有DEFAULT自增设置，未手动插入id。 + -- 2. INFO: 使用generate_series函数生成序列数据，这是PostgreSQL/IvorySQL的特性。 + -- 3. WARN: 确保在运行前users表为空或id序列不冲突，否则可能重复插入。 + -- 4. WARN: 邮箱格式为简单模拟，实际环境中可能需要更复杂的逻辑或去重检查。 +(1 row) +---- + +*不区分大小写查询* + +[source,sql] +---- +SELECT generate_query('show users from beijing, beijing is non-Case insensitive'); +---- + +输出： + +---- +[INFO] Text generation successful - model: claude-sonnet-4-5-20250929, response_id: msg_20260211142845878f5f1a5a2f44a7 + + generate_query +----------------------------------------- + SELECT id, name, email, age, created_at, city, status + FROM public.users + WHERE LOWER(city) = LOWER('beijing') LIMIT 100; ++ + -- Explanation: + -- Selects all user details for users located in Beijing, performing a case-insensitive match on the city column. ++ + -- Warnings: + -- 1. INFO: Using LOWER() on both sides ensures case-insensitive matching but may prevent the database from using a standard index on the city column if one exists. + -- 2. INFO: Row limit of 100 applied to prevent large result sets. ++ + -- Note: Row limit was automatically applied to this query for safety +(1 row) +---- + +=== explain_query 示例 + +[source,sql] +---- +SELECT explain_query('SELECT * FROM orders WHERE user_id = 12'); +---- + +输出： + +---- +[INFO] Text generation successful - model: claude-sonnet-4-5-20250929, response_id: msg_20260211175909d47a6871bcca4897 + + explain_query +-------------------------------------------------------------------------------------------------------------- + 1. 查询概述 ++ + - 该查询旨在从 orders 表中检索 user_id 等于 12 的所有记录（SELECT *）。 + - 这是一个典型的根据特定字段（user_id）筛选数据的查询。 ++ + 2. 性能摘要 ++ + - 总执行时间: 0.021 毫秒 + - 规划时间: 0.430 毫秒 + - 总成本: 18.12 + - 返回行数: 0 行 (Actual Rows: 0) + - 扫描行数: 0 行 (Rows Removed by Filter: 0) ++ + 3. 执行计划分析 ++ + - 关键步骤: 顺序扫描 + - 数据库对 orders 表执行了全表扫描操作。 + - 计划器预计会找到 3 行数据，但实际执行返回了 0 行。 + - 过滤条件: orders.user_id = 12，这意味着数据库必须读取表中的每一行来检查这个条件。 ++ + 4. 性能问题 ++ + - 全表扫描风险: 虽然目前表的数据量很小（执行时间仅为 0.021ms），但使用了 Seq Scan（顺序扫描）意味着数据库没有使用索引。如果 orders 表随着时间推移增长到包含数百万行数据，这种查询方式将变得极其缓慢（高 I/O 消耗）。 + - 缺失索引: 计划显示没有使用任何索引来定位 user_id = 12 的行，这表明在 user_id 列上可能缺少必要的 B-Tree 索引。 ++ + 5. 优化建议 ++ + - 主要建议: 在 user_id 列上创建索引以避免全表扫描。这将把查询从 O(N)（扫描所有行）转变为 O(log N)（索引查找）。 + - SQL 优化示例: ++ + CREATE INDEX idx_orders_user_id ON orders(user_id); ++ + 6. 索引建议 ++ + - 推荐索引: 在 orders 表的 user_id 列上创建 B-Tree 索引。 + - 理由: 查询条件基于 user_id 的等值比较 (=)。创建索引后，IvorySQL (PostgreSQL) 将能够利用索引快速定位数据，显著减少查询时间和资源消耗，特别是在数据量大的情况下。 +(1 row) +---- + +== 最佳实践 + +=== 提示词（Prompt）编写建议 + +* **使用英语**：虽然 AI 支持多种语言，但英语效果最佳 +* **了解数据库结构**：对数据库结构理解越深入，生成的查询越准确 +* **迭代优化**：从宽泛的开始，然后逐步添加细节以改进结果 +* **明确指定**：如果知道特定的表或列，请在提示中提及，这有助于 AI 生成精确的查询 + +=== 错误处理示例 + +当查询中引用的表不存在时，系统会返回错误信息： + +[source,sql] +---- +SELECT generate_query('列出所有的商品和价格'); +---- + +错误输出： + +---- +[INFO] Text generation successful - model: claude-sonnet-4-5-20250929, response_id: msg_20260209135642777cbc5c82ca4a85 + +ERROR: Query generation failed: Cannot generate query. Referenced table(s) for 'products' or 'goods' do not exist in the database. Available tables: public.orders, public.student_scores, public.users, sys.dual +---- + +在这种情况下，AI 会告知可用的表列表，帮助用户调整查询。 + diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_bigm.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_bigm.adoc new file mode 100644 index 0000000..71587c9 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_bigm.adoc @@ -0,0 +1,65 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_bigm + +== 概述 +pg_bigm 为 IvorySQL 提供二元分词全文检索能力，适配中日韩文本，快速实现模糊检索与相似度查询。 + +== 安装 +IvorySQL的安装包里已经集成了 pg_bigm 插件，如果使用安装包安装的IvorySQL，通常不需要再手动安装 pg_bigm 即可使用。其它安装方式可以参考下面的源码安装步骤。 + +[TIP] +源码安装环境为 Ubuntu 24.04(x86_64)，环境中已经安装了IvorySQL5及以上版本，安装路径为/usr/ivory-5 + +=== 源码安装 +从https://github.com/pgbigm/pg_bigm/releases/tag/v1.2-20250903 下载 pg_bigm v1.2 代码。 + +** 安装 pg_bigm +[literal] +---- +# 源代码解压缩后进入其目录 +cd pg_bigm-1.2-20250903 + +# 编译代码，设置PG_CONFIG环境变量值为pg_config路径，eg：/usr/ivory-5/bin/pg_config +make USE_PGXS=1 PG_CONFIG=/usr/ivory-5/bin/pg_config +sudo make USE_PGXS=1 PG_CONFIG=/usr/ivory-5/bin/pg_config install +---- + +== 创建Extension +psql 连接到数据库，执行如下命令： +[literal] +---- +ivorysql=# CREATE EXTENSION pg_bigm; +CREATE EXTENSION + +ivorysql=# SELECT * FROM pg_available_extensions WHERE name = 'pg_bigm'; + name | default_version | installed_version | comment +---------+-----------------+-------------------+------------------------------------------------------------------ + pg_bigm | 1.2 | 1.2 | text similarity measurement and index searching based on bigrams +(1 row) +---- + +== 使用 +[literal] +---- +# 首先创建索引 +CREATE TABLE pg_tools (tool text, description text); + +INSERT INTO pg_tools VALUES ('pg_hint_plan', 'Tool that allows a user to specify an optimizer HINT to PostgreSQL'); +INSERT INTO pg_tools VALUES ('pg_dbms_stats', 'Tool that allows a user to stabilize planner statistics in PostgreSQL'); +INSERT INTO pg_tools VALUES ('pg_bigm', 'Tool that provides 2-gram full text search capability in PostgreSQL'); +INSERT INTO pg_tools VALUES ('pg_trgm', 'Tool that provides 3-gram full text search capability in PostgreSQL'); + +CREATE INDEX pg_tools_idx ON pg_tools USING gin (description gin_bigm_ops); + +# 执行全文检索 +SELECT * FROM pg_tools WHERE description LIKE '%search%'; + + tool | description +---------+--------------------------------------------------------------------- + pg_bigm | Tool that provides 2-gram full text search capability in PostgreSQL + pg_trgm | Tool that provides 3-gram full text search capability in PostgreSQL +(2 rows) +---- \ No newline at end of file diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_bulkload.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_bulkload.adoc new file mode 100644 index 0000000..be48455 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_bulkload.adoc @@ -0,0 +1,81 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_bulkload + +== 概述 +pg_bulkload 是 PostgreSQL 高性能批量数据导入插件，可绕过常规读写机制大幅提升文本数据入库效率，支持数据过滤、错误容错、并行加载与断点恢复，适用于海量历史数据迁移、数仓批量同步、离线日志入库等大批量数据加载场景，相比原生 COPY 命令导入速度显著提升。 + +== 安装 +IvorySQL的安装包里已经集成了pg_bulkload插件，如果使用安装包安装的IvorySQL，通常不需要再手动安装pg_bulkload即可使用。其它安装方式可以参考下面的源码安装步骤。 + +[TIP] +源码安装环境为 Ubuntu 24.04(x86_64)，环境中已经安装了IvorySQL5及以上版本，安装路径为/usr/ivory-5 + +=== 源码安装 +从https://github.com/ossc-db/pg_bulkload/releases/tag/VERSION3_1_23 下载 pg_bulkload v3.1.23代码。 + +** 安装依赖 +[literal] +---- +sudo apt install gawk +---- + +** 安装pg_bulkload +[literal] +---- +# 源代码解压缩后进入其目录 +cd pg_bulkload-VERSION3_1_23 + +# 修改 Makefile 以适应IvorySQl的 non-PIE 静态库 + + LDFLAGS+=-Wl,--build-id ++ ++# Workaround for non-PIE static libraries (e.g., IvorySQL's libpgcommon.a) ++# Some distributions build libpgcommon.a without -fPIE, causing link failures ++# when the system defaults to PIE executables. ++ifdef DISABLE_PIE ++CFLAGS+=-no-pie ++LDFLAGS+=-no-pie ++endif + +# 编译代码，设置PG_CONFIG环境变量值为pg_config路径，eg：/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config clean +make PG_CONFIG=/usr/ivory-5/bin/pg_config DISABLE_PIE=1 +sudo make PG_CONFIG=/usr/ivory-5/bin/pg_config +---- + +== 创建Extension +psql 连接到数据库，执行如下命令： +[literal] +---- +ivorysql=# CREATE extension pg_bulkload; +CREATE EXTENSION + +ivorysql=# SELECT * FROM pg_available_extensions WHERE name = 'pg_bulkload'; + name | default_version | installed_version | comment +-------------+-----------------+-------------------+----------------------------------------------------------------- + pg_bulkload | 3.1.23 | 3.1.23 | pg_bulkload is a high speed data loading utility for PostgreSQL +(1 row) +---- + +== 使用 +[literal] +---- +ivorysql=# create database testdb; +CREATE DATABASE + +ivorysql=# \c testdb +You are now connected to database "testdb" as user "highgo". + +testdb=# create table tb_asher (id int,name text); +CREATE TABLE +testdb=# \q + +# 模拟CSV 文件 +seq 100000| awk '{print $0"|asher"}' > bulk_asher.txt + +# 将bulk_asher.txt里的数据加载到testdb 库下的 tb_asher表中 +/usr/ivory-5/bin/pg_bulkload -i ./bulk_asher.txt -O tb_asher -l ./tb_asher_output.log -P ./tb_asher_bad.txt -o "TYPE=CSV" -o "DELIMITER=|" -d testdb -U highgo -h 127.0.0.1 +---- \ No newline at end of file diff --git a/CN/modules/ROOT/pages/master/5.4.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_cron.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/5.4.adoc rename to CN/modules/ROOT/pages/master/ecosystem_components/pg_cron.adoc diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_curl.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_curl.adoc new file mode 100644 index 0000000..3964c9f --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_curl.adoc @@ -0,0 +1,198 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_curl + +== 概述 + +pg_curl 是一个基于 libcurl 的 PostgreSQL 扩展，它将 cURL 的强大数据传输能力直接引入数据库内部，允许用户通过 SQL 函数调用完成各种网络协议的数据传输操作，无需借助外部程序或中间件。 + +与只支持 HTTP 的轻量级扩展不同，pg_curl 全面封装了 libcurl 的 easy interface API，支持 DICT、FILE、FTP、FTPS、GOPHER、HTTP、HTTPS、IMAP、IMAPS、LDAP、LDAPS、MQTT、POP3、POP3S、RTMP、RTMPS、RTSP、SCP、SFTP、SMB、SMBS、SMTP、SMTPS、TELNET、TFTP、WS 和 WSS 等二十余种协议，提供了极为丰富的网络交互能力。 + +典型应用场景包括：在触发器或存储过程中向外部系统发送 HTTP 通知；直接从数据库发送邮件；通过 FTP/SFTP 上传或下载文件；调用第三方 REST API 并将结果写回数据库表；以及在数据处理流程中与 MQTT 消息代理、LDAP 目录等各类后端服务进行集成。 + +pg_curl 采用 MIT 许可证，项目地址：https://github.com/RekGRpth/pg_curl 。 + +== 功能特点 + +* *多协议支持*：基于 libcurl，支持 HTTP/HTTPS、FTP/FTPS、SMTP/SMTPS、IMAP、POP3、SCP、SFTP、MQTT、LDAP 等二十余种协议，覆盖绝大多数网络集成场景。 +* *完整的 HTTP 方法支持*：支持 GET、POST（URL 编码、JSON、multipart/form-data）、PUT、DELETE、PATCH 等标准 HTTP 方法，并可通过 `curl_easy_setopt_customrequest` 指定任意自定义方法。 +* *灵活的请求构造*：提供细粒度的 `curl_easy_setopt_*` 系列函数，可精确控制请求头、认证信息、超时、代理、TLS 证书、Cookie 等各项参数。 +* *多种认证方式*：支持 Basic Auth（用户名/密码）、Bearer Token、NTLM、Digest、OAuth 等多种认证机制。 +* *文件传输*：支持通过 FTP/FTPS/SFTP/SCP 上传（`curl_easy_setopt_readdata`）和下载文件，上传数据以 `bytea` 类型直接传入。 +* *邮件发送*：通过 SMTP/SMTPS 直接在数据库内发送邮件，支持设置发件人、收件人、邮件头及 MIME 正文。 +* *响应解析*：提供 `curl_easy_getinfo_data_in()`、`curl_easy_getinfo_header_in()` 等函数获取响应体和响应头，并可通过正则表达式将头信息解析为键值表。 +* *错误处理*：`curl_easy_getinfo_errcode()`、`curl_easy_getinfo_errdesc()` 及 `curl_easy_getinfo_errbuf()` 提供完整的错误码和错误描述，便于调试。 +* *超时与中断*：通过向 libcurl 注册进度回调并周期性检查 PostgreSQL 的取消标志（`QueryCancelPending`），支持被 `statement_timeout`、`pg_cancel_backend()` 等方式中断，防止长时间请求阻塞连接。 +* *URL 编码工具*：内置 `curl_easy_escape()` 和 `curl_easy_unescape()` 函数，方便在 SQL 中进行 URL 编码和解码。 + +== 安装 + +[TIP] +以下示例环境为 Ubuntu 24.04（x86_64），已安装 IvorySQL 5 及以上版本，安装路径为 `/usr/local/ivorysql/ivorysql-5`。 + +=== 安装依赖 + +pg_curl 依赖系统的 libcurl 开发库，安装前需确认已安装该依赖： + +[source,shell] +---- +sudo apt install libcurl4-openssl-dev +---- + +=== 源码编译安装 + +从 https://github.com/RekGRpth/pg_curl 获取源码，然后执行编译安装： + +[source,shell] +---- +git clone https://github.com/RekGRpth/pg_curl.git +cd pg_curl +# 确保 pg_config 可访问，或通过 PG_CONFIG 显式指定 +PG_CONFIG=/usr/local/ivorysql/ivorysql-5/bin/pg_config make +PG_CONFIG=/usr/local/ivorysql/ivorysql-5/bin/pg_config make install +---- + +安装成功后，`pg_curl.so` 及对应的 SQL 脚本会被放置到 IvorySQL 的扩展目录中。 + +== 创建 Extension 并验证版本 + +使用 psql 连接到数据库（PG 模式端口 5432 或 Oracle 模式端口 1521 均可），执行如下命令： + +[source,sql] +---- +CREATE EXTENSION pg_curl; + +SELECT name, default_version, installed_version, comment + FROM pg_available_extensions + WHERE name = 'pg_curl'; +---- + +预期输出类似： + +[source,text] +---- + name | default_version | installed_version | comment +---------+-----------------+-------------------+---------------------------------------------------------------------------------------------------------------------------------------- + pg_curl | 2.4 | 2.4 | PostgreSQL cURL allows most curl actions, including data transfer with URL syntax via HTTP, HTTPS, FTP, FTPS, GOPHER, TFTP, SCP, ... +(1 row) +---- + +== 使用 + +pg_curl 采用"先配置、再执行、后取结果"的调用模式，核心流程如下： + +. 调用 `curl_easy_reset()` 初始化（或重置）当前 cURL 会话。 +. 调用各 `curl_easy_setopt_*`、`curl_header_append`、`curl_postfield_append` 等函数配置请求参数。 +. 调用 `curl_easy_perform()` 执行请求。 +. 调用 `curl_easy_getinfo_*` 系列函数获取响应结果或错误信息。 + +=== HTTP GET + +[source,sql] +---- +BEGIN; +SELECT curl_easy_reset(); +SELECT curl_easy_setopt_url('https://httpbin.org/get?'); +SELECT curl_url_append('key1', 'value1'); +SELECT curl_url_append('key2', 'hello world'); -- 自动 URL 编码 +SELECT curl_easy_perform(); +SELECT convert_from(curl_easy_getinfo_data_in(), 'utf-8'); +END; +---- + +=== HTTP POST（JSON） + +POST 表单（`curl_postfield_append`）和 multipart（`curl_mime_data`）的用法与此类似，替换数据设置函数即可。 + +[source,sql] +---- +BEGIN; +SELECT curl_easy_reset(); +SELECT curl_easy_setopt_postfields(convert_to('{"name":"IvorySQL"}', 'utf-8')); +SELECT curl_easy_setopt_url('https://httpbin.org/post'); +SELECT curl_header_append('Content-Type', 'application/json; charset=utf-8'); +SELECT curl_easy_perform(); +SELECT convert_from(curl_easy_getinfo_data_in(), 'utf-8'); +END; +---- + +=== 认证与请求头 + +Basic Auth 使用 `curl_easy_setopt_username` / `curl_easy_setopt_password`；Bearer Token 及其他自定义头均通过 `curl_header_append` 添加： + +[source,sql] +---- +BEGIN; +SELECT curl_easy_reset(); +SELECT curl_easy_setopt_url('https://httpbin.org/bearer'); +SELECT curl_header_append('Authorization', 'Bearer '); +SELECT curl_easy_perform(); +SELECT convert_from(curl_easy_getinfo_data_in(), 'utf-8')::jsonb; +END; +---- + +=== 错误处理与超时 + +`curl_easy_perform()` 成功返回后，可通过以下函数检查执行状态（`errcode` 为 0 表示成功）： + +[source,sql] +---- +BEGIN; +SELECT curl_easy_reset(); +SELECT curl_easy_setopt_url('https://httpbin.org/get'); +SELECT curl_easy_perform(); +SELECT + curl_easy_getinfo_errcode() AS errcode, + curl_easy_getinfo_errdesc() AS errdesc; +END; +---- + +pg_curl 通过 libcurl 进度回调检查 PostgreSQL 取消标志，`statement_timeout` 触发时请求会被立即中止并回滚当前事务： + +[source,sql] +---- +BEGIN; +SET LOCAL statement_timeout = '3s'; +SELECT curl_easy_reset(); +SELECT curl_easy_setopt_url('https://httpbin.org/delay/10'); +SELECT curl_easy_perform(); -- 超时后抛出 query_canceled，事务回滚 +END; +---- + +== IvorySQL 适配说明 + +=== 双端口兼容 + +IvorySQL 采用双端口架构，同时支持 PostgreSQL 模式（默认端口 *5432*）和 Oracle 兼容模式（默认端口 *1521*）： + +* 连接 *5432* 端口（PG 模式）：接受标准 PostgreSQL SQL 语法。 +* 连接 *1521* 端口（Oracle 模式）：接受 Oracle 兼容 SQL 语法。 + +pg_curl 在两种模式下均可正常使用，`CREATE EXTENSION pg_curl` 在 1521 和 5432 端口均能成功执行。 + +=== Oracle 模式注意事项 + +在 Oracle 模式（1521 端口）下使用 pg_curl 时，以下几点需要注意： + +* pg_curl 的所有函数均通过普通 `SELECT` 语句调用（如 `SELECT curl_easy_reset()`），在 Oracle 模式下同样有效；也可以使用 Oracle 惯用的 `SELECT curl_easy_reset() FROM DUAL` 写法。 +* `ivorysql.enable_emptystring_to_NULL` 参数会影响空字符串的处理行为。回归测试中已设置 `SET ivorysql.enable_emptystring_to_NULL = 'off'` 以确保空值参数（如空表单字段）按预期传递，生产环境中需根据实际需求配置该参数。 + +=== 在 PL/iSQL 中使用 + +pg_curl 可在 IvorySQL 的 PL/iSQL（Oracle 兼容存储过程语言）中调用，实现数据变更时自动推送外部通知。PL/iSQL 使用 Oracle 风格的 `IS` 关键字声明存储过程，调用返回值的函数时通过赋值给局部变量来丢弃结果： + +[source,sql] +---- +-- 在 PL/iSQL 存储过程中调用 pg_curl（Oracle 模式，IS 语法） +CREATE OR REPLACE PROCEDURE notify_external(p_payload IN VARCHAR2) IS + v_ok BOOLEAN; +BEGIN + v_ok := curl_easy_reset(); + v_ok := curl_easy_setopt_postfields(convert_to(p_payload, 'utf-8')); + v_ok := curl_easy_setopt_url('https://hooks.example.com/notify'); + v_ok := curl_header_append('Content-Type', 'application/json'); + v_ok := curl_easy_perform(); +END; +---- diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_hint_plan.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_hint_plan.adoc new file mode 100644 index 0000000..c04a24e --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_hint_plan.adoc @@ -0,0 +1,212 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_hint_plan + +== 概述 + +pg_hint_plan 是一个扩展，允许通过 SQL 注释中的 hints 来控制 PostgreSQL/IvorySQL 的执行计划。它能够在不修改 SQL 逻辑的情况下优化查询性能，在 PostgreSQL 和 Oracle 兼容模式下均可正常工作。 + + +== 安装 + +[TIP] +源码安装环境为 Ubuntu 24.04(x86_64)，环境中已经安装了IvorySQL 5及以上版本，安装路径为/usr/ivory-5 + +=== 源码安装 + +[literal] +---- +# 克隆 PG18 分支源码 + +git clone --branch PG18 https://github.com/ossc-db/pg_hint_plan.git +cd pg_hint_plan + +# 编译安装插件 +make PG_CONFIG=/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config install + +---- + +=== 修改数据库配置文件 + +修改 ivorysql.conf 文件，添加 pg_hint_plan 到 shared_preload_libraries： + +[literal] +---- +shared_preload_libraries = 'gb18030_2022, liboracle_parser, ivorysql_ora, pg_hint_plan' +---- + +重启数据库后配置生效。 + +=== 加载扩展 + +[literal] +---- +postgres=# LOAD 'pg_hint_plan'; +LOAD +---- + +== Hint 类型 + +pg_hint_plan 提供了多种类型的 hints 来控制查询执行计划的不同方面。 + +=== 扫描方法 Hints + +指定表扫描方法的 hints： + +* `SeqScan(table)` - 顺序扫描 +* `IndexScan(table[ index])` - 索引扫描，可指定索引名 +* `BitmapScan(table[ index])` - 位图扫描，可指定索引名 +* `TidScan(table)` - TID 扫描 + +否定形式的 hints（禁止使用某种扫描方法）： + +* `NoSeqScan(table)` +* `NoIndexScan(table)` +* `NoBitmapScan(table)` +* `NoTidScan(table)` + +=== 连接方法 Hints + +指定表连接方法的 hints： + +* `HashJoin(table table[ table...])` - 哈希连接 +* `NestedLoop(table table[ table...])` - 嵌套循环连接 +* `MergeJoin(table table[ table...])` - 合并连接 + +否定形式的 hints： + +* `NoHashJoin(table table)` +* `NoNestedLoop(table table)` +* `NoMergeJoin(table table)` + +=== 连接顺序 Hints + +* `Leading(table table[ table...])` - 指定连接顺序，表按顺序连接 + +=== 并行执行 Hints + +* `Parallel(table count[ workers])` - 设置并行工作进程数量 + * `count` - 是否使用并行（0 表示不使用，1 表示使用） + * `workers` - 并行工作进程数量（可选，默认值为规划器计算的值） + +=== 其他 Hints + +* `Set(enable_*)` - 设置 GUC 参数 +* `Rows(table table[ table...] correction)` - 修正行数估计值 + +== 使用示例 + +=== 基本用法示例 + +使用 HashJoin 和 SeqScan hints： + +[literal] +---- +postgres=# /*+ + HashJoin(a b) + SeqScan(a) +*/ +EXPLAIN SELECT * +FROM pgbench_branches b +JOIN pgbench_accounts a ON b.bid = a.bid +ORDER BY a.aid; +---- + +=== 复杂多 Hint 示例 + +组合使用多个 hints 来控制复杂查询的执行计划： + +[literal] +---- +postgres=# /*+ + NestedLoop(t1 t2) + IndexScan(t2 t2_pkey) + MergeJoin(t1 t2 t3) + Leading(t1 t2 t3) +*/ +EXPLAIN SELECT * FROM table1 t1 +JOIN table2 t2 ON t1.id = t2.id +JOIN table3 t3 ON t2.id = t3.id; +---- + +=== 并行执行示例 + +为多个表设置并行度和连接方法： + +[literal] +---- +postgres=# /*+ + Parallel(t1 3) + Parallel(t2 5) + HashJoin(t1 t2) +*/ +EXPLAIN SELECT * FROM large_table1 t1 +JOIN large_table2 t2 ON t1.id = t2.id; +---- + +=== 指定索引扫描示例 + +强制使用特定索引进行扫描： + +[literal] +---- +postgres=# /*+ + IndexScan(employees emp_name_idx) +*/ +EXPLAIN SELECT * FROM employees +WHERE last_name = 'SMITH'; +---- + +== Hint 表（可选功能） + +pg_hint_plan 提供了一个可选的 hint 表功能，可以在表中持久化存储 hints。 + +=== 创建 Hint 表 + +[literal] +---- +postgres=# CREATE TABLE hint_plan.hints ( + id serial PRIMARY KEY, + norm_query_string text NOT NULL, + application_name text, + hints text NOT NULL, + CONSTRAINT hint_plan_hints_norm_query_string_key UNIQUE (norm_query_string, application_name) +); +---- + +=== 插入 Hints + +[literal] +---- +postgres=# INSERT INTO hint_plan.hints (norm_query_string, application_name, hints) +VALUES ( + 'SELECT * FROM table1 WHERE id = ?;', + 'psql', + 'SeqScan(table1)' +); +---- + +通过 hint 表，可以为特定查询自动应用 hints，无需修改 SQL 语句。 + +== IvorySQL 兼容性 + +=== Oracle 兼容模式 + +pg_hint_plan 在 PostgreSQL 和 Oracle 兼容模式下均可正常工作。hint 语法保持一致，并与 IvorySQL 特有的数据类型（VARCHAR2、NUMBER 等）兼容。 + +=== Oracle 模式示例 + +[literal] +---- +-- 连接到 1521 端口（Oracle 模式） +postgres=# /*+ + IndexScan(employees emp_name_idx) +*/ +SELECT * FROM employees WHERE last_name = 'SMITH'; +---- + +在 Oracle 兼容模式下，hint 的使用方法与 PostgreSQL 模式完全相同，可以正常控制包含 IvorySQL 特有数据类型和语法的查询执行计划。 + diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_partman.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_partman.adoc new file mode 100644 index 0000000..db01f3c --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_partman.adoc @@ -0,0 +1,195 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_partman + +== 概述 +pg_partman 用于简化和自动化对 PostgreSQL 原生声明式分区表的管理，它提供了自动创建、维护和清理分区的功能，尤其是基于时间范围和整数范围分区。 + +PostgreSQL 支持的 3 种分区方式：范围分区、列表分区、哈希分区。其中，pg_partman 扩展不支持哈希分区，对列表分区支持有限，它主要用于按时间（如天、周、月）或数值区间（如自增 ID）进行的范围分区。 + +注意，pg_partman 作为 PostgreSQL 的扩展，专门设计用于对 PostgreSQL 的分区表进行管理，所以它不支持在 Oracle 兼容模式下运行。 + +项目地址： + +版本：5.2-STABLE + +开源协议：PostgreSQL License + +== 安装启用 + +=== 前提 +已安装 IvorySQL 5.0，且 pg_config 命令在命令行能成功执行。 + +=== 源码安装 +从 https://github.com/pgpartman/pg_partman 获取源码： +[literal] +---- +git clone https://github.com/pgpartman/pg_partman.git +cd pg_partman +git switch 5.2-STABLE + +make && sudo make install +---- + +=== 启用扩展 +[literal] +---- +-- 登录进入数据库 +CREATE SCHEMA partman; +CREATE EXTENSION pg_partman SCHEMA partman; +---- + +[NOTE] +pg_partman 安装启用后不会自动创建 SCHEMA，如果没有特别指定，PG 会将扩展对象默认安装到当前 search_path 中的第一个有效 SCHEMA（通常是 public）。推荐将其安装到独立 SCHEMA 中。 + +== 使用流程 +pg_partman 本身不负责创建分区主表，它的主要功能是在已有分区表的基础上，自动创建和维护分区子表。所以在使用 pg_partman 之前，要先手动创建一个已经启动分区机制的主表。主表创建完成后，先将主表注册给 pg_partman 管理，注册时会指定分区策略等。注册成功会在数据库中创建一个命名为 template_[schema]_[table_name] 的模板表，这个模板表不会存数据，是后面所有子分区的模板。注册完成后必须通过手动调用或通过脚本自动调用 run_maintenance()/run_maintenance_proc() 来创建分区、清理分区。 + +下面给出两个使用案例，有其他使用需求请参考 https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md[pg_partman 官方文档]。 + +== 案例一 基于时间字段分区 +场景假设：有一个日志表 logs，每天产生大量数据，希望按天自动创建子分区，并保留最近 30 天的数据，自动删除更旧的分区。 + +=== 手动创建分区主表 +[literal] +---- +CREATE TABLE public.logs ( + id BIGSERIAL, + log_time TIMESTAMPTZ NOT NULL DEFAULT now(), + message TEXT +) PARTITION BY RANGE (log_time); +---- + +=== 注册分区主表到 pg_partman +注册 logs 表，按天分区，从当前时间开始预创建未来 3 天，保留最近 30 天： +[literal] +---- +SELECT partman.create_parent( + p_parent_table => 'public.logs', -- 要管理的分区父表，注意要显式带 SCHEMA + p_control => 'log_time', -- 分区字段 + p_interval => '1 day', -- 按天分区 + p_type => 'range', -- 使用范围分区 + p_premake => 3, -- 预创建3个未来的分区 + p_start_partition => '2025-12-24', -- 起始分区 + p_default_table => false -- 是否创建默认分区 +); +---- + +上面 SQL 会从 2025-12-24 开始按天创建分区，并且预创建以当前服务器时间为基准的未来 3 个分区，分区命名以该分区表的起始时间为后缀。 + +[literal] +---- +INSERT INTO public.logs (log_time) VALUES ('2025-12-28 00:01:00'); +---- + +=== 注册清理（可选） +[literal] +---- +UPDATE partman.part_config +SET retention = '30 days' +WHERE parent_table = 'public.logs'; +---- + +=== 手动维护 +[literal] +---- +select partman.run_maintenance(); +---- +或 +[literal] +---- +CALL partman.run_maintenance_proc(); +---- + +[NOTE] +==== +调用 run_maintenance() 时，会根据分区策略自动预生成新的分区表，预生成的基准是表的分区字段的数据。如果最近一条插入数据的 log_time 字段是 20251228，则以此为基准预生成 logs_p20251229/logs_p20251230/logs_p20251231 三张分区子表。 + +另外，run_maintenance() 也会执行清理机制，清理机制以服务器时间为基准，上面的例子会将服务器当前时间 30 天之前的子表从主表记录上移除，但不会真正删除，以避免误删数据。此时 \d+ logs 查看分区主表的分区信息，看不到 30 天之前被移除的表，但 \dt 查看数据库所有的表，能看到被移除分区的子表依然存在。这些子表可以手动删除。 +==== + +=== 自动维护 +使用操作系统的 crontab： + +创建 shell 脚本 /usr/local/bin/partman_maintenance.sh： +[literal] +---- +cd /usr/local/bin +vi partman_maintenance.sh +---- +脚本内容： +[literal] +---- +#!/bin/bash +psql -U [db_username] -d [db_name] -c "CALL partman.run_maintenance_proc();" +---- + +赋予执行权限： +[literal] +---- +chmod +x partman_maintenance.sh +---- + +配置执行计划： +[literal] +---- +crontab -e +# 添加一行，配置每天凌晨一点执行一次 +0 1 * * * /usr/local/bin/partman_maintenance.sh +---- + +== 案例二 基于自增ID字段分区 +场景假设：创建一个订单表，每 10,000 个 ID 创建一个分区。 + +=== 手动创建分区主表 +[literal] +---- +CREATE TABLE orders ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, -- 分区键 + amount NUMERIC +); +---- + +=== 注册分区主表到 pg_partman +[literal] +---- +SELECT partman.create_parent( + p_parent_table => 'public.orders', -- 要管理的分区父表，注意要带 SCHEMA + p_control => 'id', -- 分区字段 + p_interval => '10000', -- 每1万条数据一个分区 + p_type => 'range', -- 使用范围分区 + p_premake => 3, -- 预创建3个未来的分区 + p_start_partition => '0', -- 起始分区 + p_default_table => false -- 是否创建默认分区 +); +---- + +=== 手动维护 +[literal] +---- +CALL partman.run_maintenance_proc(); +---- + +== 常用操作 + +=== 获取被 pg_partman 管理的分区父表 +[literal] +---- +SELECT parent_table FROM partman.part_config; +---- + +=== 将分区父表从 pg_partman 的管理中移除 +删除维护信息： +[literal] +---- +DELETE FROM partman.part_config +WHERE parent_table = 'table_name'; +---- + +删除模板： +[literal] +---- +DROP TABLE template_[schema]_[table_name]; +---- diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_profile.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_profile.adoc new file mode 100644 index 0000000..498b40d --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_profile.adoc @@ -0,0 +1,105 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_profile + +== 概述 +pg_profile 是一个用于 PostgreSQL 数据库性能分析的扩展工具，主要用于统计目标数据库中的资源密集型活动，帮助用户深入了解数据库运行状态、找出性能瓶颈并进行优化。它的报告本质上是“两个采样点的差量分析”，得出采样间隔内的增量负载，采样点序号从 1 开始计数。 + +pg_profile 硬依赖 dblink 与 plpgsql（其自身完全由 SQL 和 PL/pgSQL 编写，无需任何外部库或软件）；此外，建议安装 pg_stat_statements 扩展，以便在报告中获取 SQL 语句级别的统计信息。pg_profile 的版本与 PostgreSQL 的版本强相关，当前最高支持到 PG 18，具体的版本支持情况请查看 pg_profile 官方 github 仓库的 README. + +IvorySQL 的 PG 模式和 Oracle 兼容模式都已经适配 pg_profile。 + +项目地址： + +开源协议：PostgreSQL License + +== 安装启用 + +=== 源码编译 +执行源码编译和安装： +[literal] +---- +# 构建并安装 pg_profile 及其依赖扩展 +make -C contrib/dblink install +make -C contrib/pg_stat_statements install +make -C contrib/pg_profile install +---- + +安装产物为 `pg_profile.control` 与 `pg_profile--4.11.sql`。 + +=== 修改配置 +pg_profile 依赖的 pg_stat_statements 必须通过 `shared_preload_libraries` 在服务启动时加载。 + +编辑 `ivorysql.conf` ，在 `shared_preload_libraries` 配置项末尾追加 `pg_stat_statements`： +[literal] +---- +# ivorysql.conf +shared_preload_libraries = 'liboracle_parser, ivorysql_ora, pg_stat_statements' +---- + +=== 重启服务 +[literal] +---- +pg_ctl -D $PGDATA restart +---- + +=== 安装扩展 +PG 模式与 Oracle 模式会话下命令相同： +[literal] +---- +CREATE SCHEMA profile; +CREATE SCHEMA dblink; +CREATE SCHEMA statements; +CREATE EXTENSION dblink SCHEMA dblink; +CREATE EXTENSION pg_stat_statements SCHEMA statements; +CREATE EXTENSION pg_profile SCHEMA profile; + +SELECT extname, extversion FROM pg_extension WHERE extname = 'pg_profile'; + extname | extversion +------------+------------ + pg_profile | 4.11 +---- + +== 使用流程 + +=== 创建样本 +[literal] +---- +-- 第一次采样 +SELECT * FROM profile.take_sample(); + server | result | elapsed +--------+--------+------------- + local | OK | 00:00:01.34 + +-- ……业务负载运行一段时间后，再次采样 +SELECT * FROM profile.take_sample(); +---- + +生产环境通常用定时任务周期采样（如 30 分钟一次），可配合 pg_cron 或外部 crontab： +[literal] +---- +*/30 * * * * psql -c 'SELECT profile.take_sample()' >/dev/null +---- + +=== 查看样本 +[literal] +---- +SELECT sample, sample_time, sizes_collected FROM profile.show_samples(); + sample | sample_time | sizes_collected +--------+------------------------+----------------- + 1 | 2026-06-12 09:00:00+00 | t + 2 | 2026-06-12 09:30:00+00 | t +---- + +=== 生成报告 +[literal] +---- +-- 生成样本 1 到样本 2 之间的负载报告（HTML 文本） +\o report_1_2.html +SELECT profile.get_report(1, 2); +\o +---- + +以上函数在 Oracle 模式会话（Oracle 端口/1521）中调用方式与结果完全一致。 diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_readonly.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_readonly.adoc new file mode 100644 index 0000000..9004ba3 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_readonly.adoc @@ -0,0 +1,94 @@ +:sectnums: +:sectnumlevels: 5 + += pg_readonly + +== 概述 +pg_readonly 是一个能够将所有 PostgreSQL 的数据库设为只读的插件。 + +pg_readonly 没有特定的GUC参数，它依靠内存中的全局标志来管理只读状态，并提供了SQL函数来设置或者查询全局标志。 +这个全局标志是集群级别的，集群中的数据库要么全部是只读的，要么全部是可读可写的。 + +只读模式通过过滤SQL语句实现。 +[literal] +---- +允许不包含有写动作函数的SELECT语句； +DML (INSERT, UPDATE, DELETE) 和 DDL 语句包括 TRUNCATE 完全不允许； +DCL 语句 GRANT 与 REVOKE 也是禁止的； +---- + +== 安装 + +[TIP] +源码安装环境为 Ubuntu 24.04(x86_64)，环境中已经安装了IvorySQL，安装路径为/path-to/ivorysql + +=== 源码安装 + +[literal] +---- +# 从 https://github.com/pierreforstmann/pg_readonly/releases/tag/1.0.6 下载 1.0.6 的源码包 1.0.6.tar.gz +tar xvf 1.0.6.tar.gz +cd pg_readonly-1.0.6 + +# 编译安装 +make PG_CONFIF=/path-to/ivorysql/bin/pg_config +make PG_CONFIF=/path-to/ivorysql/bin/pg_config install +---- + +=== 配置预加载库 +修改 data 目录中的 ivorysql.conf 文件，向 shared_preload_libraries 中追加 pg_readonly。 +shared_preload_libraries = 'pg_readonly' + +== 创建Extension并确认pg_readonly版本 + +psql 连接到数据库，执行如下命令： +[literal] +---- +ivorysql=# CREATE EXTENSION pg_readonly; +CREATE EXTENSION + +ivorysql=# SELECT * FROM pg_available_extensions WHERE name = 'pg_readonly'; + name | default_version | installed_version | location | comment +-------------+-----------------+-------------------+----------+---------------------------- + pg_readonly | 1.0.6 | 1.0.6 | $system | cluster database read only +(1 row) +---- + +== 使用 + +=== 检查单个函数 + +[literal] +---- +-- 查询当前集群只读状态 +select get_cluster_readonly(); + get_cluster_readonly +---------------------- + f +(1 row) + +-- 设置当前集群为只读 +select set_cluster_readonly(); + set_cluster_readonly +---------------------- + t +(1 row) + +-- 只读的集群只允许 SELECT 语句 +select * from t; + x | y +---+--- +(0 rows) + +update t set x=33 where y='abc'; +ERROR: cannot execute UPDATE in a read-only transaction + +select 1 into tmp; +ERROR: cannot execute UPDATE in a read-only transaction + +create table tmp(c text); +ERROR: cannot execute UPDATE in a read-only transaction +---- + +更多详细使用方法和高级特性，请参阅 https://github.com/pierreforstmann/pg_readonly 。 + diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_repack.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_repack.adoc new file mode 100644 index 0000000..37d83cf --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_repack.adoc @@ -0,0 +1,102 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_repack + +== 概述 +pg_repack 是 PostgreSQL 的一个扩展，用于在线重整表和索引、消除存储膨胀。它能达到与 `VACUUM FULL` 类似的效果，回收空间，但几乎不阻塞业务读写。 + +IvorySQL 的 PG 模式和 Oracle 兼容模式都已经适配 pg_repack。 + +项目地址： + +官方文档： + +开源协议：BSD-3-Clause License + +== 原理 +VACUUM FULL 因没有"重写期间的并发变更捕获机制"，重写整表时全程持有 ACCESS EXCLUSIVE 锁，期间表不可读写，锁时长随表增大。pg_repack 通过"影子表 + 触发器 + 文件节点交换"把强锁压缩到首尾两个瞬间，从而支持在线重整。 + +=== 变更捕获与数据拷贝 +* 先在原表上安装行级触发器，把此后发生的 INSERT/UPDATE/DELETE 记入日志表（repack.log_）——这一步需短暂的 ACCESS EXCLUSIVE 锁以原子地装上触发器，装好即释放; + +* 创建与原表结构相同的影子表（repack.table_），在一致性 MVCC 快照下把原表数据批量拷入;此阶段只持 SHARE UPDATE EXCLUSIVE，原表照常读写; + +* 在影子表上重建索引，并回放日志，使影子表追平原表最新状态。 + +=== 交换与清理 +* 取得短暂的 ACCESS EXCLUSIVE 锁，回放剩余日志后，在系统目录层面交换原表与影子表的物理文件节点（relfilenode），表的 OID 保持不变——交换是瞬时的; + +* 删除触发器、日志表，并删除影子表（此时它已指向旧堆，删除即回收旧物理文件）;最后对表执行 ANALYZE。 + +[NOTE] +==== +使用前提：目标表必须有主键或非空唯一键；操作过程中约需要两倍表大小的临时磁盘空间。 +==== + +== 安装启用 +环境中已经安装了 IvorySQL5 及以上版本。pg_repack 由两部分组成：服务端扩展（`pg_repack.so` 与 SQL 脚本）和客户端命令行工具 `pg_repack`。 + +假设当前已安装 IvorySQL5，并且 pg_config 工具可用。 + +=== 源码安装 +拉取 pg_repack 源码： +[literal] +---- +git clone --branch ver_1.5.3 https://github.com/reorg/pg_repack.git +---- + +执行编译和安装： +[literal] +---- +cd pg_repack +make +make install +---- + +=== 创建扩展 +[literal] +---- +[ivorysql@localhost ivorysql]$ psql +psql (18.0) +Type "help" for help. + +ivorysql=# CREATE EXTENSION pg_repack; +CREATE EXTENSION +---- + +[NOTE] +==== +`CREATE EXTENSION pg_repack` 需要超级用户执行；客户端工具 `pg_repack` 默认也要求以超级用户身份连接运行。 +==== + +== 使用流程 +pg_repack 通过命令行客户端驱动，连接参数与 psql 一致（`-h` 主机、`-p` 端口、`-U` 用户、`-d` 数据库，或使用 `PGHOST` / `PGPORT` / `PGUSER` 环境变量）。 + +=== 重整整个数据库中所有符合条件的表 +[literal] +---- +pg_repack -d ivorysql +---- + +=== 重整指定表 +推荐使用 schema.table 形式： +[literal] +---- +pg_repack -d ivorysql -t public.big_table +---- + +=== 仅重建索引 +[literal] +---- +pg_repack -d ivorysql -t big_table --only-indexes # 重建该表的全部索引 +pg_repack -d ivorysql -i public.big_table_idx # 重建单个索引 +---- + +=== 预演模式 +只显示将要执行的动作，不实际重整： +[literal] +---- +pg_repack -d ivorysql -t big_table --dry-run +---- diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_show_plans.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_show_plans.adoc new file mode 100644 index 0000000..0a0e2db --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_show_plans.adoc @@ -0,0 +1,111 @@ +:sectnums: +:sectnumlevels: 5 + += pg_show_plans + +== 概述 + +pg_show_plans 是一个 PostgreSQL 扩展，用于显示当前所有正在运行的 SQL 语句的查询计划。查询计划可以以多种格式展示，包括 `TEXT`、`JSON` 和 `YAML`。 + +NOTE: 该扩展在共享内存中创建哈希表。哈希表大小不可动态调整——一旦填满，将无法添加新的查询计划记录。 + +== 安装 + +[TIP] +源码安装环境为 Ubuntu 24.04（x86_64），环境中已安装 IvorySQL 5 及以上版本，安装路径为 `/usr/ivory-5`。 + +=== 源码安装 + +在开始之前，请确保 `pg_config` 可在 PATH 中访问。 + +[source,bash] +---- +git clone https://github.com/cybertec-postgresql/pg_show_plans.git +cd pg_show_plans + +make PG_CONFIG=/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config install +---- + +=== 配置 shared_preload_libraries + +在 `ivorysql.conf` 中将 `pg_show_plans` 添加到 `shared_preload_libraries` 参数： + +[source,conf] +---- +shared_preload_libraries = 'liboracle_parser, ivorysql_ora, pg_show_plans' +---- + +重启 IvorySQL 实例使配置生效： + +[source,bash] +---- +pg_ctl restart -D data +---- + +=== 创建扩展 + +服务器重启后，连接到目标数据库并创建扩展： + +[source,sql] +---- +CREATE EXTENSION pg_show_plans; +---- + +== 使用方法 + +=== 查看当前查询计划 + +使用 `pg_show_plans` 视图，可以查看所有当前正在运行的 SQL 语句的执行计划： + +[source,sql] +---- +SELECT * FROM pg_show_plans; +---- + +[source,text] +---- + pid | level | userid | dbid | plan +-------+-------+--------+-------+----------------------------------------------------------------------- + 11473 | 0 | 10 | 16384 | Function Scan on pg_show_plans (cost=0.00..10.00 rows=1000 width=56) + 11504 | 0 | 10 | 16384 | Function Scan on print_item (cost=0.25..10.25 rows=1000 width=524) + 11504 | 1 | 10 | 16384 | Result (cost=0.00..0.01 rows=1 width=4) +(3 rows) +---- + +=== 同时查看查询计划与 SQL 语句 + +使用 `pg_show_plans_q` 视图，可以同时查看查询计划和对应的 SQL 语句。该视图将 `pg_show_plans` 与 `pg_stat_activity` 进行关联： + +[source,sql] +---- +SELECT * FROM pg_show_plans_q; +---- + +[source,text] +---- +-[ RECORD 1 ]------------------------------------------------------------------------------------ +pid | 11473 +level | 0 +plan | Sort (cost=72.08..74.58 rows=1000 width=80) + | Sort Key: pg_show_plans.pid, pg_show_plans.level + | -> Hash Left Join (cost=2.25..22.25 rows=1000 width=80) + | Hash Cond: (pg_show_plans.pid = s.pid) + | Join Filter: (pg_show_plans.level = 0) + | -> Function Scan on pg_show_plans (cost=0.00..10.00 rows=1000 width=48) + | -> Hash (cost=1.00..1.00 rows=100 width=44) + | -> Function Scan on pg_stat_get_activity s (cost=0.00..1.00 rows=100 width=44) +query | SELECT p.pid, p.level, p.plan, a.query FROM pg_show_plans p + | LEFT JOIN pg_stat_activity a + | ON p.pid = a.pid AND p.level = 0 ORDER BY p.pid, p.level; +-[ RECORD 2 ]------------------------------------------------------------------------------------ +pid | 11517 +level | 0 +plan | Function Scan on print_item (cost=0.25..10.25 rows=1000 width=524) +query | SELECT * FROM print_item(1,20); +-[ RECORD 3 ]------------------------------------------------------------------------------------ +pid | 11517 +level | 1 +plan | Result (cost=0.00..0.01 rows=1 width=4) +query | +---- diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_stat_monitor.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_stat_monitor.adoc new file mode 100644 index 0000000..b115c68 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_stat_monitor.adoc @@ -0,0 +1,98 @@ +:sectnums: +:sectnumlevels: 5 + += pg_stat_monitor + +== 概述 + +pg_stat_monitor 是 PostgreSQL 的查询性能监控工具。它能收集性能统计数据，并通过统一视图和直方图形式直观展示查询性能指标。 + +该工具可帮助数据库用户全面掌握查询来源、执行情况、计划统计与详细信息，以及查询元数据。这极大地提升了可观测性，使用户能够有效调试和优化查询性能。 + +== 安装 + +IvorySQL的安装包里已经集成了pg_stat_monitor插件，如果使用安装包安装的IvorySQL，通常不需要再手动安装pg_stat_monitor。如您使用源码方式安装了IvorySQL，可以继续通过源码方式继续安装pg_stat_monitor插件，IvorySQL社区为您提供了源码安装步骤： + +要从源代码构建 pg_stat_monitor，您需要以下组件: + +* git +* make +* gcc +* pg_config + +您可以从 https://github.com/Percona/pg_stat_monitor/releases[GitHub 上的发布页面] 下载 pg_stat_monitor 指定版本的源代码，或者使用 git 命令： + +[source,bash] +---- +git clone https://github.com/percona/pg_stat_monitor.git +---- + +编译并安装扩展程序，假如环境中已经安装了IvorySQL，安装路径为/usr/ivory-5 + +[source,bash] +---- +cd pg_stat_monitor +export PG_CONFIG=/usr/ivory-5/bin/pg_config +make USE_PGXS=1 +make USE_PGXS=1 install +---- + +== 加载模块 + +在启动时，将 pg_stat_monitor 添加到 shared_preload_libraries 配置参数中，以加载该库。这是因为 pg_stat_monitor 需要额外的共享内存。 + +修改 shared_preload_libraries 参数： + +[source,conf] +---- +shared_preload_libraries = 'pg_stat_monitor'; +---- + +注意：若 shared_preload_libraries 参数中已存在其他模块（例如 pg_stat_statements），需用逗号分隔全部列出。pg_stat_monitor 必须排在 pg_stat_statements 之后，例如： + +[source,conf] +---- +shared_preload_libraries = 'liboracle_parser, ivorysql_ora, pg_stat_statements, pg_stat_monitor'; +---- + +配置完成后，重启 IvorySQL实例使配置生效。 + +[source,bash] +---- +pg_ctl restart -D data +---- + +将 pg_stat_monitor 添加至 shared_preload_libraries 后，该扩展会立即开始收集所有现有数据库的统计信息。要查看监控数据，需在每个待监控的数据库中创建视图。 + +== 创建扩展视图 + +请使用具有超级用户或数据库所有者权限的账户执行以下操作。以超级用户身份连接至目标数据库，运行以下命令创建扩展： + +[source,sql] +---- +CREATE EXTENSION pg_stat_monitor; +---- + +完成设置后，即可查看pg_stat_monitor收集的统计信息。 + +== 使用 + +以获取查询执行时间信息为例，连接数据库执行以下SQL： + +[source,sql] +---- +SELECT userid, total_exec_time, min_exec_time, max_exec_time, mean_exec_time, query FROM pg_stat_monitor; +---- + +[source,text] +---- +userid | total_exec_time | min_exec_time | max_exec_time | mean_exec_time | query +--------+-----------------+---------------+---------------+----------------+---------------------------------------------------------------------------------------------- + 10 | 1.532168 | 0.749108 | 0.78306 | 0.766084 | SELECT userid, datname, queryid, substr(query,$1, $2) AS query, calls FROM pg_stat_monitor + 10 | 0.755857 | 0.755857 | 0.755857 | 0.755857 | SELECT application_name, client_ip, substr(query,$1,$2) as query FROM pg_stat_monitor + 10 | 0 | 0 | 0 | 0 | SELECT userid, total_time, min_time, max_time, mean_time, query FROM pg_stat_monitor; + 10 | 0 | 0 | 0 | 0 | SELECT userid, total_exec_time, min_time, max_time, mean_time, query FROM pg_stat_monitor; +(4 rows) +---- + +更多关于pg_stat_monitor的使用，请参阅 pg_stat_monitor https://docs.percona.com/pg-stat-monitor/user_guide.html[官方文档] \ No newline at end of file diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_textsearch.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_textsearch.adoc new file mode 100644 index 0000000..d888bca --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_textsearch.adoc @@ -0,0 +1,91 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_textsearch + +== 概述 +pg_textsearch 是由Timescale团队开发的一个PostgreSQL扩展，旨在为Postgres提供高性能、现代化的全文搜索能力，并针对AI工作负载和混合搜索进行了优化。 + +== 安装 + +[TIP] +源码安装环境为 Ubuntu 24.04(x86_64)，环境中已经安装了IvorySQL5及以上版本，安装路径为/usr/ivory-5 + +=== 源码安装 + +[literal] +---- +# 从 https://github.com/timescale/pg_textsearch/archive/refs/tags/v0.6.1.tar.gz 下载源码包 + +tar xzvf v0.6.1.tar.gz +cd pg_textsearch-0.6.1 + +# 编译安装插件 +make PG_CONFIG=/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config install + +---- + +[TIP] +如果出现找不到xlocale.h的错误，需要手动修改 /usr/ivory-5/include/postgresql/server/pg_config.h +删除或者注释掉 #define HAVE_XLOCALE_H 1 这一行 + +=== 修改数据库配置文件 + +修改 ivorysql.conf 文件，添加 pg_textsearch 到 shared_preload_libraries + +[literal] +---- +shared_preload_libraries = 'gb18030_2022, liboracle_parser, ivorysql_ora, pg_textsearch' +---- + +重启数据库后配置生效。 + +=== 创建Extension + +[literal] +---- +postgres=# create extension pg_textsearch; +WARNING: pg_textsearch v0.6.1 is a prerelease. Do not use in production. +CREATE EXTENSION +---- + +== 使用 + +创建一个带有文本内容的表： +[literal] +---- +postgres=# CREATE TABLE documents (id bigserial PRIMARY KEY, content text); +CREATE TABLE + +postgres=# INSERT INTO documents (content) VALUES + ('PostgreSQL is a powerful database system'), + ('BM25 is an effective ranking function'), + ('Full text search with custom scoring'); +INSERT 0 3 +---- + +在文本列上创建 pg_textsearch 索引: +[literal] +---- +postgres=# CREATE INDEX docs_idx ON documents USING bm25(content) WITH (text_config='english'); +NOTICE: BM25 index build started for relation docs_idx +NOTICE: Using text search configuration: english +NOTICE: Using index options: k1=1.20, b=0.75 +NOTICE: BM25 index build completed: 3 documents, avg_length=4.33 +CREATE INDEX +---- + +使用@操作符获取最相关的文档： +[literal] +---- +postgres=# SELECT * FROM documents ORDER BY content <@> 'database system' LIMIT 5; + id | content +----+------------------------------------------ + 1 | PostgreSQL is a powerful database system + 2 | BM25 is an effective ranking function + 3 | Full text search with custom scoring +(3 rows) +---- + diff --git a/CN/modules/ROOT/pages/master/5.8.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pgaudit.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/5.8.adoc rename to CN/modules/ROOT/pages/master/ecosystem_components/pgaudit.adoc diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pgbackrest.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pgbackrest.adoc new file mode 100644 index 0000000..435aa16 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pgbackrest.adoc @@ -0,0 +1,500 @@ + +:sectnums: +:sectnumlevels: 5 + += pgBackRest + +== 概述 + +pgBackRest 是一款可靠的 PostgreSQL 备份和恢复解决方案。其核心基于 PostgreSQL 的物理级备份（数据目录的文件）和WAL归档管理机制。特点是项目质量很高：100%测试覆盖、高质量编码、精简依赖。 + +NOTE: pgBackRest在 v2.48.0 版本引入了对 pg_control 文件的 CRC 校验，由于 IvorySQL 修改了 pg_control 文件布局，所以需要在 stanza 配置中使用 pg-version-force 选项指定 IvorySQL 对应的 PostgreSQL 版本。 + +NOTE: pgBackRest 无需修改即可在 IvorySQL 5.x 下使用，但这仅是一个巧合，实际使用中仍建议增加 pg-version-force 选项。 + +项目地址： + +版本：v2.58.0 + +开源协议：MIT License + +== 安装 + +[TIP] +官网推荐从包管理器安装。 + +[TIP] +源码测试安装环境为 Ubuntu 26.04。 + +=== 依赖 + +[source,bash] +---- +sudo apt update && \ +sudo apt-get install python3-setuptools meson gcc \ + libpq-dev libssl-dev libxml2-dev \ + pkg-config liblz4-dev libzstd-dev \ + libbz2-dev libz-dev libyaml-dev \ + libssh2-1-dev +---- + +=== 源码安装 + +[source,bash] +---- +cd ~ +mkdir -p build +wget -q -O - \ + https://github.com/pgbackrest/pgbackrest/archive/release/2.58.0.tar.gz | \ + tar zx -C build +meson setup build/pgbackrest build/pgbackrest-release-2.58.0 +ninja -C build/pgbackrest + +# 编译完成后，将在 build/src 路径下生成 pgbackrest 可执行文件 +./build/src/pgbackrest --version +---- + +=== 验证安装 + +NOTE: pgBackRest本体只有一个可执行文件，使用时通过传入配置文件即可。本文仅作使用说明，无需额外安装步骤。 + +== 配置 + +NOTE: pgBackRest 支持多备份仓库，详情参考： + +NOTE: 本文演示中，PG数据库和备份仓库均在同一台主机上，备份仓库为本地文件。 + +需要在两两处对进行配置： + +[cols="1,2"] +|=== +| 文件名 | 说明 + +| postgresql.conf +| 配置 `PostgreSQL` ，增加 `archive_command` ，使能 `WAL` 备份 + +| pgbackrest.conf +| 对 `pgBackRest` 本身的配置，包括源数据库、备份仓库等的配置 +|=== + +在 `postgresql.conf` 末尾附加： + +NOTE: 仅为示例，具体操作参考文档后续部分。 + +[source,conf] +---- +port = 6551 +ivorysql.port = 6553 +listen_addresses = '127.0.0.1' +unix_socket_directories = '/tmp/.pgbr_sock_1000_6551' +archive_mode = on +archive_command = '/path/to/pgbackrest --config=/path/to/pgbackrest.conf --stanza=ivory archive-push %p' +log_min_messages = info +---- + +创建 `pgbackrest.conf` ： + +NOTE: 仅为示例，具体操作参考文档后续部分。 + +[source,conf] +---- +[global] +# 配置了多个repos的话，WAL是全冗余、备份需要手动执行 +repo1-path=/tmp/pgbackrest_ivorysql/repo +# 完整备份保留时间 +repo1-retention-full=9999 +# log文件 +log-path=/tmp/pgbackrest_ivorysql/log +# console的loglevel +log-level-console=info +# 写入文件的loglevel +log-level-file=detail +# 不等下一个checkpoint +start-fast=y + +[$STANZA] +# 配置多个的pg的情况是primary+replica，编号不代表主备，自动识别 +pg1-path=/tmp/pgbackrest_ivorysql/pg1/data +pg1-port=6551 +pg1-socket-path='/tmp/.pgbr_sock_1000_6551' +pg-version-force=18 +---- + +== 使用 + +=== pgBackRest 命令 + +[source,bash] +---- +# 使用方式 +pgbackrest [选项] [命令] +---- + +`pgBackRest` 命令列表： + +[source,bash] +---- +# 增加或修改备份的注释，注释以JSON形式记录在backup.info文件 +annotate + +# 备份仓库中获取WAL段，PostgreSQL 通过 restore_command 调用。 +# 执行pgbackrest restore 时自动把它写进 postgresql.auto.conf： +# "restore_command = 'pgbackrest --stanza=ivory archive-get %f "%p"'" +archive-get + +# 向备份仓库存入WAL段，把 PostgreSQL 刚写完的 WAL 段推送到 pgbackrest 仓库。 +# 它和 archive-get 一进一出，构成 PITR 的基础。 +# 需要手动写入postgresql.conf: "archive_command = 'pgbackrest --stanza=ivory archive-push %p'" +archive-push + +# 备份PG数据库，对整个数据目录做一次一致性备份并写入仓库。 +backup + +# 检查配置、核对仓库信息、归档链路测试（会执行pg_switch_wal） +check + +# 删除超期的备份和不再需要的 WAL 归档 +expire + +# 显示帮助信息 +help + +# 获取备份的信息 +info + +# 用于调试，把仓库里的任意文件取出来写到 stdout 或本地文件。 +repo-get + +# 通过存储驱动列出仓库里的文件和目录 +repo-ls + +# 把仓库里的备份还原成可启动的数据目录，并自动配置好后续的 WAL 回放。 +restore + +# 运行多主机部署时的传输服务（TLS模式）：在主机间搬运备份数据。PG和Repo主机都要运行。 +server + +# pgBackRest server探活命令 +server-ping + +# 初始化命令：在仓库里为一套集群建立 stanza 的元数据骨架 +stanza-create + +# 删除stanza +stanza-delete + +# 数据库大版本升级后，需要同步升级stanza +stanza-upgrade + +# 不启动任何进程/服务，唯一的工作是删掉 stop 命令留下的停止文件，让 pgbackrest 操作恢复运行。 +start + +# 创建停止文件，backup、archive-push、expire、stanza-create/upgrade 及 remote 请求在启动时检测到该文件即拒绝。 +stop + +# 检查已存在的备份和 WAL 是否完好 +verify + +# 获取当前版本 +version +---- + +`pgBackRest` 内部命令列表，这些命令不应该在生产场景下使用，可能导致数据损坏： + +[source,bash] +---- +# 以可读格式转储某次备份的 manifest +manifest + +# 往仓库写文件（自动加密），repo-get 的反向 +repo-put + +# 删仓库文件/目录（--recurse） +repo-rm +---- + +=== 执行操作 + +TIP: 由于操作过程较为复杂，为了便于测试，将命令进行参数化，并构造了辅助函数。下面的命令，请在同一终端回话内执行。 + +变量准备： + +[source,bash] +---- +# 根据实际情况修改：IVORY_PG_VERSION_MAJOR、IVORY_BIN、PGBACKREST_BIN，其他可默认 +IVORY_PG_VERSION_MAJOR=18 +IVORY_BIN="/usr/local/ivorysql/bin" +PGBACKREST_BIN="/home/lct/repos/pgbackrest/build/src/pgbackrest" +TEST_BASE="/tmp/pgbackrest_ivorysql_regress" +PG_PORT="6551" +STANDBY_PORT="6552" +ORA_PORT="6553" +STANDBY_ORA_PORT="6554" +DB_MODE="oracle" +STANZA="ivory" + +PG1_DATA="$TEST_BASE/pg1/data" +PG2_DATA="$TEST_BASE/pg2/data" +REPO_PATH="$TEST_BASE/repo" +# unix socket 路径有 107 字节上限，固定放在 /tmp 下的短目录 +SOCK_PATH="${SOCK_PATH:-/tmp/.pgbr_sock_$(id -u)_$PG_PORT}" +LOG_PATH="$TEST_BASE/log" +CONF="$TEST_BASE/pgbackrest.conf" +RUN_LOG="$TEST_BASE/regression.log" +TEST_DB="regress" +RESTORE_POINT="pgb_regress_rp" + +sql() +{ + local port="$1" db="$2" query="$3" + "$IVORY_BIN/psql" -h "$SOCK_PATH" -p "$port" -d "$db" -U "$(id -un)" \ + -v ON_ERROR_STOP=1 -Atc "$query" 2>>"$RUN_LOG" +} +switch_wal_and_wait() +{ + local wal i + # 先产生一条 WAL 记录，避免当前段为空时 pg_switch_wal() 空转（不会触发归档） + sql "$ORA_PORT" postgres "SELECT pg_create_restore_point('pgb_wal_sync')" >/dev/null || return 1 + wal=$(sql "$ORA_PORT" postgres "SELECT pg_walfile_name(pg_switch_wal())") || return 1 + for i in $(seq 1 60); do + local archived + archived=$(sql "$ORA_PORT" postgres "SELECT last_archived_wal FROM pg_stat_archiver") + [[ -n "$archived" && ! "$archived" < "$wal" ]] && return 0 + sleep 1 + done + return 1 +} +---- + +执行前准备： + +[source,bash] +---- +rm -rf "$SOCK_PATH" +mkdir -p "$PG1_DATA" "$REPO_PATH" "$SOCK_PATH" "$LOG_PATH" +chmod 700 "$PG1_DATA" "$SOCK_PATH" +: >"$RUN_LOG" +export LD_LIBRARY_PATH="$(dirname "$IVORY_BIN")/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + +# 确认一下版本 +${IVORY_BIN}/pg_ctl --version +${PGBACKREST_BIN} --version +---- + +初始化主库及配置： + +[source,bash] +---- +# 初始化，使用oracle模式 +"$IVORY_BIN/initdb" -D "$PG1_DATA" -m "$DB_MODE" -E UTF8 --no-locale -A trust -U "$(id -un)" + +# 配置IvorySQL +cat >>"$PG1_DATA/postgresql.conf" <"$CONF" < 100;"; + +# 执行恢复 +"$IVORY_BIN/pg_ctl" -D "$PG1_DATA" -w -t 60 -m fast stop +"$PGBACKREST_BIN" --config="$CONF" --stanza="$STANZA" \ + --delta --type=name --target="$RESTORE_POINT" \ + --target-action=promote restore; + +# 验证，这次的md5输出应该与上面的一致 +"$IVORY_BIN/pg_ctl" -D "$PG1_DATA" -l "$LOG_PATH/pg1.log" -w start +sql "$ORA_PORT" "$TEST_DB" \ + "SELECT md5(string_agg(id::text || ':' || name, ',' ORDER BY id)) FROM t_ora;"; +---- diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pgbouncer.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pgbouncer.adoc new file mode 100644 index 0000000..0973b42 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pgbouncer.adoc @@ -0,0 +1,221 @@ +:sectnums: +:sectnumlevels: 5 + += PgBouncer + +== 概述 + +PgBouncer 是轻量级连接池中间件，部署在应用层与数据库之间，通过复用后端连接来降低连接开销、保护数据库资源、提高应用并发性能。 + +PgBouncer 使用标准 PostgreSQL 通信协议，IvorySQL 完全兼容该协议。 + +*三种连接池模式：* + +[cols="1,2,2"] +|=== +| 模式 | 说明 | 适用场景 + +| session +| 客户端连接期间独占一个后端连接 +| 需要过程内 COMMIT、完整会话特性 + +| transaction +| 每个事务结束后归还连接 +| 最常用，连接复用率最高 + +| statement +| 每条语句后归还连接 +| 限制最多，不支持显式事务 +|=== + +== 安装 + +[TIP] +源码测试安装环境为 Ubuntu 24.04。 + +=== 依赖 + +[literal] +---- +# Ubuntu / Debian +sudo apt install libevent-dev libssl-dev pkg-config + +# RHEL / Rocky Linux +sudo dnf install libevent-devel openssl-devel pkgconfig +---- + +=== 源码安装 + +[literal] +---- +wget https://github.com/pgbouncer/pgbouncer/archive/refs/tags/pgbouncer_1_25_1.zip +unzip pgbouncer_1_25_1.zip +cd pgbouncer_1_25_1 + +./autogen.sh + +./configure \ + --prefix=/usr/ivory-5 \ + --with-openssl \ + --with-pam + +make -j4 +make install +cp pgbouncer /usr/ivory-5/bin/ +---- + +=== 验证安装 + +[literal] +---- +pgbouncer --version +# PgBouncer 1.25.1 +# libevent 2.1.12-stable +# tls: OpenSSL 3.0.2 15 Mar 2022 +---- + +== 配置 + +=== 连接 IvorySQL PG 模式（端口 5432） + +创建 `/etc/pgbouncer/pgbouncer.ini`： + +[literal] +---- +[databases] +postgres = host=127.0.0.1 port=5432 dbname=postgres + +[pgbouncer] +listen_addr = 127.0.0.1 +listen_port = 6432 +auth_type = trust +auth_file = /etc/pgbouncer/userlist.txt +pool_mode = transaction +max_client_conn = 200 +default_pool_size = 20 +logfile = /var/log/pgbouncer/pgbouncer.log +pidfile = /var/run/pgbouncer/pgbouncer.pid +---- + +=== 连接 IvorySQL Oracle 兼容模式（端口 1521） + +[literal] +---- +[databases] +postgres = host=127.0.0.1 port=1521 dbname=postgres + +[pgbouncer] +listen_addr = 127.0.0.1 +listen_port = 2521 +auth_type = trust +auth_file = /etc/pgbouncer/userlist.txt + +# Oracle 兼容模式建议使用 session 模式 +pool_mode = session + +max_client_conn = 200 +default_pool_size = 20 +logfile = /var/log/pgbouncer/pgbouncer_oracle.log +pidfile = /var/run/pgbouncer/pgbouncer_oracle.pid +---- + +=== 用户认证文件 + +[literal] +---- +# /etc/pgbouncer/userlist.txt +# 格式："用户名" "密码"（trust 模式密码留空） +"postgres" "" +"app_user" "app_password" +---- + +=== 启动与停止 + +[literal] +---- +# 前台启动（调试） +pgbouncer /etc/pgbouncer/pgbouncer.ini + +# 后台启动 +pgbouncer -d /etc/pgbouncer/pgbouncer.ini + +# 重载配置（不中断连接） +kill -HUP $(cat /var/run/pgbouncer/pgbouncer.pid) + +# 停止 +kill -INT $(cat /var/run/pgbouncer/pgbouncer.pid) +---- + +== 使用 + +=== 客户端连接 + +通过 PgBouncer 连接与直连 IvorySQL 语法完全一致，只需修改端口： + +[literal] +---- +# PG 模式（经由 PgBouncer） +psql -U postgres -p 6432 -d postgres + +# Oracle 兼容模式（经由 PgBouncer） +psql -U postgres -p 2521 -d postgres +---- + +=== 管理控制台 + +PgBouncer 提供内置管理数据库： + +[literal] +---- +psql -U postgres -p 6432 -d pgbouncer +---- + +[literal] +---- +-- 查看连接池状态 +SHOW POOLS; + +-- 查看统计信息 +SHOW STATS; + +-- 查看客户端连接 +SHOW CLIENTS; + +-- 查看后端连接 +SHOW SERVERS; + +-- 重载配置 +RELOAD; +---- + +=== Oracle 兼容模式 + +[literal] +---- +-- 确认 Oracle 模式已激活 +SHOW ivorysql.compatible_mode; +-- ivorysql.compatible_mode +-- -------------------------- +-- oracle + +-- Oracle 数据类型与函数 +CREATE TABLE bouncer_test ( + id NUMBER(10) PRIMARY KEY, + name VARCHAR2(100), + hired DATE DEFAULT SYSDATE +); + +INSERT INTO bouncer_test VALUES (1, 'Alice', SYSDATE); + +SELECT id, + NVL(name, 'N/A') AS name, + DECODE(id, 1, 'CEO', 'Staff') AS title, + TO_CHAR(hired, 'YYYY-MM-DD') AS hire_date +FROM bouncer_test; + +-- Oracle 序列 +CREATE SEQUENCE ora_seq START WITH 100 INCREMENT BY 10; +SELECT ora_seq.NEXTVAL FROM DUAL; -- 100 +SELECT ora_seq.NEXTVAL FROM DUAL; -- 110 +SELECT ora_seq.CURRVAL FROM DUAL; -- 110 +---- diff --git a/CN/modules/ROOT/pages/master/5.3.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pgddl.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/5.3.adoc rename to CN/modules/ROOT/pages/master/ecosystem_components/pgddl.adoc diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pgdog.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pgdog.adoc new file mode 100644 index 0000000..5b2b7a3 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pgdog.adoc @@ -0,0 +1,279 @@ + +:sectnums: +:sectnumlevels: 5 + += PgDog + +== 概述 +PgDog 是一个专为 PostgreSQL 设计的高性能、开源集群中间件（代理工具），采用 Rust 语言编写。它集成了自动分片、连接池和负载均衡功能，能让开发者在无需修改任何应用程序代码的前提下，实现 PostgreSQL 数据库的水平扩展与高可用管理。 + +NOTE: PgDog 使用 PostgreSQL 原生的 pg_query 模块实现语句解析，所以暂时不支持在 Oracle 兼容模式下运行。 + +项目地址： + +版本：v0.1.45 + +开源协议：AGPL-3.0 License + +== 安装 + +[TIP] +源码测试安装环境为 Ubuntu 26.04。 + +=== 依赖 + +[source,bash] +---- +sudo apt update && \ +sudo apt install -y cmake clang curl pkg-config \ + libssl-dev git build-essential mold rustup \ + docker +---- + +[TIP] +本文使用说明需要两个IvorySQL数据库实例，可通过本文提供的docker-compose文件快速搭建。安装docker-compose请参考： + +=== 源码安装 + +[source,bash] +---- +wget https://github.com/pgdogdev/pgdog/archive/refs/tags/v0.1.45.tar.gz +tar -zxf v0.1.45.tar.gz +cd pgdog-0.1.45 + +# 编译完后，会在 target/release 目录下生成可执行文件 +cargo build --release +---- + +=== 验证安装 + +[source,bash] +---- +# PgDog在本文档编写时处于快速迭代开发阶段，所以下面命令的输出不完整 +./target/release/pgdog --version +# 输出：PgDog v +---- + +== 配置 + +本文将配置两个分片，对PgDog的自动分片功能为例进行使用说明。 + +PgDog通过两个文件进行配置： + +[cols="1,2"] +|=== +| 文件名 | 说明 + +| pgdog.toml +| 包含PgDog的端口配置、后端PostgreSQL服务的配置等基础配置信息 + +| users.toml +| 访问PgDog的用户名和密码在这里配置 +|=== + + +创建 `pgdog.toml` ： + +[source,toml] +---- +[general] +host = "0.0.0.0" +port = 6432 +default_pool_size = 10 + +# ---- ivory_shard：分片到两个 IvorySQL 后端 ---- +# host、port以及database_name，如果不是使用本文提供的docker-compose搭建的环境，需要按照实际情况修改 +[[databases]] +name = "ivory_shard" +host = "ivory-shard0" +port = 5432 +database_name = "testdb" +user = "ivorysql" # 如果不提供，默认使用 users.toml 中对应的 name +password = "ivorysql" # 如果不提供，默认使用 users.toml 中对应的 password +shard = 0 + +[[databases]] +name = "ivory_shard" +host = "ivory-shard1" +port = 5432 +database_name = "testdb" +shard = 1 + +# ---- 分片键声明 ---- +# 配置必须与实际表结构统一 +[[sharded_tables]] +database = "ivory_shard" +name = "orders" +column = "customer_id" +data_type = "bigint" +---- + +创建 `users.toml` ： + +[source,toml] +---- +[admin] +name = "admin" +user = "admin" +password = "pgdog" + +[[users]] +name = "ivorysql" +password = "ivorysql" +database = "ivory_shard" +pool_size = 10 +---- + +创建 `docker-compose.shard.yml` ： + +[TIP] + `volumes` 字段请根据实际的配置文件位置进行修改 + +[source,dockerfile] +---- +# 分片测试拓扑（独立 compose）：2 个分片后端。 +# ivory-shard0 IvorySQL 5.4 (pg) host:5443 +# ivory-shard1 IvorySQL 5.4 (pg) host:5444 + +x-ivory: &ivory + image: registry.highgo.com/ivorysql/ivorysql:5.4-bookworm + environment: &ivoryenv + MODE: pg + IVORYSQL_USER: ivorysql + IVORYSQL_PASSWORD: ivorysql + IVORYSQL_DB: testdb + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ivorysql -d testdb"] + interval: 5s + timeout: 3s + retries: 30 + +services: + ivory-shard0: + <<: *ivory + container_name: ivory-shard0 + ports: ["5443:5432"] + ivory-shard1: + <<: *ivory + container_name: ivory-shard1 + ports: ["5444:5432"] +---- + +== 使用 + +=== 启动PgDog + +[source,bash] +---- +./target/release/pgdog --config ./pgdog.toml --users ./users.toml +---- + + +=== 管理控制台 + +PgDog 提供了内置的管理数据库，用户名密码通过 `users.toml` 中 `+[[admin]]+` 字段进行配置。 + +[source,bash] +---- +psql "postgres://admin:pgdog@localhost:6433/admin" +---- + +[source,sql] +---- +-- 查看客户端连接及实时统计 +SHOW CLIENTS + +-- 查看从PgDog发起的PostgreSQL连接 +SHOW SERVERS + +-- 查看连接池信息 +SHOW POOLS + +-- 查看当前从pgdog.toml加载的配置 +SHOW CONFIG + +-- 查看连接池统计 +SHOW STATS + +-- 在同一网络中运行的 PgDog 进程列表。需要启用服务发现(service discovery) +SHOW PEERS + +-- 从磁盘重新加载配置。关于哪些选项可以在运行时修改,请参阅 pgdog.toml 和 users.toml +RELOAD + +-- 使用现有配置重新创建所有服务器连接 +RECONNECT + +-- 暂停所有连接池。客户端将一直等待连接,直到连接池恢复。可用于平滑重启 PostgreSQL 服务器 +PAUSE + +-- 恢复所有连接池。客户端可以重新取用连接 +RESUME + +-- 列出当前缓存中的prepared statements +SHOW PREPARED + +-- 列出当前位于 AST 缓存中、用于查询路由的语句 +SHOW QUERY_CACHE + +-- 暂停所有查询,以便在多个 PgDog 实例之间同步配置变更 +MAINTENANCE + +-- 显示每个数据库的 PostgreSQL 复制状态,包括副本延迟 +SHOW REPLICATION +---- + +=== 连接PgDog + +[source,bash] +---- +psql "postgres://ivorysql:ivorysql@localhost:6433/ivory_shard" +---- + +=== 执行操作 + +[TIP] +请确认分片后端没有 `orders` 表，PgDog会自动创建。 + +[source,sql] +---- +-- 创建表 +CREATE TABLE orders ( + order_id bigint, + customer_id bigint, + amount numeric(10,2), + PRIMARY KEY (order_id, customer_id) +); + +-- 插入数据：这两条数据将分别插入到两个分片后端 +-- BUG：这里不能使用 generate_series 函数，因为PgDog暂时对这个函数进行透传 +INSERT INTO orders values(1, 1, 1); +INSERT INTO orders values(2, 2, 2); +INSERT INTO orders values(3, 3, 3); +INSERT INTO orders values(4, 4, 4); +INSERT INTO orders values(5, 5, 5); +INSERT INTO orders values(6, 6, 6); +INSERT INTO orders values(7, 7, 7); + +-- 查询数据 +SELECT * FROM orders; +---- + +=== 分别连接两个分片后端确认数据 + +[TIP] +这里将看到两个分片后端的条目并不是均分的，这是因为PgDog根据 `+[[sharded_tables]]+` 中配置的 `customer_id` 字段，提取了数值并对其进行HASH分片。 + +[source,bash] +---- +# 分片1 +psql "postgres://ivorysql:ivorysql@localhost:5443/testdb", +# 分片2 +psql "postgres://ivorysql:ivorysql@localhost:5444/testdb", +---- + +在分片后端分别执行查询确认数据 +[source,sql] +---- +SELECT * FROM orders; +---- diff --git a/CN/modules/ROOT/pages/master/5.7.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pgroonga.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/5.7.adoc rename to CN/modules/ROOT/pages/master/ecosystem_components/pgroonga.adoc diff --git a/CN/modules/ROOT/pages/master/5.9.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pgrouting.adoc similarity index 98% rename from CN/modules/ROOT/pages/master/5.9.adoc rename to CN/modules/ROOT/pages/master/ecosystem_components/pgrouting.adoc index 9ef7989..60c6fa9 100644 --- a/CN/modules/ROOT/pages/master/5.9.adoc +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pgrouting.adoc @@ -39,7 +39,7 @@ make sudo make install ``` -== 创建Extension并确认ddlx版本 +== 创建Extension并确认版本 psql 连接到数据库，执行如下命令： ``` diff --git a/CN/modules/ROOT/pages/master/5.5.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pgsql_http.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/5.5.adoc rename to CN/modules/ROOT/pages/master/ecosystem_components/pgsql_http.adoc diff --git a/CN/modules/ROOT/pages/master/5.2.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pgvector.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/5.2.adoc rename to CN/modules/ROOT/pages/master/ecosystem_components/pgvector.adoc diff --git a/CN/modules/ROOT/pages/master/5.6.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/plpgsql_check.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/5.6.adoc rename to CN/modules/ROOT/pages/master/ecosystem_components/plpgsql_check.adoc diff --git a/CN/modules/ROOT/pages/master/5.1.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/postgis.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/5.1.adoc rename to CN/modules/ROOT/pages/master/ecosystem_components/postgis.adoc diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/redis_fdw.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/redis_fdw.adoc new file mode 100644 index 0000000..d6014e4 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/redis_fdw.adoc @@ -0,0 +1,110 @@ + +:sectnums: +:sectnumlevels: 5 + += redis_fdw + +== 概述 +redis_fdw 实现PostgreSQL与Redis键值数据库的连接，支持SELECT、INSERT、UPDATE、DELETE等操作，适用于PostgreSQL 10+及Redis 6.0左右版本，可处理哈希、列表等多种数据类型。在 PostgreSQL 和 Oracle 兼容模式下均可正常工作。 + +== 安装 + +[TIP] +源码安装环境为 Ubuntu 24.04(x86_64)，环境中已经安装了IvorySQL5及以上版本，安装路径为/usr/ivory-5。 + +=== 源码安装 + +[literal] +---- +# 安装 Hiredis 库 +wget https://github.com/redis/hiredis/archive/refs/tags/v1.3.0.tar.gz +tar xzvf v1.3.0.tar.gz +cd hiredis-1.3.0 +make +sudo make install +sudo ldconfig + +# 安装本地 Redis 服务 +sudo apt update +sudo apt install -y redis-server + +# 本地Redis服务默认开启，127.0.0.1:6379 + +# 使用 redis-cli 命令 创建一个管理员用户：（临时创建的用户重启 Redis 会丢失） +# 用户：highgo，密码：Admin@123，启用，全键全命令 +ACL SETUSER highgo on >Admin@123 ~* +@all + +# 下载 redis_fdw 源码包 +git clone https://github.com/pg-redis-fdw/redis_fdw.git -b REL_18_STABLE +cd redis_fdw + +# 编译安装 redis_fdw 插件 +make PG_CONFIG=/usr/ivory-5/bin/pg_config +sudo make install PG_CONFIG=/usr/ivory-5/bin/pg_config + +---- + +[TIP] +如果出现找不到xlocale.h的错误，需要手动修改 /usr/ivory-5/include/postgresql/server/pg_config.h +删除或者注释掉 #define HAVE_XLOCALE_H 1 这一行 + +=== 创建Extension + +[literal] +---- +postgres=# create extension redis_fdw; +CREATE EXTENSION +---- + +== 使用 + +[literal] +---- +// 创建外部服务器 +// Create a foreign server with appropriate configuration +postgres=# CREATE SERVER redis_server +postgres-# FOREIGN DATA WRAPPER redis_fdw +postgres-# OPTIONS ( address '127.0.0.1', port '6379'); +CREATE SERVER +---- + +[literal] +---- +// 创建用户映射 +// User mapping +postgres=# CREATE USER MAPPING FOR highgo +postgres-# SERVER redis_server +postgres-# OPTIONS (password 'Admin@123'); +CREATE USER MAPPING +---- + +[literal] +---- +// 创建一个简单表 +postgres=# CREATE FOREIGN TABLE redis_db0 ( +postgres(# key text, +postgres(# val text +postgres(# ) +postgres-# SERVER redis_server +postgres-# OPTIONS ( +postgres(# database '0' +postgres(# ); +CREATE FOREIGN TABLE +---- + +[literal] +---- +// 插入数据 +postgres=# insert into redis_db0 values('k2', 'v2'); +INSERT 0 1 +---- + +[literal] +---- +// 读取数据 +postgres=# select * from redis_db0; + key | val +-----+----- + k2 | v2 +(1 row) +---- \ No newline at end of file diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/set_user.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/set_user.adoc new file mode 100644 index 0000000..267d924 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/set_user.adoc @@ -0,0 +1,251 @@ + +:sectnums: +:sectnumlevels: 5 + += set_user + +== 概述 +set_user 是 PostgreSQL 安全审计扩展，由 pgaudit 项目维护，优化原生用户切换能力。支持普通用户间互切，也可受控切换至超级用户；可配置白名单限定允许切换的用户，所有切换操作留存审计日志，切换到超级用户时强制完整记录 SQL，拦截修改数据库配置、调用系统命令等高风险操作，规范临时提权，部署需预加载插件并重启数据库，再创建扩展使用。 + +项目地址： + +版本：REL4_2_0 + +开源协议：PostgreSQL License + +== 原理介绍 + +PostgreSQL原生的 `SET ROLE` / `SET SESSION AUTHORIZATION` 有两个安全短板：一是提权后可以随手 `SET log_statement = 'none'` 关掉日志、`RESET ROLE` 悄悄切回来，审计留不下痕迹；二是要让 DBA 能干超级用户的活，通常得给他们能直接登录 superuser 账号，权限过大且不可控。 + +set_user 的思路是"不禁止提权，但让提权全程留痕、无法抵赖"。部署后，所有超级用户账号可设为 NOLOGIN，DBA 以普通账号登录，需要时调用 set_user_u('postgres') 提权。从提权到 reset_user() 恢复的整个窗口内：用户切换被记入日志，log_statement 被强制改为 all 使每条 SQL 落盘，日志前缀自动追加 AUDIT 标签便于过滤告警；同时 ALTER SYSTEM、COPY PROGRAM、SET log_statement、SET ROLE 及 set_config() 后门等所有可能破坏审计或逃逸身份的通道被全部封锁。由于 session_user 始终保持真实登录者，"谁在什么时候以什么身份做了什么"在日志中一目了然。 + +实现上，它以 C 扩展形式通过 shared_preload_libraries 加载，核心依赖三个内核机制：ProcessUtility_hook 拦截危险语句，object_access_hook 封堵函数级后门，事务提交回调保证切换的事务安全。权限控制采用双闸设计——SQL 层的 GRANT EXECUTE 决定谁能调用，配置层的白名单（superuser_allowlist 等）可随时热调整收口。此外还提供带口令锁的 set_user(user, token) 供连接池代持连接时防逃逸，以及不可逆的 set_session_auth() 用于连接移交前的永久降权。 + +set_user 并不能阻止超级用户作恶——它的定位是确保任何特权操作都无法逃避审计，配合日志告警系统，构成 PostgreSQL 最小权限运维体系的关键一环。 + +== 使用说明 + +[NOTE] +需要将 `set_user` 添加到 `shared_preload_libraries`。如果有其他插件实现 `post-execution hooks` 那么需要将 `set_user` 列在这些插件之前。 + +[IMPORTANT] +superuser 账号（如 postgres）需要设为 NOLOGIN，DBA 用普通账号登录，需要时调用 `set_user_u('postgres')` 提权。 + +** 配置介绍 + +[cols="1,1,1,1", options="header"] +|=== +| 配置项 | 说明 | 示例 | 功能 +| set_user.block_alter_system | `on` （默认）或者 `off` | `on` | 阻止 ALTER SYSTEM 命令 +| set_user.block_copy_program | `on` （默认）或者 `off` | `on` | 阻止 COPY PROGRAM 命令 +| set_user.block_log_statement | `on` （默认）或者 `off` | `on` | 阻止修改 log_statement +| set_user.nosuperuser_target_allowlist | 通配符 '*' （默认）或字符串 | 'dba1, dba2, +admin_group' | 允许 set_user() 切换到的目标用户名单 +| set_user.superuser_allowlist | 通配符 '*' （默认）或字符串 | 'dba1, dba2, +admin_group' | 允许调用 set_user_u() 提权的用户名单 +| set_user.superuser_audit_tag | 字符串 | 'AUDIT' | 日志前缀标签 +| set_user.exit_on_error | `on` （默认）或者 `off` | `on` | 出现错误时是否退出当前会话 +|=== + + +** 函数介绍 + +[cols="1,1,1,1", options="header"] +|=== +| 函数 | 默认权限 | 参数 | 说明 +| `set_user(text)` | REMOVE FROM PUBLIC | 目标非superuser用户名 | 将当前会话的用户身份切换为指定的非superuser用户 +| `set_user(text, text)` | REMOVE FROM PUBLIC | 目标非superuser用户名，随机token | 同上，但需要提供一个随机token以增加安全性 +| `set_user_u(text)` | REMOVE FROM PUBLIC | 目标superuser用户名 | 将当前会话的用户身份切换为指定的superuser用户 +| `reset_user()` | GRANT TO PUBLIC | 无 | 将当前会话的用户身份恢复为原始用户 +| `reset_user(text)` | GRANT TO PUBLIC | 随机token | 带口令恢复，但是需要提供对应的token +| `set_session_auth(text)` | REMOVE FROM PUBLIC | 目标非superuser用户名 | 类似 `SET SESSION AUTHORIZATION` 但是只能切换到普通用户，并且不能切换回来 +|=== + +== 安装 + +[TIP] +源码测试安装环境为 Ubuntu 26.04。需要安装IvorySQL，具体安装步骤请参考： + +=== 源码安装 + +** 设置IvorySQL安装路径 +[source,bash] +---- +export IVY_BIN_DIR=/usr/local/ivorysql/bin +export TEST_DB_DIR=/tmp/test_db +---- + +** 获取源码 +[source,bash] +---- +git clone https://github.com/pgaudit/set_user.git +cd set_user +git checkout REL4_2_0 +---- + +** 编译安装 +[source,bash] +---- +make USE_PGXS=1 PG_CONFIG=${IVY_BIN_DIR}/pg_config clean +make USE_PGXS=1 PG_CONFIG=${IVY_BIN_DIR}/pg_config +make USE_PGXS=1 PG_CONFIG=${IVY_BIN_DIR}/pg_config install +---- + +** 配置 +[source,ini] +---- +shared_preload_libraries = 'set_user' +---- + +=== 创建测试数据库 + +[NOTE] +需要将 `set_user` 添加到 `shared_preload_libraries`。 + +** 初始化数据库 +[source,bash] +---- +# 初始化数据库 +${IVY_BIN_DIR}/initdb -U postgres -D ${TEST_DB_DIR} + +# 将 set_user 添加到 shared_preload_libraries 中，也可以手动配置 +sed -i "s/\(shared_preload_libraries = '.*\)'/\1, set_user'/g" ${TEST_DB_DIR}/ivorysql.conf + +# 修改PG端口和Oracle兼容端口，避免冲突 +cat << EOF >> ${TEST_DB_DIR}/ivorysql.conf +ivorysql.port = 1522 +port = 5433 +EOF + +# 启动测试数据库 +${IVY_BIN_DIR}/pg_ctl -D ${TEST_DB_DIR} -l ${TEST_DB_DIR}/logfile start +---- + +=== 使用测试 + +** 连接数据库 + +[source,bash] +---- +PGHOST=127.0.0.1 PGPORT=1522 PGUSER=postgres $IVY_BIN_DIR/psql +---- + +** 测试示例 + +[TIP] +这里使用默认配置，即白名单为 '*。 + +[source,sql] +---- +CREATE EXTENSION set_user; +LOAD 'set_user'; + +--- 创建两个用户 +CREATE USER dba; +CREATE USER bob; + +-- 让 dba 可以执行 set_user() +GRANT EXECUTE ON FUNCTION set_user(text) TO dba; +GRANT EXECUTE ON FUNCTION set_user(text,text) TO dba; +GRANT EXECUTE ON FUNCTION set_user_u(text) TO dba; + +-- 切换到用户 dba，模拟dba登录 +SET SESSION AUTHORIZATION dba; +SELECT SESSION_USER, CURRENT_USER; +--- session_user | current_user +--- --------------+-------------- +--- dba | dba +--- (1 row) + +--- 提权， 这里使用 set_user() 提权失败，因为目标用户是 superuser +SELECT set_user('postgres'); +--- ERROR: switching to superuser not allowed +--- HINT: Use 'set_user_u' to escalate. + +--- 提权， 这里使用 set_user_u() 提权成功 +SELECT set_user_u('postgres'); +--- set_user_u +--- ------------ +--- OK +--- (1 row) + +--- 查看提权后的用户状态 +SELECT SESSION_USER, CURRENT_USER; +--- session_user | current_user +--- --------------+-------------- +--- dba | postgres +--- (1 row) + +--- 测试重复 set_user，会失败 +SELECT set_user('bob'); +--- ERROR: must reset previous user prior to setting again + +--- 提权中，测试 ALTER SYSTEM 会失败 +ALTER SYSTEM SET wal_level = minimal; +--- ERROR: ALTER SYSTEM blocked by set_user config + +--- 提权中，测试 COPY PROGRAM 会失败 +COPY (select 42) TO PROGRAM 'cat'; +--- ERROR: COPY PROGRAM blocked by set_user config + +--- 提权中，测试 SET log_statement 会失败 +SET log_statement = 'none'; +--- ERROR: "SET log_statement" blocked by set_user config + +--- 恢复初始用户 +SELECT reset_user(); +SELECT SESSION_USER, CURRENT_USER; +--- session_user | current_user +--- --------------+-------------- +--- dba | dba +--- (1 row) + +--- 切换到普通用户bob +SELECT set_user('bob'); +SELECT SESSION_USER, CURRENT_USER; +--- session_user | current_user +--- --------------+-------------- +--- dba | bob +--- (1 row) + +--- 恢复初始用户 +SELECT reset_user(); +SELECT SESSION_USER, CURRENT_USER; +--- session_user | current_user +--- --------------+-------------- +--- dba | dba +--- (1 row) +---- + +查看 `logfile` 能看到提权后的所有操作都被记录了下来，并且日志前缀带有 `AUDIT` 标签。每个用户切换都有一条： `LOG: XXX transitioning to YYY` 对应。 + +[source,log] +---- +2026-07-17 13:55:34.905 CST [2670610] LOG: Role dba transitioning to Superuser Role postgres +2026-07-17 13:55:34.905 CST [2670610] STATEMENT: SELECT set_user_u('postgres'); +2026-07-16 16:01:21.193 CST [2644289] AUDIT: LOG: statement: SELECT SESSION_USER, CURRENT_USER; +2026-07-16 16:01:43.825 CST [2644289] AUDIT: LOG: statement: SELECT set_user('bob'); +2026-07-16 16:01:43.829 CST [2644289] AUDIT: ERROR: must reset previous user prior to setting again +2026-07-16 16:01:43.829 CST [2644289] AUDIT: STATEMENT: SELECT set_user('bob'); +2026-07-16 16:02:20.177 CST [2644289] AUDIT: LOG: statement: ALTER SYSTEM SET wal_level = minimal; +2026-07-16 16:02:20.177 CST [2644289] AUDIT: ERROR: ALTER SYSTEM blocked by set_user config +2026-07-16 16:02:20.177 CST [2644289] AUDIT: STATEMENT: ALTER SYSTEM SET wal_level = minimal; +2026-07-17 13:56:26.924 CST [2670610] STATEMENT: SELECT set_user_u('postgres'); +2026-07-17 14:01:12.017 CST [2670610] AUDIT: LOG: statement: select reset_user(); +2026-07-17 14:01:12.017 CST [2670610] AUDIT: LOG: Superuser Role postgres transitioning to Role dba +2026-07-17 14:18:42.884 CST [2671784] LOG: Role dba transitioning to Role bob +2026-07-17 14:18:42.884 CST [2671784] STATEMENT: select set_user('bob'); +2026-07-17 14:18:53.301 CST [2671784] LOG: Role bob transitioning to Role dba +2026-07-17 14:18:53.301 CST [2671784] STATEMENT: select reset_user(); +2026-07-17 14:26:16.737 CST [2671784] LOG: Role dba transitioning to Role bob +2026-07-17 14:26:16.737 CST [2671784] STATEMENT: SELECT set_user('bob'); +---- + +=== 清理测试数据库 + +[source,bash] +---- +# 关闭测试数据库 +${IVY_BIN_DIR}/pg_ctl -D ${TEST_DB_DIR} -l ${TEST_DB_DIR}/logfile stop + +# 删除测试数据库 +rm -rf ${TEST_DB_DIR} +---- \ No newline at end of file diff --git a/CN/modules/ROOT/pages/master/5.10.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/system_stats.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/5.10.adoc rename to CN/modules/ROOT/pages/master/ecosystem_components/system_stats.adoc diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc new file mode 100644 index 0000000..10d4c8c --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc @@ -0,0 +1,137 @@ + +:sectnums: +:sectnumlevels: 5 + += wal2json + +== 概述 +wal2json 是一个用于 PostgreSQL 逻辑解码的输出插件，这个插件为每个事务生成一个JSON对象。 + +== 安装 + +[TIP] +源码安装环境为 Ubuntu 24.04(x86_64)，环境中已经安装了IvorySQL5及以上版本，安装路径为/usr/ivory-5 + +=== 源码安装 + +[literal] +---- +# 从 https://github.com/eulerto/wal2json/releases/tag/wal2json_2_6 下载 2.6 的源码包 wal2json_2_6.zip + +unzip wal2json_2_6.zip +cd wal2json_2_6 + +# 编译安装插件 +make PG_CONFIG=/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config install + +---- + +[TIP] +如果出现找不到xlocale.h的错误，需要手动修改 /usr/ivory-5/include/postgresql/server/pg_config.h +删除或者注释掉 #define HAVE_XLOCALE_H 1 这一行 + +=== 修改数据库配置文件 + +修改 postgresql.conf 文件 +启用wal_level为logical，并设置最大复制槽数和最大wal发送进程数 + +[literal] +---- +wal_level = logical +max_replication_slots = 10 +max_wal_senders = 10 +---- + +pg_hba.conf不需要修改，有以下内容即可（仅限本地链接测试）： +[literal] +---- + local replication all trust + host replication all 127.0.0.1/32 trust + host replication all ::1/128 trust +---- + +重启数据库后配置生效。 + +== 使用 + +在第一个终端中执行命令： +[literal] +---- +sudo -u ivorysql /usr/ivory-5/bin/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json + +启动监听，实时输出变更的JSON格式 +sudo -u ivorysql /usr/ivory-5/bin/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - +---- + +在第二个终端中连接数据库： +[literal] +---- +/usr/ivory-5/bin/psql -d postgres -p 1521 +---- + +执行下面的SQL语句： +[literal] +---- +CREATE TABLE test_cdc (id int primary key, name varchar(50)); +INSERT INTO test_cdc VALUES (1, 'test1'); +UPDATE test_cdc SET name = 'test1_update' WHERE id = 1; +DELETE FROM test_cdc WHERE id = 1; +DROP TABLE test_cdc; +---- + +此时在第一个终端上可以看到下面的输出： +[literal] +---- +{ + "change": [ + ] +} +{ + "change": [ + { + "kind": "insert", + "schema": "public", + "table": "test_cdc", + "columnnames": ["id", "name"], + "columntypes": ["integer", "sys.oravarcharbyte(50)"], + "columnvalues": [1, "test1"] + } + ] +} +{ + "change": [ + { + "kind": "update", + "schema": "public", + "table": "test_cdc", + "columnnames": ["id", "name"], + "columntypes": ["integer", "sys.oravarcharbyte(50)"], + "columnvalues": [1, "test1_update"], + "oldkeys": { + "keynames": ["id"], + "keytypes": ["integer"], + "keyvalues": [1] + } + } + ] +} +{ + "change": [ + { + "kind": "delete", + "schema": "public", + "table": "test_cdc", + "oldkeys": { + "keynames": ["id"], + "keytypes": ["integer"], + "keyvalues": [1] + } + } + ] +} +{ + "change": [ + ] +} +---- diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/zhparser.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/zhparser.adoc new file mode 100644 index 0000000..533421c --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/zhparser.adoc @@ -0,0 +1,93 @@ +:sectnums: +:sectnumlevels: 5 + += zhparser + +== 概述 +zhparser 是一个用于中文全文搜索的PostgreSQL插件，它基于SCWS（即：简易中文分词系统）实现了一个中文解析器。 + +== 安装 + +[TIP] +源码安装环境为 Ubuntu 24.04(x86_64)，环境中已经安装了IvorySQL，安装路径为/path-to/ivorysql + +=== 源码安装 + +[literal] +---- +首先安装依赖 + +wget http://www.xunsearch.com/scws/down/scws-1.2.3.tar.bz2 +tar xf scws-1.2.3.tar.bz2 +cd scws-1.2.3 + +编译安装scws +./configure +make +sudo make install + +# 使用git下载zhparser源代码，使用master分支 +git clone https://github.com/amutu/zhparser +cd zhparser + +# 编译安装 +make PG_CONFIF=/path-to/ivorysql/bin/pg_config +make PG_CONFIF=/path-to/ivorysql/bin/pg_config install +---- + +== 创建Extension以及全文检索配置，给分词词性绑定词典规则 + +psql 连接到数据库，执行如下命令： +[literal] +---- +-- create the extension +CREATE EXTENSION zhparser; + +-- make test configuration using parser +CREATE TEXT SEARCH CONFIGURATION testzhcfg (PARSER = zhparser); + +-- add token mapping +ALTER TEXT SEARCH CONFIGURATION testzhcfg ADD MAPPING FOR n,v,a,i,e,l WITH simple; +---- + +== 使用 + +[literal] +---- +-- ts_parse +ivorysql=# SELECT * FROM ts_parse('zhparser', 'hello world! 2010年保障房建设在全国范围内获全面启动，从中央到地方纷纷加大 了保障房的建设和投入力度 。2011年，保障房进入了更大规模的建设阶段。住房城乡建设部党组书记、部长姜伟新去年底在全国住房城乡建设工作会议上表示，要继续推进保障性安居工程建设。'); + tokid | token +-------+---------- + 101 | hello + 101 | world + 117 | ! + 101 | 2010 + 113 | 年 + 118 | 保障 + 110 | 房建 + ...... + +-- test to_tsvector +ivorysql=# SELECT to_tsvector('testzhcfg','“今年保障房新开工数量虽然有所下调，但实际的年度在建规模以及竣工规模会超以往年份，相对应的对资金的需求也会 创历>史纪录。”陈国强说。在他看来，与2011年相比，2012年的保障房建设在资金配套上的压力将更为严峻。'); + + to_tsvector + +----------------------------------------------------------------------------------------------------------------------------------------------------- +----------------------------------------------------------------------------------------------------------------------------------------------------- +--------------------------- + '2011':27 '2012':29 '上':35 '下调':7 '严峻':37 '会':14 '会创':20 '保障':1,30 '压力':36 '史':21 '国强':24 '在建':10 '实际':8 '对应':17 '年份':16 '年 +':9 '开工':4 '房':2 '房建':31 '数量':5 '新':3 '有所':6 '相比':28 '看来':26 '竣工':12 '纪录':22 '规模':11,13 '设在':32 '说':25 '资金':18,33 '超':15 ' +套':34 '陈':23 '需求':19 +(1 row) + +-- test to_tsquery +ivorysql=# SELECT to_tsquery('testzhcfg', '保障房资金压力'); + to_tsquery +--------------------------------------- + '保障' <-> '房' <-> '资金' <-> '压力' +(1 row) + +---- + +更多详细使用方法和高级特性，请参阅 https://github.com/amutu/zhparser 。 + diff --git a/CN/modules/ROOT/pages/master/6.5.adoc b/CN/modules/ROOT/pages/master/gb18030.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.5.adoc rename to CN/modules/ROOT/pages/master/gb18030.adoc diff --git a/CN/modules/ROOT/pages/master/3.3.adoc b/CN/modules/ROOT/pages/master/getting-started/daily_maintenance.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/3.3.adoc rename to CN/modules/ROOT/pages/master/getting-started/daily_maintenance.adoc diff --git a/CN/modules/ROOT/pages/master/3.2.adoc b/CN/modules/ROOT/pages/master/getting-started/daily_monitoring.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/3.2.adoc rename to CN/modules/ROOT/pages/master/getting-started/daily_monitoring.adoc diff --git a/CN/modules/ROOT/pages/master/3.1.adoc b/CN/modules/ROOT/pages/master/getting-started/quick_start.adoc similarity index 94% rename from CN/modules/ROOT/pages/master/3.1.adoc rename to CN/modules/ROOT/pages/master/getting-started/quick_start.adoc index 3ae85bc..e1a2b30 100644 --- a/CN/modules/ROOT/pages/master/3.1.adoc +++ b/CN/modules/ROOT/pages/master/getting-started/quick_start.adoc @@ -116,6 +116,8 @@ ivorysql 3238 1551 0 20:35 pts/0 00:00:00 grep --color=auto postgres $ docker pull ivorysql/ivorysql:5.0-ubi8 ``` +TIP: 国内用户也可使用 IvorySQL 社区的 Harbor 镜像服务加速拉取，例如：`docker pull registry.highgo.com/ivorysql/ivorysql:5.0-ubi8`。 + ** 运行IvorySQL ``` $ docker run --name ivorysql -p 5434:5432 -e IVORYSQL_PASSWORD=your_password -d ivorysql/ivorysql:5.0-ubi8 @@ -148,4 +150,4 @@ TIP: Docker运行IvorySQL时，需要添加额外参数，如 psql -d ivorysql - 现在可以开始使用IvorySQL啦！就是这么简单！ -想要获得更多安装方式，请参考xref:master/4.1.adoc[安装指南] +想要获得更多安装方式，请参考xref:master/installation_guide.adoc[安装指南] diff --git a/CN/modules/ROOT/pages/master/4.1.adoc b/CN/modules/ROOT/pages/master/installation_guide.adoc similarity index 95% rename from CN/modules/ROOT/pages/master/4.1.adoc rename to CN/modules/ROOT/pages/master/installation_guide.adoc index 3818f81..6c8fc00 100644 --- a/CN/modules/ROOT/pages/master/4.1.adoc +++ b/CN/modules/ROOT/pages/master/installation_guide.adoc @@ -14,7 +14,7 @@ IvorySQL安装方式包括以下5种： - <<源码安装>> - <> -本章将详细介绍每种方式的安装、运行及卸载过程。想要更快获得IvorySQL，请参阅xref:master/3.1.adoc#快速开始[快速开始]。 +本章将详细介绍每种方式的安装、运行及卸载过程。想要更快获得IvorySQL，请参阅xref:master/getting-started/quick_start.adoc#快速开始[快速开始]。 同样，安装前请先创建一个用户，并赋予其root权限，安装、使用和卸载均以该用户执行。这里以ivorysql用户为例。 @@ -43,6 +43,8 @@ $ sudo dnf install -y ivorysql5-5.0 $ docker pull ivorysql/ivorysql:5.0-ubi8 ``` +TIP: 国内用户也可使用 IvorySQL 社区的 Harbor 镜像服务加速拉取，将上述地址替换为 `registry.highgo.com/ivorysql/ivorysql:5.0-ubi8` 即可。 + ** 运行IvorySQL ``` $ docker run --name ivorysql -p 5434:5432 -e IVORYSQL_PASSWORD=your_password -d ivorysql/ivorysql:5.0-ubi8 @@ -89,7 +91,7 @@ $ sudo yum --disablerepo=* localinstall *.rpm == 源码安装 ** 安装依赖 ``` -$ sudo dnf install -y bison readline-devel zlib-devel openssl-devel +$ sudo dnf install -y bison readline-devel zlib-devel openssl-devel uuid-devel $ sudo dnf groupinstall -y 'Development Tools' ``` ** 获取IvorySQL源代码 diff --git a/CN/modules/ROOT/pages/master/4.5.adoc b/CN/modules/ROOT/pages/master/migration_guide.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/4.5.adoc rename to CN/modules/ROOT/pages/master/migration_guide.adoc diff --git a/CN/modules/ROOT/pages/master/4.4.adoc b/CN/modules/ROOT/pages/master/operation_guide.adoc similarity index 97% rename from CN/modules/ROOT/pages/master/4.4.adoc rename to CN/modules/ROOT/pages/master/operation_guide.adoc index 2eb6f51..211d5fd 100644 --- a/CN/modules/ROOT/pages/master/4.4.adoc +++ b/CN/modules/ROOT/pages/master/operation_guide.adoc @@ -150,6 +150,111 @@ pg_upgrade https://www.postgresql.org/docs/current/pgupgrade.html[文档]概述 这种升级方法可以用内置的逻辑复制工具和外部的逻辑复制系统如pglogical，Slony，Londiste，和Bucardo。 +== 使用 pg_upgrade 将 PostgreSQL 升级到 IvorySQL + +`pg_upgrade` 支持将原生 PostgreSQL 集群升级到 IvorySQL。当源集群为原生 PostgreSQL ，而目标为 IvorySQL 集群时，必须使用 `-g`（`--using-ora-pg`）参数。 + +从 PostgreSQL 升级到 IvorySQL 属于“跨产品”迁移，建议在生产环境操作前完成完整的测试验证，并制定详细的回滚方案。 + +=== 参数说明 + +[cols="1,3"] +|=== +| 参数 | 说明 + +| `-g` / `--using-ora-pg` +| 表示源集群为 PostgreSQL ，而目标为 IvorySQL 集群。启用此选项后，`pg_upgrade` 在连接两端集群时统一使用 PostgreSQL 标准端口，从而正确处理升级。 +|=== + +=== 升级步骤 + +==== 第一步：初始化新 IvorySQL 集群 + +[source,bash] +---- +# 使用 IvorySQL 的 initdb 初始化新集群 +/opt/ivorysql/bin/initdb -D /data/ivorysql/data +---- + +==== 第二步：停止源 PostgreSQL 集群 + +[source,bash] +---- +/usr/lib/postgresql/16/bin/pg_ctl -D /data/pg/data stop +---- + +==== 第三步：运行升级检查（可选） + +使用 `-c` 仅做兼容性检查，不修改任何数据： + +[source,bash] +---- +/opt/ivorysql/bin/pg_upgrade \ + -b /usr/lib/postgresql/16/bin \ # 源 PG 可执行文件目录 + -B /opt/ivorysql/bin \ # 目标 IvorySQL 可执行文件目录 + -d /data/pg/data \ # 源 PG 数据目录 + -D /data/ivorysql/data \ # 目标 IvorySQL 数据目录 + -g \ # 源为 PostgreSQL，目标为 IvorySQL + -c # 仅检查，不升级 +---- + +==== 第四步：执行升级 + +[source,bash] +---- +/opt/ivorysql/bin/pg_upgrade \ + -b /usr/lib/postgresql/16/bin \ + -B /opt/ivorysql/bin \ + -d /data/pg/data \ + -D /data/ivorysql/data \ + -g +---- + +==== 第五步：启动 IvorySQL 并验证 + +[source,bash] +---- +/opt/ivorysql/bin/pg_ctl -D /data/ivorysql/data start + +# 验证数据库是否正常 +/opt/ivorysql/bin/psql -p 5432 -c "SELECT version();" +---- + +==== 第六步：清理旧集群 + +升级完成后，`pg_upgrade` 会生成一个清理脚本： + +[source,bash] +---- +./delete_old_cluster.sh +---- + +=== 常用参数速查 + +[cols="1,2,2"] +|=== +| 参数 | 长选项 | 说明 + +| `-b` | `--old-bindir` | 源集群可执行文件目录 +| `-B` | `--new-bindir` | 目标集群可执行文件目录 +| `-d` | `--old-datadir` | 源集群数据目录 +| `-D` | `--new-datadir` | 目标集群数据目录 +| `-g` | `--using-ora-pg` | 源为 PostgreSQL 升级到 IvorySQL +| `-p` | `--old-port` | 源集群端口 +| `-P` | `--new-port` | 目标集群 PG 端口 +| `-q` | `--old-oraport` | 源集群 Oracle 端口 +| `-Q` | `--new-oraport` | 目标集群 Oracle 端口 +| `-j` | `--jobs` | 并行进程数 +| `-k` | `--link` | 使用硬链接替代文件复制 +| `-c` | `--check` | 仅检查兼容性，不执行升级 +| `-v` | `--verbose` | 输出详细日志 +|=== + +=== 注意事项 + +* `-g` 参数仅在 **源为纯 PostgreSQL 集群**、目标为 **IvorySQL** 时使用。若源集群已是 IvorySQL，则无需此参数。 +* 升级期间源集群和目标集群均须处于停止状态。 +* 升级前务必对源集群做完整备份。 == 管理IvorySQL版本 diff --git a/CN/modules/ROOT/pages/master/oracle_builtin_functions/rawtohex.adoc b/CN/modules/ROOT/pages/master/oracle_builtin_functions/rawtohex.adoc new file mode 100644 index 0000000..dec604e --- /dev/null +++ b/CN/modules/ROOT/pages/master/oracle_builtin_functions/rawtohex.adoc @@ -0,0 +1,55 @@ + +:sectnums: +:sectnumlevels: 5 + + += **功能概述** + +IvorySQL提供兼容Oracle内置函数 ```RAWTOHEX('parameter')``` ，用于将RAW类型转换成十六进制字符串。 + +== 实现原理 + +PostgreSQL 提供的pg_catalog.encode(bytea, 'hex')函数，可直接完成二进制到十六进制的转换。 +考虑到当前系统中已经存在的HEXTORAW函数通过封装pg_catalog.decode 的SQL方式实现，本次开发的 +RAWTOHEX 函数将使用相同的方式来实现，即使用 SQL 函数包装 PostgreSQL 内置函数pg_catalog.encode， +而非编写 C 扩展。 + +raw、text、bytea以及varchar2这四种类型做为输入需要被支持。 +sys.raw 是 bytea 的 domain 类型（typtype = 'd'，typbasetype = bytea），PostgreSQL 支持 domain 到 base type 的隐式转换，RAWTOHEX(bytea) 可自动接受 sys.raw 输入。 +sys.oravarcharchar（即 varchar2）到 pg_catalog.text 存在 IMPLICIT cast（datatype--1.0.sql），RAWTOHEX(text) 可自动接受 varchar2 输入。 +因此，定义两个重载（而不是四个）。 + +``` +sys.rawtohex(bytea) RETURNS varchar2 +sys.rawtohex(text) RETURNS varchar2 +``` + +具体功能则是在 `builtin_functions--1.0.sql` 中实现。 +```sql +/* support rawtohex function for oracle compatibility */ +CREATE OR REPLACE FUNCTION sys.rawtohex(bytea) +RETURNS varchar2 +AS $$ SELECT CASE WHEN pg_catalog.octet_length($1) > 0 THEN upper(pg_catalog.encode($1, 'hex'))::varchar2 END; $$ +LANGUAGE SQL +PARALLEL SAFE +STRICT +IMMUTABLE; + +CREATE OR REPLACE FUNCTION sys.rawtohex(text) +RETURNS varchar2 +AS $$ SELECT CASE WHEN pg_catalog.octet_length($1) > 0 THEN upper(pg_catalog.encode($1::bytea, 'hex'))::varchar2 END; $$ +LANGUAGE SQL +PARALLEL SAFE +STRICT +IMMUTABLE; +``` + +== RAWTOHEX 典型用例 +[cols="8,2"] +|==== +|*用例语句*|*返回值* +|SELECT sys.rawtohex('\xDEADBEEF'::bytea); | DEADBEEF +|SELECT sys.rawtohex('\xFF'::raw); | FF +|SELECT sys.rawtohex('hello'::text); | 68656C6C6F +|SELECT sys.rawtohex('hello'::varchar2); | 68656C6C6F +|==== diff --git a/CN/modules/ROOT/pages/master/oracle_builtin_functions/stragg.adoc b/CN/modules/ROOT/pages/master/oracle_builtin_functions/stragg.adoc new file mode 100644 index 0000000..522a301 --- /dev/null +++ b/CN/modules/ROOT/pages/master/oracle_builtin_functions/stragg.adoc @@ -0,0 +1,139 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += STRAGG 聚合函数的实现 + +== 目的 + +IvorySQL 在 `contrib/ivorysql_ora` 扩展中新增 `sys.stragg(text)` 聚合函数， +实现与 Oracle 同名函数一致的字符串聚合行为：将组内所有非 NULL 值用逗号连接为一个字符串。 + +== 实现说明 + +=== 状态布局设计 + +STRAGG 的聚合状态复用 PostgreSQL 内置 `string_agg` 所使用的 `StringInfo` 布局， +从而可以直接引用 `string_agg_finalfn`、`string_agg_combine`、 +`string_agg_serialize`、`string_agg_deserialize` 四个内置函数， +无需另行实现 finalize、并行合并及序列化逻辑。 + +`StringInfo` 状态的约定如下： + +[source,c] +---- +/* + * data = "," + val1 + "," + val2 + ... + * 首元素前也预置一个逗号，便于 finalfn 统一处理 + * cursor = 1 + * 记录前导分隔符的字节长度（逗号占 1 字节） + * string_agg_finalfn 返回 &data[cursor]，即自动去掉前导逗号 + */ +---- + +=== 转换函数（`stragg_transfn`） + +转换函数位于 +`contrib/ivorysql_ora/src/builtin_functions/misc_functions.c`。 + +[source,c] +---- +PG_FUNCTION_INFO_V1(stragg_transfn); + +Datum +stragg_transfn(PG_FUNCTION_ARGS) +{ + StringInfo state; + MemoryContext aggcontext; + MemoryContext oldcontext; + + if (!AggCheckCallContext(fcinfo, &aggcontext)) + elog(ERROR, "stragg_transfn called in non-aggregate context"); + + state = PG_ARGISNULL(0) ? NULL : (StringInfo) PG_GETARG_POINTER(0); + + /* 跳过 NULL 输入，与 Oracle STRAGG 行为一致 */ + if (!PG_ARGISNULL(1)) + { + text *value = PG_GETARG_TEXT_PP(1); + + if (state == NULL) + { + oldcontext = MemoryContextSwitchTo(aggcontext); + state = makeStringInfo(); + MemoryContextSwitchTo(oldcontext); + + /* 首个值：预置分隔符，cursor 记录其长度 */ + appendStringInfoChar(state, ','); + state->cursor = 1; + } + else + { + appendStringInfoChar(state, ','); + } + + appendBinaryStringInfo(state, VARDATA_ANY(value), VARSIZE_ANY_EXHDR(value)); + } + + if (state) + PG_RETURN_POINTER(state); + PG_RETURN_NULL(); +} +---- + +关键设计点： + +* 状态在聚合上下文（`aggcontext`）中分配，生命周期覆盖整个聚合过程。 +* `StringInfo` 内部缓冲区按需翻倍扩容，追加操作均摊 O(1)，总时间复杂度 O(N)， + 优于纯 SQL 拼接方案的 O(N²)。 +* NULL 输入由 `PG_ARGISNULL(1)` 判断并跳过，状态不受污染。 + +=== SQL 定义 + +转换函数和聚合定义位于 +`contrib/ivorysql_ora/src/builtin_functions/builtin_functions--1.0.sql`。 + +[source,sql] +---- +CREATE FUNCTION sys.stragg_transfn(internal, text) +RETURNS internal +AS 'MODULE_PATHNAME', 'stragg_transfn' +LANGUAGE C +CALLED ON NULL INPUT +PARALLEL SAFE; + +CREATE AGGREGATE sys.stragg(text) ( + SFUNC = sys.stragg_transfn, + STYPE = internal, + FINALFUNC = string_agg_finalfn, + COMBINEFUNC = string_agg_combine, + SERIALFUNC = string_agg_serialize, + DESERIALFUNC = string_agg_deserialize, + PARALLEL = SAFE +); +---- + +`FINALFUNC`、`COMBINEFUNC`、`SERIALFUNC`、`DESERIALFUNC` 均直接引用 PostgreSQL +内置的 `string_agg` 系列函数，因为 STRAGG 与 `string_agg` 使用完全相同的 +`StringInfo` 状态格式。 + +=== 并行聚合支持 + +通过指定 `COMBINEFUNC = string_agg_combine` 和序列化/反序列化函数， +STRAGG 支持 PostgreSQL 的并行聚合执行路径。各并行工作进程独立维护局部 +`StringInfo` 状态，最终由 leader 进程通过 `string_agg_combine` 合并。 + +=== 回归测试 + +测试文件：`contrib/ivorysql_ora/sql/ora_stragg.sql`， +对应预期输出：`contrib/ivorysql_ora/expected/ora_stragg.out`。 + +在 `Makefile` 的 `ORA_REGRESS` 列表中已加入 `ora_stragg`， +可通过以下命令运行： + +[source,bash] +---- +cd contrib/ivorysql_ora +make installcheck +---- diff --git a/CN/modules/ROOT/pages/master/6.4.1.adoc b/CN/modules/ROOT/pages/master/oracle_builtin_functions/sys_context.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.4.1.adoc rename to CN/modules/ROOT/pages/master/oracle_builtin_functions/sys_context.adoc diff --git a/CN/modules/ROOT/pages/master/6.4.2.adoc b/CN/modules/ROOT/pages/master/oracle_builtin_functions/userenv.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/6.4.2.adoc rename to CN/modules/ROOT/pages/master/oracle_builtin_functions/userenv.adoc diff --git a/CN/modules/ROOT/pages/master/7.6.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/anonymous_block.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.6.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/anonymous_block.adoc diff --git a/CN/modules/ROOT/pages/master/7.8.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/builtin_types_functions.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.8.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/builtin_types_functions.adoc diff --git a/CN/modules/ROOT/pages/master/7.22.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_call_into.adoc similarity index 96% rename from CN/modules/ROOT/pages/master/7.22.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_call_into.adoc index cfd1820..f3cf534 100644 --- a/CN/modules/ROOT/pages/master/7.22.adoc +++ b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_call_into.adoc @@ -115,6 +115,14 @@ ivorysql=# PRINT default_result; === 在参数或INTO子句中引用绑定变量 [source,sql] ---- +-- 创建独立函数 stand_alone_func +CREATE OR REPLACE FUNCTION stand_alone_func(p_input NUMBER) +RETURN NUMBER +IS +BEGIN + RETURN p_input * 2; +END; +/ -- 设置输入绑定变量 ivorysql=# VARIABLE input_num NUMBER = 7; -- 使用绑定变量作为参数调用函数 diff --git a/CN/modules/ROOT/pages/master/7.3.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_case_conversion.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.3.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_case_conversion.adoc diff --git a/CN/modules/ROOT/pages/master/oracle_compatibility/compat_create_index_online.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_create_index_online.adoc new file mode 100644 index 0000000..fb3ea4b --- /dev/null +++ b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_create_index_online.adoc @@ -0,0 +1,115 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += 索引 ONLINE 参数 + +== 目的 + +本文档解释 IvorySQL 中 `ONLINE` 参数在创建索引时的功能，实现该参数功能以保持与 Oracle 行为一致。 + +== 功能说明 + +- `ONLINE` 参数在创建索引时被指定，可允许 DML 并发执行，类似 PostgreSQL 中的 `CONCURRENTLY`，但不能与 `CONCURRENTLY` 同时出现。 +- `ONLINE` 必须支持出现在列列表 `)` 之后、`WHERE` 子句之前的任意位置，且与 `TABLESPACE`、`PARALLEL`（如已支持）等其他属性的顺序无关。 +- 临时表上的 `CREATE INDEX ... ONLINE` 自动降级为普通构建（不报错，与 `CONCURRENTLY` 行为一致）。 +- 分区表上的 `CREATE INDEX ... ONLINE` **自动降级为普通构建**（不报错）。 + +== 测试用例 + +=== 测试环境准备 + +[source,sql] +---- +-- 基础测试表 +CREATE TABLE tbl_ci_online ( + id NUMBER(10) PRIMARY KEY, + name VARCHAR2(100), + dept_id NUMBER(10), + salary NUMBER(10,2), + status VARCHAR2(20) +); +INSERT INTO tbl_ci_online + SELECT g, 'name'||g, MOD(g,20), g*100.0, CASE WHEN MOD(g,2)=0 THEN 'ACTIVE' ELSE 'INACTIVE' END + FROM generate_series(1, 1000) g; + +-- 唯一索引测试表 +CREATE TABLE tbl_ci_unique ( + id NUMBER(10) PRIMARY KEY, + email VARCHAR2(200) NOT NULL +); +INSERT INTO tbl_ci_unique SELECT g, 'user'||g||'@example.com' FROM generate_series(1, 200) g; + +-- 分区表 +CREATE TABLE tbl_ci_part ( + id NUMBER(10), + region VARCHAR2(20) +) PARTITION BY RANGE (id); +CREATE TABLE tbl_ci_part_p1 PARTITION OF tbl_ci_part FOR VALUES FROM (1) TO (501); +CREATE TABLE tbl_ci_part_p2 PARTITION OF tbl_ci_part FOR VALUES FROM (501) TO (1001); +INSERT INTO tbl_ci_part SELECT g, CASE WHEN g<=500 THEN 'east' ELSE 'west' END + FROM generate_series(1, 1000) g; +---- + +=== 基础 ONLINE 构建 + +[source,sql] +---- +-- 最简单形式 +CREATE INDEX idx_online_name ON tbl_ci_online (name) ONLINE; + +-- 验证索引已建立并有效 +SELECT indisvalid FROM pg_index + WHERE indexrelid = 'idx_online_name'::regclass; +-- 期望：t + +-- 多列索引 +CREATE INDEX idx_online_multi ON tbl_ci_online (dept_id, salary) ONLINE; + +-- 表达式索引 +CREATE INDEX idx_online_expr ON tbl_ci_online (lower(name)) ONLINE; +---- + +=== 分区表 ONLINE + +[source,sql] +---- +-- 分区表父级索引 ONLINE +-- 注：ONLINE 在分区表上静默降级为普通构建，与 Oracle 行为一致 +CREATE INDEX idx_part_online ON tbl_ci_part (id) ONLINE; + +-- 验证索引已创建且有效（降级为普通构建，父级索引 + 各分区子索引均 VALID） +SELECT c.relname, i.indisvalid +FROM pg_index i +JOIN pg_class c ON c.oid = i.indexrelid +WHERE c.relname LIKE '%idx_part_online%' +ORDER BY c.relname; +-- 期望：idx_part_online（父）及两个分区子索引 indisvalid = t + +-- 分区表 ONLINE + TABLESPACE +CREATE INDEX idx_part_online_tbs ON tbl_ci_part (region) ONLINE TABLESPACE pg_default; +---- + +=== CONCURRENTLY 与 ONLINE 互斥 + +[source,sql] +---- +-- CONCURRENTLY 在前，ONLINE 在后 +CREATE INDEX CONCURRENTLY idx_both ON tbl_ci_online (name) ONLINE; +-- 期望：ERROR: cannot use both CONCURRENTLY and ONLINE + +-- 验证原有 CONCURRENTLY 语法不受影响 +CREATE INDEX CONCURRENTLY idx_still_conc ON tbl_ci_online (dept_id); +-- 期望：成功 +---- + +=== 测试环境清理 + +[source,sql] +---- +DROP TABLE IF EXISTS tbl_ci_online CASCADE; +DROP TABLE IF EXISTS tbl_ci_part CASCADE; +DROP TABLE IF EXISTS tbl_ci_unique CASCADE; +---- + diff --git a/CN/modules/ROOT/pages/master/7.21.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_empty_string_to_null.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.21.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_empty_string_to_null.adoc diff --git a/CN/modules/ROOT/pages/master/7.18.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_force_view.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.18.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_force_view.adoc diff --git a/CN/modules/ROOT/pages/master/7.7.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_function_procedure.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.7.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_function_procedure.adoc diff --git a/CN/modules/ROOT/pages/master/7.5.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_like_operator.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.5.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_like_operator.adoc diff --git a/CN/modules/ROOT/pages/master/7.19.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_nested_function.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.19.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_nested_function.adoc diff --git a/CN/modules/ROOT/pages/master/7.17.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_nls_parameter.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.17.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_nls_parameter.adoc diff --git a/CN/modules/ROOT/pages/master/7.15.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_out_parameter.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.15.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_out_parameter.adoc diff --git a/CN/modules/ROOT/pages/master/oracle_compatibility/compat_read_only_view.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_read_only_view.adoc new file mode 100644 index 0000000..1731336 --- /dev/null +++ b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_read_only_view.adoc @@ -0,0 +1,168 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += Read Only View + +== 目的 + +本文档解释 IvorySQL 中 `WITH READ ONLY` 视图的功能，实现只读视图功能以保持与 Oracle 行为一致。 + +== 功能说明 + +- `WITH READ ONLY`：创建视图时指定此选项，可防止对视图进行 INSERT、UPDATE、DELETE 和 MERGE 操作。 +- 存储方式：`read_only=true` 存储在视图的 `reloptions` 中。 +- 互斥性：`WITH READ ONLY` 和 `WITH CHECK OPTION` 互斥，不能同时指定。 +- Force View 支持：可以与 `FORCE VIEW` 一起使用，只读属性会在视图成功编译后生效。 +- CREATE OR REPLACE 行为：如果使用 `CREATE OR REPLACE VIEW` 重新创建视图时未指定 `WITH READ ONLY`，则会清除只读属性。 + +== 测试用例 + +=== 创建只读视图 + +[source,sql] +---- +-- 创建基础表 +CREATE TABLE t_ro (a int, b text); +INSERT INTO t_ro VALUES (1, 'hello'), (2, 'world'); + +-- 创建只读视图，SELECT 成功 +CREATE VIEW ro_view AS SELECT * FROM t_ro WITH READ ONLY; +SELECT * FROM ro_view ORDER BY a; +-- 期望输出： +-- a | b +-- ---+------- +-- 1 | hello +-- 2 | world +---- + +=== 验证 DML 被阻止 + +[source,sql] +---- +-- INSERT 被阻止 +INSERT INTO ro_view VALUES (3, 'fail'); +-- 期望输出：ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. + +-- UPDATE 被阻止 +UPDATE ro_view SET b = 'fail' WHERE a = 1; +-- 期望输出：ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. + +-- DELETE 被阻止 +DELETE FROM ro_view WHERE a = 1; +-- 期望输出：ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. +---- + +=== MERGE 命令被阻止 + +[source,sql] +---- +-- MERGE with INSERT action +MERGE INTO ro_view USING (SELECT 4 AS a, 'merge_ins' AS b) AS src +ON (ro_view.a = src.a) +WHEN NOT MATCHED THEN INSERT VALUES (src.a, src.b); +-- 期望输出：ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. + +-- MERGE with UPDATE action +MERGE INTO ro_view USING (SELECT 1 AS a, 'merge_upd' AS b) AS src +ON (ro_view.a = src.a) +WHEN MATCHED THEN UPDATE SET b = src.b; +-- 期望输出：ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. +---- + +=== CREATE OR REPLACE 行为 + +[source,sql] +---- +-- 重新创建视图时保留 WITH READ ONLY：DML 仍然被阻止 +CREATE OR REPLACE VIEW ro_view AS SELECT a, b FROM t_ro WITH READ ONLY; +INSERT INTO ro_view VALUES (3, 'fail'); +-- 期望输出：ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. + +-- 重新创建视图时不指定 WITH READ ONLY：只读属性被清除，视图变为可更新 +CREATE OR REPLACE VIEW ro_view AS SELECT a, b FROM t_ro; +INSERT INTO ro_view VALUES (3, 'now_writable'); +SELECT * FROM ro_view ORDER BY a; +-- 期望输出： +-- a | b +-- ---+-------------- +-- 1 | hello +-- 2 | world +-- 3 | now_writable +---- + +=== FORCE VIEW 与 WITH READ ONLY + +[source,sql] +---- +-- 创建时基础表不存在，视图为占位符 +CREATE FORCE VIEW force_ro_view AS SELECT * FROM nonexistent_for_ro WITH READ ONLY; +-- 期望输出：WARNING: View created with compilation errors + +-- 创建基础表 +CREATE TABLE nonexistent_for_ro (a int, b text); + +-- 显式编译视图 +ALTER VIEW force_ro_view COMPILE; + +-- 编译成功后 DML 被阻止 +INSERT INTO force_ro_view VALUES (1, 'fail'); +-- 期望输出：ERROR: cannot modify view "force_ro_view" +-- HINT: The view is defined as read-only. +---- + +=== 递归视图与 WITH READ ONLY + +[source,sql] +---- +CREATE RECURSIVE VIEW ro_recursive_view (a) AS + SELECT 1 + UNION ALL + SELECT a + 1 FROM ro_recursive_view WHERE a < 3 +WITH READ ONLY; + +SELECT * FROM ro_recursive_view ORDER BY a; +-- 期望输出： +-- a +-- --- +-- 1 +-- 2 +-- 3 + +INSERT INTO ro_recursive_view VALUES (99); +-- 期望输出：ERROR: cannot modify view "ro_recursive_view" +-- HINT: The view is defined as read-only. +---- + +=== 验证 reloptions 存储 + +[source,sql] +---- +SELECT relname, reloptions +FROM pg_class +WHERE relname IN ('ro_view', 'ro_recursive_view', 'force_ro_view') +ORDER BY relname; +-- 期望输出： +-- relname | reloptions +-- -------------------+------------------ +-- force_ro_view | {read_only=true} +-- ro_recursive_view | {read_only=true} +-- ro_view | +---- + +=== 清理 + +[source,sql] +---- +DROP VIEW IF EXISTS ro_view; +DROP VIEW IF EXISTS force_ro_view; +DROP TABLE t_ro; +DROP TABLE IF EXISTS nonexistent_for_ro; +---- diff --git a/CN/modules/ROOT/pages/master/7.14.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_rowid.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.14.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_rowid.adoc diff --git a/CN/modules/ROOT/pages/master/oracle_compatibility/compat_stragg.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_stragg.adoc new file mode 100644 index 0000000..08631b6 --- /dev/null +++ b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_stragg.adoc @@ -0,0 +1,172 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += STRAGG 聚合函数 + +== 目的 + +本文档说明 IvorySQL 中 `sys.stragg` 聚合函数的功能，该函数以兼容 Oracle 的方式 +将组内多行字符串值拼接为一个逗号分隔的字符串。 + +== 功能说明 + +* `sys.stragg(text)` 是一个聚合函数，将同一分组中所有非 NULL 的文本值用逗号（`,`）连接。 +* NULL 值被自动忽略，不出现在结果中，也不产生多余的逗号。 +* 分组内所有值均为 NULL 或分组为空集时，返回 `NULL`（而非空字符串）。 +* 不保证输出顺序；如需确定顺序，可在聚合调用中使用 `ORDER BY` 子句（PostgreSQL 扩展语法）。 +* 函数定义于 `sys` 模式，在 Oracle 兼容模式（不需要sys前缀）和 PG 模式下均可使用。 + +== 语法 + +[source,sql] +---- +sys.stragg(expr [ORDER BY sort_expr [ASC | DESC] [, ...]]) +---- + +[cols="1,3"] +|=== +|参数 |说明 + +|`expr` +|任意可隐式转换为 `text` 的表达式；NULL 值将被跳过。 + +|`ORDER BY` +|可选，指定组内拼接顺序。省略时顺序不确定。 +|=== + +返回类型：`text` + +== 测试用例 + +=== 测试环境准备 + +[source,sql] +---- +CREATE TABLE stragg_test (dept TEXT, name TEXT); +INSERT INTO stragg_test VALUES ('HR', 'Alice'); +INSERT INTO stragg_test VALUES ('HR', 'Bob'); +INSERT INTO stragg_test VALUES ('HR', 'Carol'); +INSERT INTO stragg_test VALUES ('IT', 'Dave'); +INSERT INTO stragg_test VALUES ('IT', 'Eve'); +---- + +=== 基础聚合 + +[source,sql] +---- +-- 单组聚合，使用 ORDER BY 保证结果确定 +SELECT sys.stragg(name ORDER BY name) FROM stragg_test WHERE dept = 'HR'; +-- 期望：Alice,Bob,Carol + +-- 配合 GROUP BY 按部门分组 +SELECT dept, sys.stragg(name ORDER BY name) +FROM stragg_test +GROUP BY dept +ORDER BY dept; +-- 期望： +-- HR | Alice,Bob,Carol +-- IT | Dave,Eve +---- + +=== NULL 值处理 + +[source,sql] +---- +-- 插入一行 NULL 值 +INSERT INTO stragg_test VALUES ('HR', NULL); + +-- NULL 被忽略，结果与无 NULL 时相同 +SELECT sys.stragg(name ORDER BY name) FROM stragg_test WHERE dept = 'HR'; +-- 期望：Alice,Bob,Carol + +-- 所有值均为 NULL 时返回 NULL +SELECT sys.stragg(name) IS NULL FROM stragg_test WHERE name IS NULL; +-- 期望：t +---- + +=== 空集处理 + +[source,sql] +---- +-- 无匹配行时返回 NULL +SELECT sys.stragg(name) IS NULL FROM stragg_test WHERE dept = 'NONE'; +-- 期望：t +---- + +=== 单值 + +[source,sql] +---- +-- 组内只有一个值时，结果中不含逗号 +SELECT sys.stragg(name) FROM stragg_test WHERE dept = 'IT' AND name = 'Dave'; +-- 期望：Dave +---- + +=== PG 兼容模式下使用 + +[source,sql] +---- +SET ivorysql.compatible_mode = pg; +SELECT sys.stragg(name ORDER BY name) FROM stragg_test WHERE dept = 'HR'; +-- 期望：Alice,Bob,Carol +RESET ivorysql.compatible_mode; +---- + +=== 测试环境清理 + +[source,sql] +---- +DROP TABLE stragg_test; +---- + +== 与 Oracle 的行为差异 + +[cols="1,2,2"] +|=== +|场景 |Oracle STRAGG |IvorySQL sys.stragg + +|空集 +|`NULL` +|`NULL`（一致） + +|全 NULL 分组 +|`NULL` +|`NULL`（一致） + +|NULL 值 +|忽略 +|忽略（一致） + +|拼接顺序 +|不确定 +|不确定；可用 `ORDER BY` 子句指定 + +|`ORDER BY` 子句 +|不支持（官方无此语法） +|支持（PostgreSQL 聚合扩展语法） +|=== + +== 与 LISTAGG 的比较 + +[cols="1,2,2"] +|=== +|特性 |`LISTAGG`（Oracle / IvorySQL）|`STRAGG`（Oracle / IvorySQL） + +|分隔符 +|可指定任意分隔符 +|固定为逗号 + +|`ORDER BY` +|通过 `WITHIN GROUP (ORDER BY ...)` 指定 +|不保证顺序（IvorySQL 支持 PostgreSQL 聚合 `ORDER BY`） + +|结果长度限制 +|Oracle 限制 4000 字节（IvorySQL 通过 `ora_listagg_check` 检查） +|无限制 + +|使用场景 +|需要精确控制分隔符和顺序的场合 +|快速拼接，历史遗留代码迁移 +|=== diff --git a/CN/modules/ROOT/pages/master/7.20.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_sys_guid.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.20.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_sys_guid.adoc diff --git a/CN/modules/ROOT/pages/master/7.16.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/compat_type_rowtype.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.16.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/compat_type_rowtype.adoc diff --git a/CN/modules/ROOT/pages/master/7.13.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/invisible_column.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.13.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/invisible_column.adoc diff --git a/CN/modules/ROOT/pages/master/7.12.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/package.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.12.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/package.adoc diff --git a/CN/modules/ROOT/pages/master/7.9.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/port_ip.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.9.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/port_ip.adoc diff --git a/CN/modules/ROOT/pages/master/7.11.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/sequence.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.11.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/sequence.adoc diff --git a/CN/modules/ROOT/pages/master/oracle_compatibility/with_function_procedure.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/with_function_procedure.adoc new file mode 100644 index 0000000..1899904 --- /dev/null +++ b/CN/modules/ROOT/pages/master/oracle_compatibility/with_function_procedure.adoc @@ -0,0 +1,313 @@ +:sectnums: +:sectnumlevels: 5 + += WITH FUNCTION/PROCEDURE + +== 目的 + +本文档解释 IvorySQL 中 `WITH FUNCTION` 和 `WITH PROCEDURE` 的用途，实现 Oracle 风格的 SQL 内嵌 PL/SQL 函数和过程功能。 + +`WITH FUNCTION/PROCEDURE` 是 Oracle 数据库的 Subquery Factoring with PL/SQL Declarations 特性，允许在 SQL 的 WITH 子句（公共表表达式，CTE）中直接定义 PL/SQL 函数和过程。 + +== 功能说明 + +=== 基本语法 + +在 Oracle 兼容模式（`compatible_db = ORA_PARSER`）下，WITH 子句支持以下扩展语法： + +[source,sql] +---- +WITH + FUNCTION func_name ( [ param_list ] ) RETURN return_type + { IS | AS } + [ declare_section ] + BEGIN + statements + END [ func_name ] ; + + PROCEDURE proc_name ( [ param_list ] ) + { IS | AS } + [ declare_section ] + BEGIN + statements + END [ proc_name ] ; + + cte_name AS ( SELECT ... ) +SELECT ... +---- + +=== 核心特性 + +- **作用域限制**：函数/过程在 WITH 子句中定义，作用域仅限于当前 SQL 语句 +- **混合排列**：函数/过程定义可与 CTE（AS (SELECT ...)）混合使用 +- **完整 PL/SQL 语法**：函数体支持完整的 PL/SQL 语法（BEGIN...END） +- **不写入系统目录**：执行结束后自动销毁，不持久化到 `pg_proc` +- **仅 SELECT 上下文**：适用于 SELECT 语句及 INSERT...SELECT 语句；位于 UPDATE、DELETE、MERGE 之前时报错（与 Oracle 行为一致） + +=== 参数模式 + +内嵌过程支持标准 Oracle 参数模式： + +- `IN`（默认）：输入参数 +- `OUT`：输出参数 +- `IN OUT`：双向参数 + +内嵌函数仅支持`IN`类型输入参数。 + +== 支持的语句类型 + +WITH 内嵌函数/过程仅允许出现在以下顶层语句中： + +- `SELECT` 语句（最常见） +- `INSERT ... SELECT` 语句（Oracle 兼容形式，WITH FUNCTION 位于 INSERT INTO 之后） + +以下语句**不支持**（Oracle 不允许，报 `ERRCODE_FEATURE_NOT_SUPPORTED`）： + +- `WITH FUNCTION ... UPDATE ...`：WITH FUNCTION 不能位于 UPDATE 之前 +- `WITH FUNCTION ... DELETE ...`：WITH FUNCTION 不能位于 DELETE 之前 +- `WITH FUNCTION ... MERGE ...`：WITH FUNCTION 不能位于 MERGE 之前 + +如需在 DML 中使用可复用逻辑，应定义 schema 级别的函数（`CREATE FUNCTION`）。 + +== 语法示例 + +=== 最简单的内嵌函数 + +[source,sql] +---- +WITH + FUNCTION double_it(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n * 2; END; +SELECT double_it(5) FROM dual; +-- 期望输出：10 +---- + +=== 函数与 CTE 混合 + +[source,sql] +---- +WITH + FUNCTION tax(amt NUMBER) RETURN NUMBER AS + BEGIN RETURN amt * 0.1; END; + orders AS (SELECT 100 AS amount) +SELECT amount, tax(amount) FROM orders; +-- 期望输出：100 | 10 +---- + +=== 多个内嵌函数 + +[source,sql] +---- +WITH + FUNCTION add1(n NUMBER) RETURN NUMBER AS BEGIN RETURN n+1; END; + FUNCTION mul2(n NUMBER) RETURN NUMBER AS BEGIN RETURN n*2; END; +SELECT mul2(add1(3)) FROM dual; +-- 期望输出：8 +---- + +=== 递归函数 + +[source,sql] +---- +WITH + FUNCTION factorial(n NUMBER) RETURN NUMBER AS + BEGIN + IF n <= 1 THEN RETURN 1; END IF; + RETURN n * factorial(n-1); + END; +SELECT factorial(5) FROM dual; +-- 期望输出：120 +---- + +=== OUT 参数（仅限 PROCEDURE） + +WITH FUNCTION 不允许声明 `OUT` / `IN OUT` 参数（与 Oracle ORA-06572 行为一致）； +仅 WITH PROCEDURE 允许 `OUT` / `IN OUT` 参数。 + +[source,sql] +---- +-- 正确：PROCEDURE 可声明 OUT 参数 +WITH + PROCEDURE swap(val IN NUMBER, result OUT NUMBER) AS + BEGIN + result := val * 10; + END; +SELECT 1 FROM dual; -- 过程由同一 WITH 块内的其他子程序调用，不直接出现在 SELECT 表达式中 + +-- 错误：FUNCTION 不允许 OUT 参数 +WITH + FUNCTION bad_func(val NUMBER, result OUT NUMBER) RETURN NUMBER AS + BEGIN RETURN val; END; +SELECT bad_func(5) FROM dual; +-- 期望输出：ERROR: WITH FUNCTION "bad_func" cannot declare OUT or IN OUT parameters +---- + +=== 默认参数值 + +[source,sql] +---- +WITH + FUNCTION calc(n NUMBER DEFAULT 10) RETURN NUMBER AS + BEGIN RETURN n * 2; END; +SELECT calc() FROM dual; +-- 期望输出：20 +---- + +=== 异常处理 + +[source,sql] +---- +WITH + FUNCTION safe_div(a NUMBER, b NUMBER) RETURN NUMBER AS + BEGIN + RETURN a / b; + EXCEPTION + WHEN OTHERS THEN RETURN NULL; + END; +SELECT safe_div(1, 0) FROM dual; +-- 期望输出：NULL +---- + +=== 与 DML 集成 + +[source,sql] +---- +-- 允许：INSERT INTO ... WITH FUNCTION ... SELECT ...（Oracle 兼容形式） +WITH + FUNCTION get_bonus(sal NUMBER) RETURN NUMBER AS + BEGIN RETURN sal * 1.2; END; +INSERT INTO emp_bonus (empno, bonus) +SELECT empno, get_bonus(sal) FROM emp WHERE deptno = 10; +---- + +NOTE: Oracle 不允许 WITH FUNCTION 位于 UPDATE、DELETE、MERGE 之前。 +IvorySQL 遵循相同限制，此类用法会报 `ERRCODE_FEATURE_NOT_SUPPORTED` 错误。 + +== 作用域与可见性 + +=== 作用域规则 + +- 内嵌函数/过程的作用域仅限当前 SQL 语句（及其子查询） +- 函数/过程可相互引用（前向声明后定义，支持互递归） +- 函数/过程不能与当前数据库中已有的同名同签名函数冲突（WITH 定义优先） +- 同一 WITH 子句中不允许定义同名、同参数类型的函数/过程 + +=== 子查询中可见 + +[source,sql] +---- +WITH + FUNCTION add_tax(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n * 1.1; END; +SELECT * FROM (SELECT add_tax(amount) AS total FROM orders); +---- + +== 与现有功能的关系 + +| 现有功能 | 关系 | +|---------|------| +| PL/iSQL 嵌套子程序 | 直接复用：利用现有编译/执行基础设施 | +| 标准 CTE（WITH...AS (SELECT ...)） | 共存：在同一 WITH 子句中混用 | +| `RECURSIVE` CTE | 共存：WITH RECURSIVE 与内嵌函数可同时使用 | +| Oracle Package | 类似：Package 的过程/函数也是 session 级临时注册 | +| PL/iSQL `CREATE FUNCTION` | 不同：WITH 内嵌函数不持久化，不写入系统目录 | + +== 错误处理 + +=== 重复定义 + +[source,sql] +---- +WITH + FUNCTION dup(n NUMBER) RETURN NUMBER AS BEGIN RETURN n; END; + FUNCTION dup(n NUMBER) RETURN NUMBER AS BEGIN RETURN n * 2; END; +SELECT dup(1) FROM dual; +-- 期望输出：ERROR: WITH clause function "dup" is defined more than once with the same argument types +---- + +=== 在 PG_PARSER 模式下使用 + +[source,sql] +---- +-- 在 PG_PARSER 模式下尝试使用 WITH FUNCTION 语法 +SET compatible_db = PG_PARSER; +WITH FUNCTION foo(n NUMBER) RETURN NUMBER AS BEGIN RETURN n; END; +SELECT foo(1); +-- 期望输出：ERROR: syntax error at or near "FUNCTION" +---- + +=== 函数体语法错误 + +[source,sql] +---- +WITH + FUNCTION broken(n NUMBER) RETURN NUMBER AS + BEGIN + RETRUN n; -- 拼写错误 + END; +SELECT broken(1) FROM dual; +-- 期望输出：ERROR: syntax error at or near "RETRUN" +---- + +=== 表函数用法拒绝 + +[source,sql] +---- +WITH + FUNCTION get_rows(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n; END; +SELECT * FROM get_rows(5); +-- 期望输出：ERROR: WITH clause function cannot be used as a table or set-returning function +---- + +=== 限定名拒绝 + +[source,sql] +---- +WITH + FUNCTION public.qual_func(n NUMBER) RETURN NUMBER IS + BEGIN RETURN n; END; +SELECT qual_func(1) FROM dual; +-- 期望输出：ERROR: qualified name is not allowed in WITH FUNCTION declaration +---- + +== EXPLAIN 输出 + +=== 基本输出 + +[source,sql] +---- +EXPLAIN WITH + FUNCTION add_one(n NUMBER) RETURN NUMBER AS BEGIN RETURN n + 1; END; +SELECT add_one(5) FROM dual; +-- 期望输出包含：WITH Function: add_one(number) RETURN number +---- + +=== VERBOSE 模式 + +[source,sql] +---- +EXPLAIN (VERBOSE ON) WITH + FUNCTION double(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n * 2; END; +SELECT double(3) FROM dual; +-- 期望输出包含：Body: BEGIN RETURN n * 2; END +---- + +== 限制与约束 + +1. **仅 Oracle 解析器模式**：该特性仅在 `compatible_db = ORA_PARSER` 时生效 +2. **不写入系统目录**：内嵌函数/过程在语句执行期间动态注册，执行结束后撤销 +3. **事务安全**：注册和撤销对事务完全透明，不产生 WAL 日志 +4. **并发安全**：多个并发会话各自拥有独立的内嵌函数/过程注册上下文 +5. **作用域限制**：内嵌函数不能在定义它的语句外部调用 +6. **不支持多态参数**：声明 `ANYELEMENT` 等多态参数会在调用时失败 +7. **不支持表函数用法**：`SELECT * FROM with_func(...)` 会被拒绝 + +== 清理 + +[source,sql] +---- +-- WITH FUNCTION/PROCEDURE 的作用域随语句结束自动清理 +-- 无需手动执行清理操作 +---- diff --git a/CN/modules/ROOT/pages/master/7.10.adoc b/CN/modules/ROOT/pages/master/oracle_compatibility/xml_functions.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/7.10.adoc rename to CN/modules/ROOT/pages/master/oracle_compatibility/xml_functions.adoc diff --git a/CN/modules/ROOT/pages/master/110.adoc b/CN/modules/ROOT/pages/master/pg_reference/pg_functions_reference.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/110.adoc rename to CN/modules/ROOT/pages/master/pg_reference/pg_functions_reference.adoc diff --git a/CN/modules/ROOT/pages/master/100.adoc b/CN/modules/ROOT/pages/master/pg_reference/pg_parameters_reference.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/100.adoc rename to CN/modules/ROOT/pages/master/pg_reference/pg_parameters_reference.adoc diff --git a/CN/modules/ROOT/pages/master/33.adoc b/CN/modules/ROOT/pages/master/problem_report_guide.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/33.adoc rename to CN/modules/ROOT/pages/master/problem_report_guide.adoc diff --git a/CN/modules/ROOT/pages/master/1.adoc b/CN/modules/ROOT/pages/master/release_notes.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/1.adoc rename to CN/modules/ROOT/pages/master/release_notes.adoc diff --git a/CN/modules/ROOT/pages/master/9.adoc b/CN/modules/ROOT/pages/master/tools_reference.adoc similarity index 100% rename from CN/modules/ROOT/pages/master/9.adoc rename to CN/modules/ROOT/pages/master/tools_reference.adoc diff --git a/EN/modules/ROOT/nav.adoc b/EN/modules/ROOT/nav.adoc index 743a06a..8194fcd 100644 --- a/EN/modules/ROOT/nav.adoc +++ b/EN/modules/ROOT/nav.adoc @@ -1,84 +1,118 @@ -* xref:master/welcome.adoc[Welcome] -* xref:master/1.adoc[Release] -* xref:master/2.adoc[About] -* Getting Started with IvorySQL -** xref:master/3.1.adoc[Quick Start] -** xref:master/3.2.adoc[Monitoring] -** xref:master/3.3.adoc[Maintenance] -* IvorySQL Advanced Feature -** xref:master/4.1.adoc[Installation] -** xref:master/4.2.adoc[Cluster] -** xref:master/4.5.adoc[Migration] -** xref:master/4.3.adoc[Developer] +* xref:master/about_ivorysql.adoc[About] +** xref:master/welcome.adoc[Welcome] +** xref:master/release_notes.adoc[Release] +* Get Started +** xref:master/getting-started/quick_start.adoc[Quick Start] +* Deploy +** xref:master/installation_guide.adoc[Installation] +** xref:master/cluster_setup.adoc[Cluster] +* Oracle Compatibility +** xref:master/oracle_compatibility/compat_case_conversion.adoc[1、Case conversion] +** xref:master/oracle_compatibility/compat_like_operator.adoc[2、LIKE operator] +** xref:master/oracle_compatibility/anonymous_block.adoc[3、anonymous block] +** xref:master/oracle_compatibility/compat_function_procedure.adoc[4、functions and stored procedures] +** xref:master/oracle_compatibility/builtin_types_functions.adoc[5、Built-in data types and built-in functions] +** xref:master/oracle_compatibility/port_ip.adoc[6、ports and IP] +** xref:master/oracle_compatibility/xml_functions.adoc[7、XML Function] +** xref:master/oracle_compatibility/sequence.adoc[8、sequence] +** xref:master/oracle_compatibility/package.adoc[9、Package] +** xref:master/oracle_compatibility/invisible_column.adoc[10、Invisible Columns] +** xref:master/oracle_compatibility/compat_rowid.adoc[11、RowID Column] +** xref:master/oracle_compatibility/compat_out_parameter.adoc[12、OUT Parameter] +** xref:master/oracle_compatibility/compat_type_rowtype.adoc[13、%Type & %Rowtype] +** xref:master/oracle_compatibility/compat_nls_parameter.adoc[14、NLS Parameters] +** xref:master/oracle_compatibility/compat_force_view.adoc[15、Force View] +** xref:master/oracle_compatibility/compat_nested_function.adoc[16、Nested Subfunctions] +** xref:master/oracle_compatibility/compat_sys_guid.adoc[17、sys_guid Function] +** xref:master/oracle_compatibility/compat_empty_string_to_null.adoc[18、Empty String to NULL] +** xref:master/oracle_compatibility/compat_call_into.adoc[19、CALL INTO] +** xref:master/oracle_compatibility/compat_read_only_view.adoc[20、Read Only View] +** xref:master/oracle_compatibility/with_function_procedure_en.adoc[21、WITH FUNCTION/PROCEDURE] +** xref:master/oracle_compatibility/compat_create_index_online.adoc[22、ONLINE Parameter for CREATE INDEX] +** xref:master/oracle_compatibility/compat_stragg.adoc[23、STRAGG function] +* Containerization and Cloud Service ** Containerization -*** xref:master/4.6.1.adoc[K8S deployment] -*** xref:master/4.6.2.adoc[Operator deployment] -*** xref:master/4.6.4.adoc[Docker & Podman deployment] -*** xref:master/4.6.3.adoc[Docker Swarm & Docker Compose deployment] -** xref:master/4.4.adoc[Operation Management] +*** xref:master/containerization/k8s_deployment.adoc[K8S deployment] +*** xref:master/containerization/operator_deployment.adoc[Operator deployment] +*** xref:master/containerization/docker_podman_deployment.adoc[Docker & Podman deployment] +*** xref:master/containerization/docker_swarm_compose_deployment.adoc[Docker Swarm & Docker Compose deployment] ** Cloud Service Platform -*** xref:master/4.7.1.adoc[IvorySQL Cloud Installation] -*** xref:master/4.7.2.adoc[IvorySQL Cloud Usage] +*** xref:master/cloud_platform/ivorysql_cloud_installation.adoc[IvorySQL Cloud Installation] +*** xref:master/cloud_platform/ivorysql_cloud_usage.adoc[IvorySQL Cloud Usage] * IvorySQL Ecosystem -** xref:master/cpu_arch_adp.adoc[CPU Architecture Adaption] -** xref:master/os_arch_adp.adoc[Operating System Adaption] +** xref:master/cpu_os_adaptation/cpu_architecture_adaptation.adoc[CPU Architecture Adaption] +** xref:master/cpu_os_adaptation/os_architecture_adaptation.adoc[Operating System Adaption] ** Eco Component Adaption -*** xref:master/5.0.adoc[Overview] -*** xref:master/5.1.adoc[postgis] -*** xref:master/5.2.adoc[pgvector] -*** xref:master/5.3.adoc[pgddl(DDL Extractor)] -*** xref:master/5.4.adoc[pg_cron] -*** xref:master/5.5.adoc[pgsql-http] -*** xref:master/5.6.adoc[plpgsql_check] -*** xref:master/5.7.adoc[pgroonga] -*** xref:master/5.8.adoc[pgaudit] -*** xref:master/5.9.adoc[pgrouting] -*** xref:master/5.10.adoc[system_stats] +*** xref:master/ecosystem_components/ecosystem_overview.adoc[Overview] +*** xref:master/ecosystem_components/postgis.adoc[postgis] +*** xref:master/ecosystem_components/pgvector.adoc[pgvector] +*** xref:master/ecosystem_components/pgddl.adoc[pgddl(DDL Extractor)] +*** xref:master/ecosystem_components/pg_cron.adoc[pg_cron] +*** xref:master/ecosystem_components/pgsql_http.adoc[pgsql-http] +*** xref:master/ecosystem_components/plpgsql_check.adoc[plpgsql_check] +*** xref:master/ecosystem_components/pgroonga.adoc[pgroonga] +*** xref:master/ecosystem_components/pgaudit.adoc[pgaudit] +*** xref:master/ecosystem_components/pgrouting.adoc[pgrouting] +*** xref:master/ecosystem_components/system_stats.adoc[system_stats] +*** xref:master/ecosystem_components/pg_ai_query.adoc[pg_ai_query] +*** xref:master/ecosystem_components/wal2json.adoc[wal2json] +*** xref:master/ecosystem_components/pg_stat_monitor.adoc[pg_stat_monitor] +*** xref:master/ecosystem_components/pg_partman.adoc[pg_partman] +*** xref:master/ecosystem_components/pgbouncer.adoc[pgbouncer] +*** xref:master/ecosystem_components/age.adoc[age] +*** xref:master/ecosystem_components/pg_curl.adoc[pg_curl] +*** xref:master/ecosystem_components/pg_textsearch.adoc[pg_textsearch] +*** xref:master/ecosystem_components/pg_hint_plan.adoc[pg_hint_plan] +*** xref:master/ecosystem_components/redis_fdw.adoc[redis_fdw] +*** xref:master/ecosystem_components/pg_show_plans.adoc[pg_show_plans] +*** xref:master/ecosystem_components/pg_bulkload.adoc[pg_bulkload] +*** xref:master/ecosystem_components/pg_bigm.adoc[pg_bigm] +*** xref:master/ecosystem_components/pg_profile.adoc[pg_profile] +*** xref:master/ecosystem_components/pg_repack.adoc[pg_repack] +*** xref:master/ecosystem_components/pgdog.adoc[PgDog] +*** xref:master/ecosystem_components/pg_readonly_en.adoc[pg_readonly] +*** xref:master/ecosystem_components/zhparser_en.adoc[zhparser] +*** xref:master/ecosystem_components/pgbackrest.adoc[pgBackRest] +*** xref:master/ecosystem_components/set_user.adoc[set_user] +* Monitor and O&M +** xref:master/getting-started/daily_monitoring.adoc[Monitoring] +** xref:master/getting-started/daily_maintenance.adoc[Maintenance] +** xref:master/operation_guide.adoc[Operation Management] +* Data Migration +** xref:master/migration_guide.adoc[Migration] +* IvorySQL Developers +** xref:master/developer_guide.adoc[Developer] +** xref:master/contribution/community_contribution_guide.adoc[Community contribution] * IvorySQL Architecture Design ** Query Processing -*** xref:master/6.1.1.adoc[Dual Parser] +*** xref:master/architecture/dual_parser.adoc[Dual Parser] ** Compatibility Framework -*** xref:master/7.1.adoc[Ivorysql frame design] -*** xref:master/7.2.adoc[GUC Framework] -*** xref:master/7.4.adoc[Dual-mode design] -*** xref:master/6.2.1.adoc[initdb Process] +*** xref:master/architecture/framework_design.adoc[Ivorysql frame design] +*** xref:master/architecture/guc_framework.adoc[GUC Framework] +*** xref:master/architecture/dual_mode_design.adoc[Dual-mode design] +*** xref:master/architecture/initdb_process.adoc[initdb Process] ** Compatibility Features -*** xref:master/6.3.1.adoc[like] -*** xref:master/6.3.3.adoc[RowID] -*** xref:master/6.3.2.adoc[OUT Parameter] -*** xref:master/6.3.4.adoc[%Type & %Rowtype] -*** xref:master/6.3.5.adoc[NLS Parameters] -*** xref:master/6.3.6.adoc[Function and stored procedure] -*** xref:master/6.3.7.adoc[Nested Subfunctions] -*** xref:master/6.3.8.adoc[Force View] -*** xref:master/6.3.9.adoc[Case Conversion] -*** xref:master/6.3.10.adoc[sys_guid Function] -*** xref:master/6.3.11.adoc[Empty String to NULL] -*** xref:master/6.3.12.adoc[CALL INTO] +*** xref:master/compatibility_features_design/like_operator.adoc[like] +*** xref:master/compatibility_features_design/rowid.adoc[RowID] +*** xref:master/compatibility_features_design/out_parameter.adoc[OUT Parameter] +*** xref:master/compatibility_features_design/type_rowtype.adoc[%Type & %Rowtype] +*** xref:master/compatibility_features_design/nls_parameter.adoc[NLS Parameters] +*** xref:master/compatibility_features_design/function_procedure.adoc[Function and stored procedure] +*** xref:master/compatibility_features_design/nested_function.adoc[Nested Subfunctions] +*** xref:master/compatibility_features_design/force_view.adoc[Force View] +*** xref:master/compatibility_features_design/case_conversion.adoc[Case Conversion] +*** xref:master/compatibility_features_design/sys_guid_function.adoc[sys_guid Function] +*** xref:master/compatibility_features_design/empty_string_to_null.adoc[Empty String to NULL] +*** xref:master/compatibility_features_design/call_into.adoc[CALL INTO] +*** xref:master/compatibility_features_design/read_only_view.adoc[Read Only View] +*** xref:master/compatibility_features_design/with_function_procedure_impl_en.adoc[WITH FUNCTION/PROCEDURE] +*** xref:master/compatibility_features_design/create_index_online.adoc[ONLINE Parameter for CREATE INDEX] ** Built-in Functions -*** xref:master/6.4.1.adoc[sys_context] -*** xref:master/6.4.2.adoc[userenv] -** xref:master/6.5.adoc[GB18030 Character Set] -* List of Oracle compatible features -** xref:master/7.3.adoc[1、Case conversion] -** xref:master/7.5.adoc[2、LIKE operator] -** xref:master/7.6.adoc[3、anonymous block] -** xref:master/7.7.adoc[4、functions and stored procedures] -** xref:master/7.8.adoc[5、Built-in data types and built-in functions] -** xref:master/7.9.adoc[6、ports and IP] -** xref:master/7.10.adoc[7、XML Function] -** xref:master/7.11.adoc[8、sequence] -** xref:master/7.12.adoc[9、Package] -** xref:master/7.13.adoc[10、Invisible Columns] -** xref:master/7.14.adoc[11、RowID Column] -** xref:master/7.15.adoc[12、OUT Parameter] -** xref:master/7.16.adoc[13、%Type & %Rowtype] -** xref:master/7.17.adoc[14、NLS Parameters] -** xref:master/7.18.adoc[15、Force View] -** xref:master/7.19.adoc[16、Nested Subfunctions] -** xref:master/7.20.adoc[17、sys_guid Function] -** xref:master/7.21.adoc[18、Empty String to NULL] -** xref:master/7.22.adoc[19、CALL INTO] -* xref:master/8.adoc[Community contribution] -* xref:master/9.adoc[Tool Reference] -* xref:master/10.adoc[FAQ] +*** xref:master/oracle_builtin_functions/sys_context.adoc[sys_context] +*** xref:master/oracle_builtin_functions/userenv.adoc[userenv] +*** xref:master/oracle_builtin_functions/rawtohex.adoc[rawtohex] +*** xref:master/oracle_builtin_functions/stragg.adoc[stragg] +** xref:master/gb18030.adoc[GB18030 Character Set] +* Reference +** xref:master/tools_reference.adoc[Tool Reference] +* xref:master/contribution/faq.adoc[FAQ] diff --git a/EN/modules/ROOT/pages/master/4.7.adoc b/EN/modules/ROOT/pages/master/4.7.adoc deleted file mode 100644 index e69de29..0000000 diff --git a/EN/modules/ROOT/pages/master/5.0.adoc b/EN/modules/ROOT/pages/master/5.0.adoc deleted file mode 100644 index 1d51e2f..0000000 --- a/EN/modules/ROOT/pages/master/5.0.adoc +++ /dev/null @@ -1,29 +0,0 @@ -:sectnums: -:sectnumlevels: 5 - - -[discrete] -== IvorySQL Ecosystem Plugin Compatibility List - -IvorySQL, as an advanced open-source database compatible with Oracle and based on PostgreSQL, has powerful extension capabilities and supports a rich ecosystem of plugins. These plugins can help users enhance database functionality in different scenarios, including geospatial information processing, vector retrieval, full-text search, data definition extraction, and path planning. The following is a list of major plugins currently officially compatible with and supported by IvorySQL: - -+ - -[cols="1,2,1,3,3"] -|==== -|*Index*|*Plugin Name*|*Version*|*Function Description*|*Use Cases* -|*1*| xref:master/5.1.adoc[postgis] | 3.5.4 | Provides geospatial data support for IvorySQL, including spatial indexes, spatial functions, and geographic object storage | Geographic Information Systems (GIS), map services, location data analysis -|*2*| xref:master/5.2.adoc[pgvector] | 0.8.1 | Supports vector similarity search, can be used to store and retrieve high-dimensional vector data| AI applications, image retrieval, recommendation systems, semantic search -|*3*| xref:master/5.3.adoc[pgddl (DDL Extractor)] | 0.31 | Extracts DDL (Data Definition Language) statements from databases, facilitating version management and migration | Database version control, CI/CD integration, structure comparison and synchronization -|*4*| xref:master/5.4.adoc[pg_cron]​ | 1.6.0 | Provides database-internal scheduled task scheduling functionality, supports regular SQL statement execution | Data cleanup, regular statistics, automated maintenance tasks -|*5*| xref:master/5.5.adoc[pgsql-http]​ | 1.7.0 | Allows HTTP requests to be initiated in SQL, interacting with external web services | Data collection, API integration, microservice calls -|*6*| xref:master/5.6.adoc[plpgsql_check] | 2.8 | Provides static analysis functionality for PL/pgSQL code, can detect potential errors during development | Stored procedure development, code quality checking, debugging and optimization -|*7*| xref:master/5.7.adoc[pgroonga] | 4.0.4 | Provides full-text search functionality for non-English languages, meeting the needs of high-performance applications | Full-text search capabilities for languages like Chinese, Japanese, and Korean -|*8*| xref:master/5.8.adoc[pgaudit] | 18.0 | Provides fine-grained auditing, recording database operation logs to support security auditing and compliance checks | Database security auditing, compliance checks, audit report generation -|*9*| xref:master/5.9.adoc[pgrouting] | 3.8.0 | Provides routing computation for geospatial data, supporting multiple algorithms and data formats | Geospatial analysis, route planning, logistics optimization -|*10*| xref:master/5.10.adoc[system_stats] | 3.2 | Provide functions for accessing system-level statistics. | system monitor -|==== - -These plugins have all been tested and adapted by the IvorySQL team to ensure stable operation in the IvorySQL environment. Users can select appropriate plugins based on business needs to further enhance the capabilities and flexibility of the database system. - -We will continue to expand and enrich the IvorySQL plugin ecosystem. Community developers are welcome to submit new plugin adaptation suggestions or code contributions. For more detailed usage methods and the latest compatible versions of each plugin, please refer to the corresponding documentation chapters for each plugin. diff --git a/EN/modules/ROOT/pages/master/2.adoc b/EN/modules/ROOT/pages/master/about_ivorysql.adoc similarity index 98% rename from EN/modules/ROOT/pages/master/2.adoc rename to EN/modules/ROOT/pages/master/about_ivorysql.adoc index bca8d38..101b4d8 100644 --- a/EN/modules/ROOT/pages/master/2.adoc +++ b/EN/modules/ROOT/pages/master/about_ivorysql.adoc @@ -81,4 +81,6 @@ IvorySQL is a powerful open source object-relational database management system * Nested Subfunctions * sys_guid Function * Empty String to NULL -* CALL INTO \ No newline at end of file +* CALL INTO +* View Read Only +* WITH FUNCTION/PROCEDURE diff --git a/EN/modules/ROOT/pages/master/7.4.adoc b/EN/modules/ROOT/pages/master/architecture/dual_mode_design.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.4.adoc rename to EN/modules/ROOT/pages/master/architecture/dual_mode_design.adoc diff --git a/EN/modules/ROOT/pages/master/6.1.1.adoc b/EN/modules/ROOT/pages/master/architecture/dual_parser.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.1.1.adoc rename to EN/modules/ROOT/pages/master/architecture/dual_parser.adoc diff --git a/EN/modules/ROOT/pages/master/7.1.adoc b/EN/modules/ROOT/pages/master/architecture/framework_design.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.1.adoc rename to EN/modules/ROOT/pages/master/architecture/framework_design.adoc diff --git a/EN/modules/ROOT/pages/master/7.2.adoc b/EN/modules/ROOT/pages/master/architecture/guc_framework.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.2.adoc rename to EN/modules/ROOT/pages/master/architecture/guc_framework.adoc diff --git a/EN/modules/ROOT/pages/master/6.2.1.adoc b/EN/modules/ROOT/pages/master/architecture/initdb_process.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.2.1.adoc rename to EN/modules/ROOT/pages/master/architecture/initdb_process.adoc diff --git a/EN/modules/ROOT/pages/master/4.7.1.adoc b/EN/modules/ROOT/pages/master/cloud_platform/ivorysql_cloud_installation.adoc similarity index 98% rename from EN/modules/ROOT/pages/master/4.7.1.adoc rename to EN/modules/ROOT/pages/master/cloud_platform/ivorysql_cloud_installation.adoc index cd9d9b1..68ef2e7 100644 --- a/EN/modules/ROOT/pages/master/4.7.1.adoc +++ b/EN/modules/ROOT/pages/master/cloud_platform/ivorysql_cloud_installation.adoc @@ -363,6 +363,8 @@ https://github.com/IvorySQL/ivory-operator/tree/IVYO_REL_5_STABLE[https://github If your servers have direct access to Docker Hub, you can skip this step. Otherwise, preload the following images on every node in the Kubernetes cluster. +TIP: Users in China can pull the following images from the IvorySQL community Harbor registry by replacing the `docker.io/` prefix with `registry.highgo.com/`. + [literal] ---- docker.io/ivorysql/pgadmin:ubi8-9.9-5.0-1 diff --git a/EN/modules/ROOT/pages/master/4.7.2.adoc b/EN/modules/ROOT/pages/master/cloud_platform/ivorysql_cloud_usage.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/4.7.2.adoc rename to EN/modules/ROOT/pages/master/cloud_platform/ivorysql_cloud_usage.adoc diff --git a/EN/modules/ROOT/pages/master/4.2.adoc b/EN/modules/ROOT/pages/master/cluster_setup.adoc similarity index 93% rename from EN/modules/ROOT/pages/master/4.2.adoc rename to EN/modules/ROOT/pages/master/cluster_setup.adoc index f0782ab..3b67828 100644 --- a/EN/modules/ROOT/pages/master/4.2.adoc +++ b/EN/modules/ROOT/pages/master/cluster_setup.adoc @@ -8,9 +8,9 @@ This chapter is a demo to show you how to build an IvorySQL cluster. Just take a == Primary node === Installing and start database -For quick database installation by yum, please refer to xref:v5.0/3.adoc#quick-installation[Quick installation]。 +For quick database installation by yum, please refer to xref:master/getting-started/quick_start.adoc#quick-installation[Quick installation]。 -For more installation options, please refer to xref:v5.0/6.adoc#Installation[Installation]。 +For more installation options, please refer to xref:master/installation_guide.adoc#Installation[Installation]。 [NOTE] The master node database needs to be installed and **started**. @@ -55,9 +55,9 @@ $ pg_ctl restart == Standby node === Installing database -For quick database installation by yum, please refer to xref:v5.0/3.adoc#quick-installation[Quick installation]。 +For quick database installation by yum, please refer to xref:master/getting-started/quick_start.adoc#quick-installation[Quick installation]。 -For more installation options, please refer to xref:v5.0/6.adoc#Installation[Installation]。 +For more installation options, please refer to xref:master/installation_guide.adoc#Installation[Installation]。 [NOTE] The standby node database only needs to be installed and **not started**. diff --git a/EN/modules/ROOT/pages/master/6.3.12.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/call_into.adoc similarity index 99% rename from EN/modules/ROOT/pages/master/6.3.12.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/call_into.adoc index 6ee5244..125a151 100644 --- a/EN/modules/ROOT/pages/master/6.3.12.adoc +++ b/EN/modules/ROOT/pages/master/compatibility_features_design/call_into.adoc @@ -136,7 +136,7 @@ typedef enum IvyStmtType { IVY_STMT_UNKNOW, IVY_STMT_DO, - IVY_STMT_DOFROMCALL, /* new statementt type */ + IVY_STMT_DOFROMCALL, /* new statement type */ IVY_STMT_DOHANDLED, IVY_STMT_OTHERS } IvyStmtType; diff --git a/EN/modules/ROOT/pages/master/6.3.9.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/case_conversion.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.9.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/case_conversion.adoc diff --git a/EN/modules/ROOT/pages/master/compatibility_features_design/create_index_online.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/create_index_online.adoc new file mode 100644 index 0000000..d056ddb --- /dev/null +++ b/EN/modules/ROOT/pages/master/compatibility_features_design/create_index_online.adoc @@ -0,0 +1,143 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += ONLINE Parameter for CREATE INDEX + +== Purpose + +IvorySQL supports the `ONLINE` parameter when creating an index, allowing indexes to be built online without blocking DML operations. + +== Implementation Notes + +=== Data Structure Extension + +A new `online_keyword` field is added to `IndexStmt`. + +[source,c] +---- +bool transformed; /* true when transformIndexStmt is finished */ +bool concurrent; /* should this be a concurrent index build? */ +bool online_keyword; /* was ONLINE keyword used (as opposed to CONCURRENTLY)? */ +---- + +=== Syntax and Parsing + +==== Grammar Rule Extension + +A flexible option list `create_index_opt_list` / `create_index_opt` is introduced in `ora_gram.y`, similar to the existing `rebuild_index_opt_list`: + +[source,yacc] +---- +create_index_opt_list: + create_index_opt_list create_index_opt { $$ = lappend($1, $2); } + | /* EMPTY */ { $$ = NIL; } +; + +create_index_opt: + ONLINE + { $$ = makeDefElem("online", (Node *) makeBoolean(true), @1); } + | TABLESPACE name + { $$ = makeDefElem("tablespace", (Node *) makeString($2), @1); } +; +---- + +The two productions for `IndexStmt` are modified to replace `OptTableSpace` with `create_index_opt_list`, extracting `online` and `tablespace` from the option list in the action: + +[source,yacc] +---- +IndexStmt: CREATE opt_unique INDEX opt_concurrently opt_single_name + ON relation_expr access_method_clause '(' index_params ')' + opt_include opt_unique_null_treatment opt_reloptions + create_index_opt_list where_clause + { + IndexStmt *n = makeNode(IndexStmt); + bool online = false; + bool online_seen = false; + char *tablespace = NULL; + ListCell *lc; + + /* Parse create_index_opt_list, extract online and tablespace */ + foreach(lc, $15) + { + DefElem *opt = (DefElem *) lfirst(lc); + + if (strcmp(opt->defname, "online") == 0) + { + if (online_seen) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ONLINE specified multiple times"), + parser_errposition(opt->location))); + online = defGetBoolean(opt); + online_seen = true; + } + else if (strcmp(opt->defname, "tablespace") == 0) + { + if (tablespace != NULL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("TABLESPACE specified multiple times"), + parser_errposition(opt->location))); + tablespace = defGetString(opt); + } + } + + /* CONCURRENTLY and ONLINE are mutually exclusive */ + if ($4 && online) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot use both CONCURRENTLY and ONLINE"), + parser_errposition(@4))); + + n->unique = $2; + n->concurrent = $4 || online; + n->online_keyword = online; + n->idxname = $5; + n->relation = $7; + n->accessMethod = $8; + n->indexParams = $10; + n->indexIncludingParams = $12; + n->nulls_not_distinct = !$13; + n->options = $14; /* opt_reloptions (WITH clause) */ + n->tableSpace = tablespace; + n->whereClause = $16; + n->excludeOpNames = NIL; + n->idxcomment = NULL; + n->indexOid = InvalidOid; + n->oldNumber = InvalidRelFileNumber; + n->oldCreateSubid = InvalidSubTransactionId; + n->oldFirstRelfilelocatorSubid = InvalidSubTransactionId; + n->primary = false; + n->isconstraint = false; + n->deferrable = false; + n->initdeferred = false; + n->transformed = false; + n->if_not_exists = false; + n->reset_default_tblspc = false; + $$ = (Node *) n; + } +---- + +==== Executor Layer Changes (`indexcmds.c`) + +`DefineIndex()` already contains downgrade logic for temporary tables (`concurrent = false` when temp table). The same location must also be updated with downgrade logic for partitioned tables: + +[source,c] +---- +/* Existing: downgrade for temp tables */ +if (stmt->concurrent && get_rel_persistence(tableId) != RELPERSISTENCE_TEMP) + concurrent = true; +else + concurrent = false; + +/* New: downgrade for partitioned tables. + * Oracle returns success for CREATE INDEX ONLINE on partitioned tables + * (global non-partitioned index). PostgreSQL's concurrent + partitioned + * path raises an error, so in Oracle parser mode (stmt->online_keyword = true) + * we silently downgrade to a regular build for partitioned tables. + */ +if (concurrent && partitioned && stmt->online_keyword) + concurrent = false; +---- diff --git a/EN/modules/ROOT/pages/master/6.3.11.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/empty_string_to_null.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.11.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/empty_string_to_null.adoc diff --git a/EN/modules/ROOT/pages/master/6.3.8.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/force_view.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.8.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/force_view.adoc diff --git a/EN/modules/ROOT/pages/master/6.3.6.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/function_procedure.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.6.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/function_procedure.adoc diff --git a/EN/modules/ROOT/pages/master/6.3.1.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/like_operator.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.1.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/like_operator.adoc diff --git a/EN/modules/ROOT/pages/master/6.3.7.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/nested_function.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.7.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/nested_function.adoc diff --git a/EN/modules/ROOT/pages/master/6.3.5.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/nls_parameter.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.5.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/nls_parameter.adoc diff --git a/EN/modules/ROOT/pages/master/6.3.2.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/out_parameter.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.2.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/out_parameter.adoc diff --git a/EN/modules/ROOT/pages/master/compatibility_features_design/read_only_view.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/read_only_view.adoc new file mode 100644 index 0000000..3a53a26 --- /dev/null +++ b/EN/modules/ROOT/pages/master/compatibility_features_design/read_only_view.adoc @@ -0,0 +1,235 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += Read-Only View + +== Purpose + +Oracle database supports using the `WITH READ ONLY` clause when creating views to set them as read-only, preventing any data modification operations (INSERT, UPDATE, DELETE, MERGE) on the view. To provide Oracle compatibility, IvorySQL implements the read-only view feature. Once a view is marked as read-only, any DML operations attempting to modify the view's data will be rejected, ensuring behavioral consistency for Oracle applications running on IvorySQL. + +== Implementation Details + +=== Syntax and Parsing + +==== Syntax Rule Extension + +The `READ_ONLY_OPTION` enum value was added in the `ora_gram.y` file, and the `opt_check_option` syntax rule was extended: + +[source,yacc] +---- +opt_check_option: + WITH CHECK OPTION { $$ = CASCADED_CHECK_OPTION; } + | WITH CASCADED CHECK OPTION { $$ = CASCADED_CHECK_OPTION; } + | WITH LOCAL CHECK OPTION { $$ = LOCAL_CHECK_OPTION; } + | WITH READ ONLY { $$ = READ_ONLY_OPTION; } /* New */ + | /* EMPTY */ { $$ = NO_CHECK_OPTION; } + ; +---- + +==== ViewStmt Structure Extension + +In the `parsenodes.h` file, the `ViewCheckOption` enum was extended with `READ_ONLY_OPTION`, and the `ViewStmt` structure was extended with a `readOnly` field: + +[source,c] +---- +typedef enum ViewCheckOption { + NO_CHECK_OPTION, + LOCAL_CHECK_OPTION, + CASCADED_CHECK_OPTION, + READ_ONLY_OPTION, /* WITH READ ONLY (Oracle compat) */ +} ViewCheckOption; + +typedef struct ViewStmt { + // ... other fields + bool readOnly; /* WITH READ ONLY (Oracle compat) */ +} ViewStmt; +---- + +The `readOnly` flag is set during parsing: + +[source,c] +---- +n->readOnly = ($10 == READ_ONLY_OPTION); +n->withCheckOption = n->readOnly ? NO_CHECK_OPTION : $10; +---- + +Note that `WITH READ ONLY` and `WITH CHECK OPTION` are mutually exclusive and cannot be used together. + +=== Relation Options System + +==== read_only Relation Option + +The `read_only` view relation option was defined in the `reloptions.c` file: + +[source,c] +---- +/* reloptions.c */ +{ + {"read_only", + "Prevents INSERT, UPDATE, and DELETE on this view (Oracle compatibility)", + RELOPT_KIND_VIEW, + AccessExclusiveLock}, + false +}, + +/* in view_reloptions() function */ +{"read_only", RELOPT_TYPE_BOOL, + offsetof(ViewOptions, read_only)}, +---- + +==== ViewOptions Structure Extension + +In the `rel.h` file, the `ViewOptions` structure was extended with a `read_only` field: + +[source,c] +---- +typedef struct ViewOptions { + // ... other fields + bool read_only; /* WITH READ ONLY (Oracle compat) */ +} ViewOptions; +---- + +==== RelationIsReadOnlyView Macro + +The `RelationIsReadOnlyView` macro allows quick determination of whether a view is read-only: + +[source,c] +---- +#define RelationIsReadOnlyView(relation) \ + (AssertMacro(relation->rd_rel->relkind == RELKIND_VIEW), \ + (relation)->rd_options && \ + ((ViewOptions *) (relation)->rd_options)->read_only) +---- + +=== View Creation Processing + +The `WITH READ ONLY` option is handled in the `DefineView` function in the `view.c` file: + +[source,c] +---- +/* Handle WITH READ ONLY in DefineView() */ +if (stmt->readOnly) +{ + /* Set read_only relation option */ + options = transformViewOptions(RelationGetRelid(newRelationView), + stmt->options, true); + setRelOptions(newRelationView, options, true, AccessExclusiveLock); +} +---- + +`WITH READ ONLY` is also supported in the `compile_force_view_internal` function to ensure FORCE VIEW can properly use the read-only attribute. + +=== DML Execution Interception + +==== Execution Layer Interception + +In the `CheckValidResultRel()` function in `execMain.c`, DML operations on read-only views are intercepted: + +[source,c] +---- +/* execMain.c */ +if (RelationIsReadOnlyView(resultRel)) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot modify view \"%s\"", + RelationGetRelationName(resultRel)), + errhint("The view is defined as read-only."))); +---- + +When executing INSERT, UPDATE, DELETE, or MERGE statements, an error is thrown if the target view is marked as read-only. + +==== Rewrite Layer Interception + +The same check was added in the `rewriteTargetView()` function in `rewriteHandler.c`: + +[source,c] +---- +/* rewriteHandler.c */ +if (RelationIsReadOnlyView(view)) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot modify view \"%s\"", + RelationGetRelationName(view)), + errhint("The view is defined as read-only."))); +---- + +This ensures that modifications to read-only views are also blocked at the query rewrite layer. + +=== pg_dump Support + +Support for exporting and restoring the `WITH READ ONLY` attribute was added in `pg_dump.c`: + +[source,c] +---- +/* pg_dump.h - TableInfo structure new field */ +typedef struct _tableInfo { + // ... other fields + bool readOnly; /* WITH READ ONLY (Oracle compat) */ + // ... other fields +} _tableInfo; + +/* pg_dump.c - extract read_only option */ +if (strcmp(options[i].defname, "read_only") == 0) + info->readOnly = defGetBoolean(options[i].def); +---- + +During export, the `dumpTableSchema()` and `dumpRule()` functions output the `WITH READ ONLY` clause in the appropriate location, ensuring the read-only attribute is preserved during database migration. + +== Usage Examples + +=== Creating a Read-Only View + +[source,sql] +---- +-- Create a view with WITH READ ONLY +CREATE VIEW emp_view AS +SELECT * FROM employees +WITH READ ONLY; + +-- DML on read-only view will be rejected +INSERT INTO emp_view VALUES (1, 'John', 5000); +-- ERROR: cannot modify view "emp_view" +-- HINT: The view is defined as read-only. + +UPDATE emp_view SET salary = 6000 WHERE id = 1; +-- ERROR: cannot modify view "emp_view" +-- HINT: The view is defined as read-only. + +DELETE FROM emp_view WHERE id = 1; +-- ERROR: cannot modify view "emp_view" +-- HINT: The view is defined as read-only. +---- + +=== Create or Replace Read-Only View + +[source,sql] +---- +-- Use OR REPLACE to preserve read-only attribute +CREATE OR REPLACE VIEW emp_view AS +SELECT id, name, department FROM employees +WITH READ ONLY; +---- + +=== FORCE VIEW with Read-Only Attribute + +[source,sql] +---- +-- Create FORCE VIEW and set as read-only +CREATE OR REPLACE FORCE VIEW emp_force_view AS +SELECT * FROM employees +WITH READ ONLY; +---- + +=== Mutual Exclusivity with WITH CHECK OPTION + +[source,sql] +---- +-- Error: WITH READ ONLY and WITH CHECK OPTION cannot be used together +CREATE VIEW emp_view AS +SELECT * FROM employees +WITH CHECK OPTION +WITH READ ONLY; +-- ERROR: READ ONLY and CHECK OPTION are mutually exclusive +---- diff --git a/EN/modules/ROOT/pages/master/6.3.3.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/rowid.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.3.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/rowid.adoc diff --git a/EN/modules/ROOT/pages/master/6.3.10.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/sys_guid_function.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.10.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/sys_guid_function.adoc diff --git a/EN/modules/ROOT/pages/master/6.3.4.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/type_rowtype.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.3.4.adoc rename to EN/modules/ROOT/pages/master/compatibility_features_design/type_rowtype.adoc diff --git a/EN/modules/ROOT/pages/master/compatibility_features_design/with_function_procedure_impl_en.adoc b/EN/modules/ROOT/pages/master/compatibility_features_design/with_function_procedure_impl_en.adoc new file mode 100644 index 0000000..67a363b --- /dev/null +++ b/EN/modules/ROOT/pages/master/compatibility_features_design/with_function_procedure_impl_en.adoc @@ -0,0 +1,784 @@ +:sectnums: +:sectnumlevels: 5 + += WITH FUNCTION/PROCEDURE Implementation Notes + +== Purpose + +This document describes the internal implementation of the `WITH FUNCTION` and +`WITH PROCEDURE` features in IvorySQL. The feature allows PL/SQL functions and +procedures to be defined directly inside the WITH clause (Common Table Expression, CTE) +of a SQL statement, implementing Oracle's _Subquery Factoring with PL/SQL Declarations_. + +== Implementation Overview + +=== Layered Architecture + +The implementation spans four layers: + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Layer 1: Oracle Parser (ora_gram.y + liboracle_parser.c) │ +│ - Extend with_clause grammar to allow plsql_declarations │ +│ - Reuse OraBody_FUNC lexer mechanism to capture body as Sconst │ +│ - Output: WithClause { plsql_defs: [InlineFunctionDef...], ctes: [...]}│ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ Layer 2: Semantic Analysis (parse_cte.c + parse_func.c) │ +│ - transformWithClause() processes InlineFunctionDef nodes │ +│ - Resolve function signatures; register in ParseState.p_with_func_list│ +│ - p_subprocfunc_hook intercepts call resolution for WITH functions │ +│ - FuncExpr.function_from = FUNC_FROM_WITH_CLAUSE ('w') │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ Layer 3: Planner (planner.c / createplan.c) │ +│ - Copy Query.withFuncDefs into PlannedStmt.withFuncDefs │ +│ - No additional cost model changes │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ Layer 4: Executor (execExpr.c + pl_handler.c) │ +│ - ExecInitFunc() recognizes FUNC_FROM_WITH_CLAUSE │ +│ - Compiles WITH function body on demand (lazy compilation) │ +│ - Compilation result cached in EState.es_with_func_container │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +=== Grammar and Parsing + +==== Grammar Rule Extension + +The `with_clause` grammar rule in `ora_gram.y` is extended as follows: + +[source,yacc] +---- +with_clause: + WITH plsql_declarations cte_list + { + WithClause *n = makeNode(WithClause); + n->plsql_defs = $2; + n->ctes = $3; + n->recursive = false; + n->location = @1; + $$ = n; + } + | WITH plsql_declarations + { + WithClause *n = makeNode(WithClause); + n->plsql_defs = $2; + n->ctes = NIL; + n->recursive = false; + n->location = @1; + $$ = n; + } + | WITH cte_list /* existing syntax unchanged */ + | WITH_LA cte_list /* existing */ + | WITH RECURSIVE cte_list /* existing */ + ; + +plsql_declarations: + plsql_declaration + { $$ = list_make1($1); } + | plsql_declarations plsql_declaration + { $$ = lappend($1, $2); } + ; + +plsql_declaration: + FUNCTION ora_func_name opt_ora_func_args_with_defaults + RETURN func_return + ora_func_is_or_as Sconst ';' + { + InlineFunctionDef *n = makeNode(InlineFunctionDef); + n->funcname = strVal(llast($2)); + n->args = $3; + n->rettype = $5; + n->is_proc = false; + n->src = $7; + n->location = @2; + $$ = (Node *) n; + } + | PROCEDURE ora_func_name opt_procedure_args_with_defaults + ora_func_is_or_as Sconst ';' + { + InlineFunctionDef *n = makeNode(InlineFunctionDef); + n->funcname = strVal(llast($2)); + n->args = $3; + n->rettype = NULL; + n->is_proc = true; + n->src = $5; + n->location = @2; + $$ = (Node *) n; + } + ; +---- + +==== Reuse of the OraBody_FUNC Mechanism + +The Oracle parser already has a `set_oracle_plsql_body(OraBody_FUNC)` mechanism that +captures the entire PL/SQL body after IS/AS as a single Sconst string. WITH FUNCTION +reuses this mechanism directly; no changes to the lexer are required. + +[source,c] +---- +/* ora_gram.y */ +ora_func_is_or_as: + IS { set_oracle_plsql_body(yyscanner, OraBody_FUNC); } + | AS { set_oracle_plsql_body(yyscanner, OraBody_FUNC); } + ; +---- + +The lexer (`liboracle_parser.c:439-652`) enters body-capture mode, tracking the +BEGIN/END/FUNCTION/PROCEDURE nesting depth until the outermost END, and returns the +entire text as a single SCONST token. + +==== Handling Grammar Ambiguity + +Both `FUNCTION` and `PROCEDURE` are `unreserved_keyword` tokens and can therefore be +used as CTE names. The LALR(1) state table disambiguates automatically based on the +token that follows: + +|=== +| Token after `WITH FUNCTION\|PROCEDURE` | LALR(1) action | Path taken + +| **IDENT** / plain identifier +| shift (enters `ora_func_name`) +| plsql_declaration + +| **`AS`** +| reduce +| CTE without column-name list + +| **`(`** +| reduce +| CTE with column-name list +|=== + +=== AST Node Design + +==== InlineFunctionDef Node + +A new AST node represents an inline function/procedure definition inside the WITH clause: + +[source,c] +---- +/* parsenodes.h */ +typedef struct InlineFunctionDef +{ + NodeTag type; /* T_InlineFunctionDef */ + char *funcname; /* unqualified function/procedure name */ + List *args; /* list of FunctionParameter nodes */ + TypeName *rettype; /* return type (NULL for procedures) */ + bool is_proc; /* true = procedure, false = function */ + char *src; /* raw body text (full IS/AS...END text) */ + ParseLoc location; /* position in the original SQL text */ +} InlineFunctionDef; +---- + +==== WithClause Extension + +[source,c] +---- +/* parsenodes.h */ +typedef struct WithClause +{ + NodeTag type; + List *plsql_defs; /* NEW: InlineFunctionDef list (ORA mode) */ + List *ctes; /* existing: CommonTableExpr list */ + bool recursive; /* true = WITH RECURSIVE */ + ParseLoc location; +} WithClause; +---- + +==== FuncExpr Extension + +A new function-origin tag is added in `primnodes.h`: + +[source,c] +---- +/* primnodes.h */ +#define FUNC_FROM_WITH_CLAUSE 'w' /* inline function from WITH clause */ + +/* Update macro to exclude 'w' from the pg_proc path */ +#define FUNC_EXPR_FROM_PG_PROC(function_from) \ + (function_from != FUNC_FROM_SUBPROCFUNC && \ + function_from != FUNC_FROM_PACKAGE && \ + function_from != FUNC_FROM_PACKGE_INITBODY && \ + function_from != FUNC_FROM_WITH_CLAUSE) +---- + +==== WithFuncEntry Structure + +A lightweight structure added to ParseState stores function signatures: + +[source,c] +---- +/* parse_node.h */ +typedef struct WithFuncEntry +{ + char *funcname; + List *argtypes; /* list of Oids */ + Oid rettype; + bool is_proc; + int funcindex; /* corresponds to FuncExpr.funcid */ + InlineFunctionDef *def; /* pointer to the original definition node */ +} WithFuncEntry; +---- + +==== WithFuncContainer Runtime Container + +[source,c] +---- +/* pl_handler.h */ +typedef struct WithFuncContainer +{ + int nfuncs; /* number of WITH functions */ + PLiSQL_subproc_function **funcs; /* array of compiled functions */ + MemoryContext mcxt; /* memory context for this container */ +} WithFuncContainer; +---- + +=== Semantic Analysis + +==== transformWithClause Extension + +`transformWithFuncDefs()` is implemented in the new file `parse_with_plsql.c` and called +from `transformWithClause()` in `parse_cte.c`: + +[source,c] +---- +/* parse_with_plsql.c */ +static void +transformWithFuncDefs(ParseState *pstate, List *plsql_defs) +{ + int funcindex = 0; + ListCell *lc; + + foreach(lc, plsql_defs) + { + InlineFunctionDef *ifd = (InlineFunctionDef *) lfirst(lc); + WithFuncEntry *entry = palloc(sizeof(WithFuncEntry)); + + /* resolve parameter types */ + entry->argtypes = resolveWithFuncArgTypes(pstate, ifd->args); + + /* resolve return type */ + entry->rettype = ifd->rettype ? + typenameTypeId(pstate, ifd->rettype) : InvalidOid; + + entry->funcname = ifd->funcname; + entry->is_proc = ifd->is_proc; + entry->funcindex = funcindex++; + entry->def = ifd; + + /* check for duplicate definition */ + checkDuplicateWithFunc(pstate->p_with_func_list, entry); + + pstate->p_with_func_list = lappend(pstate->p_with_func_list, entry); + } + + /* install the function lookup hook */ + if (pstate->p_subprocfunc_hook == NULL) + pstate->p_subprocfunc_hook = withFuncLookupHook; +} +---- + +==== Function Call Resolution Hook + +`withFuncLookupHook` intercepts calls to inline functions defined in the WITH clause: + +[source,c] +---- +/* parse_with_plsql.c */ +static FuncDetailCode +withFuncLookupHook(ParseState *pstate, List *funcname, + List **fargs, List *fargnames, int nargs, + Oid *argtypes, bool expand_variadic, bool expand_defaults, + bool proc_call, Oid *funcid, Oid *rettype, bool *retset, + int *nvargs, Oid *vatype, Oid **true_typeids, + List **argdefaults, void **pfunc) +{ + char *fname; + ListCell *lc; + + if (list_length(funcname) != 1) + return FUNCDETAIL_NOTFOUND; + + fname = strVal(linitial(funcname)); + + foreach(lc, pstate->p_with_func_list) + { + WithFuncEntry *entry = (WithFuncEntry *) lfirst(lc); + + if (strcmp(entry->funcname, fname) != 0) + continue; + + /* check argument count and type compatibility */ + if (!matchWithFuncArgs(entry, nargs, argtypes, fargnames, + true_typeids, argdefaults)) + continue; + + *funcid = (Oid) entry->funcindex; + *rettype = entry->rettype; + *retset = false; + *nvargs = 0; + *vatype = InvalidOid; + *pfunc = NULL; + + return entry->is_proc ? FUNCDETAIL_PROCEDURE : FUNCDETAIL_NORMAL; + } + + return FUNCDETAIL_NOTFOUND; +} +---- + +=== Executor Design + +==== ExecInitFunc Extension + +`FUNC_FROM_WITH_CLAUSE` is recognized in `execExpr.c`: + +[source,c] +---- +/* execExpr.c */ +if (funcexpr->function_from == FUNC_FROM_WITH_CLAUSE) +{ + scratch->d.func.finfo = palloc0(sizeof(FmgrInfo)); + scratch->d.func.fcinfo_data = palloc0(SizeForFunctionCallInfo(nargs)); + flinfo = scratch->d.func.finfo; + fcinfo = scratch->d.func.fcinfo_data; + + /* use the dedicated dispatch function */ + flinfo->fn_addr = plisql_with_func_call_handler; + flinfo->fn_oid = funcid; + + /* fn_extra stores the EState pointer */ + flinfo->fn_extra = state->parent->state; + + fmgr_info_set_expr((Node *) node, flinfo); + InitFunctionCallInfoData(*fcinfo, flinfo, nargs, inputcollid, NULL, NULL); + scratch->d.func.fn_addr = flinfo->fn_addr; + scratch->d.func.nargs = nargs; + return; +} +---- + +==== Runtime Dispatch Function + +[source,c] +---- +/* pl_handler.c */ +Datum +plisql_with_func_call_handler(PG_FUNCTION_ARGS) +{ + EState *estate = (EState *) fcinfo->flinfo->fn_extra; + int funcindex = (int) fcinfo->flinfo->fn_oid; + WithFuncContainer *container; + + /* lazy load: compile all WITH functions on first call */ + if (estate->es_with_func_container == NULL) + estate->es_with_func_container = buildWithFuncContainer(estate); + + container = estate->es_with_func_container; + + Assert(funcindex >= 0 && funcindex < container->nfuncs); + subprocfunc = container->funcs[funcindex]; + + return execWithFunction(subprocfunc, fcinfo); +} +---- + +==== WithFuncContainer Compilation + +[source,c] +---- +/* pl_handler.c */ +WithFuncContainer * +buildWithFuncContainer(EState *estate) +{ + PlannedStmt *pstmt = estate->es_plannedstmt; + MemoryContext oldcxt = MemoryContextSwitchTo(estate->es_query_cxt); + WithFuncContainer *container; + int nfuncs, i; + ListCell *lc; + + nfuncs = list_length(pstmt->withFuncDefs); + container = palloc0(sizeof(WithFuncContainer)); + container->nfuncs = nfuncs; + container->funcs = palloc0(nfuncs * sizeof(PLiSQL_subproc_function *)); + container->mcxt = CurrentMemoryContext; + + /* establish a shared compilation context to support cross-function calls */ + plisql_push_subproc_func(); + plisql_start_subproc_func(); + + /* Pass 1: register all function signatures */ + i = 0; + foreach(lc, pstmt->withFuncDefs) + { + InlineFunctionDef *ifd = (InlineFunctionDef *) lfirst(lc); + List *argitems = buildArgItemsFromFuncParams(ifd->args); + PLiSQL_type *rettype = ifd->rettype ? + buildPLiSQLType(ifd->rettype) : NULL; + + PLiSQL_subproc_function *subprocfunc = + plisql_build_subproc_function(ifd->funcname, argitems, + rettype, ifd->location); + subprocfunc->is_proc = ifd->is_proc; + subprocfunc->src = ifd->src; + + container->funcs[i++] = subprocfunc; + } + + /* Pass 2: compile each function body */ + i = 0; + foreach(lc, pstmt->withFuncDefs) + { + InlineFunctionDef *ifd = (InlineFunctionDef *) lfirst(lc); + PLiSQL_subproc_function *subprocfunc = container->funcs[i++]; + + PLiSQL_stmt_block *action = + compileWithFuncBody(ifd->src, subprocfunc); + + plisql_set_subprocfunc_action(subprocfunc, action); + } + + plisql_pop_subproc_func(); + + MemoryContextSwitchTo(oldcxt); + return container; +} +---- + +=== Memory Lifecycle + +[source] +---- +SQL text parsing phase + ├── InlineFunctionDef nodes + │ Lifetime: same as the parse tree (ParseState.p_mem_cxt) + └── WithFuncEntry list + Lifetime: same as ParseState + +Query analysis phase + └── Query.withFuncDefs (InlineFunctionDef list, pointer copy) + Lifetime: same as Query + +Planning phase + └── PlannedStmt.withFuncDefs (same) + Lifetime: same as PlannedStmt (may be held by plancache) + +Execution phase + ├── EState.es_with_func_container (WithFuncContainer) + │ Memory context: estate->es_query_cxt + │ Lifetime: bound to EState, freed at ExecutorEnd() + ├── PLiSQL_subproc_function array + │ Lifetime: same as WithFuncContainer + └── PLiSQL_function (compilation result) + Lifetime: freed when query execution ends +---- + +=== Key Design Principles + +1. **Reuse OraBody_FUNC**: The Oracle parser's existing + `set_oracle_plsql_body(OraBody_FUNC)` mechanism is reused directly; no changes to + the lexer are needed. + +2. **No system catalog writes**: Compiled results are stored locally in EState, not + written to `pg_proc`, generate no WAL, and are freed automatically at transaction end. + +3. **Deferred compilation**: Function bodies are compiled during executor initialization + (`ExecInitFunc`), not during parsing or planning. This avoids lifecycle issues that + would arise from storing compiled pointers inside Plan nodes (which may be cached). + +4. **Isolated from PG_PARSER**: All new logic is guarded by + `compatible_db == ORA_PARSER`; the standard PostgreSQL parser path is unaffected. + +5. **Two-pass compilation**: A two-pass design supports mutual calls between functions + (Pass 1 registers all signatures; Pass 2 compiles each function body). + +=== Cross-Function Calls + +Thanks to the two-pass compilation design, **the declaration order of functions inside +the WITH clause does not matter**: A can call B even if B is defined after A, and vice +versa. No Oracle PL/SQL-style explicit forward declarations are required. + +[source,sql] +---- +-- Cross-function call example +WITH + FUNCTION mul2(n NUMBER) RETURN NUMBER AS BEGIN RETURN n*2; END; + FUNCTION add1(n NUMBER) RETURN NUMBER AS BEGIN RETURN n+1; END; +SELECT mul2(add1(3)) FROM dual; -- add1 is defined second, but mul2 can call it +---- + +=== EXPLAIN Output + +==== Basic Output + +[source,sql] +---- +EXPLAIN WITH + FUNCTION add_one(n NUMBER) RETURN NUMBER AS BEGIN RETURN n + 1; END; +SELECT add_one(5) FROM dual; +---- +Output includes: `WITH Function: add_one(number) RETURN number` + +==== VERBOSE Mode + +[source,sql] +---- +EXPLAIN (VERBOSE ON) WITH + FUNCTION double(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n * 2; END; +SELECT double(3) FROM dual; +---- +Output includes: `Body: BEGIN RETURN n * 2; END` + +=== Error Handling + +==== Duplicate Definition + +[source,sql] +---- +WITH + FUNCTION dup(n NUMBER) RETURN NUMBER AS BEGIN RETURN n; END; + FUNCTION dup(n NUMBER) RETURN NUMBER AS BEGIN RETURN n * 2; END; +SELECT dup(1) FROM dual; +-- ERROR: WITH clause function "dup" is defined more than once with the same argument types +---- + +==== Rejection in PG_PARSER Mode + +[source,sql] +---- +SET compatible_db = PG_PARSER; +WITH FUNCTION foo(n NUMBER) RETURN NUMBER AS BEGIN RETURN n; END; +SELECT foo(1); +-- ERROR: syntax error at or near "FUNCTION" +---- + +==== Function Body Compilation Error + +[source,sql] +---- +WITH + FUNCTION broken(n NUMBER) RETURN NUMBER AS + BEGIN + RETRUN n; -- typo + END; +SELECT broken(1) FROM dual; +-- ERROR: syntax error at or near "RETRUN" +---- +Error context: `while compiling WITH FUNCTION "broken_body"` + +==== Rejection of Table Function Usage + +[source,sql] +---- +WITH + FUNCTION get_rows(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n; END; +SELECT * FROM get_rows(5); +-- ERROR: WITH clause function cannot be used as a table or set-returning function +---- + +==== Rejection of Qualified Names + +[source,sql] +---- +WITH + FUNCTION public.qual_func(n NUMBER) RETURN NUMBER IS + BEGIN RETURN n; END; +SELECT qual_func(1) FROM dual; +-- ERROR: qualified name is not allowed in WITH FUNCTION declaration +---- + +=== File Inventory + +==== New Files + +|=== +| File | Description + +| `src/backend/oracle_parser/ora_with_function.c` +| WITH function runtime logic stub + +| `src/backend/parser/parse_with_plsql.c` +| `transformWithFuncDefs`, `withFuncLookupHook` + +| `src/include/oracle_parser/ora_with_function.h` +| Header: `WithFuncEntry`, `WithFuncContainer` + +| `src/oracle_test/regress/sql/with_function.sql` +| Regression tests (32 test cases) + +| `src/oracle_test/regress/expected/with_function.out` +| Expected output for regression tests +|=== + +==== Modified Existing Files + +|=== +| File | Changes + +| `src/include/nodes/parsenodes.h` +| Add `InlineFunctionDef`; extend `WithClause.plsql_defs` and `Query.withFuncDefs` + +| `src/include/nodes/plannodes.h` +| Extend `PlannedStmt.withFuncDefs` + +| `src/include/nodes/primnodes.h` +| Add `FUNC_FROM_WITH_CLAUSE = 'w'`; update `FUNC_EXPR_FROM_PG_PROC` macro + +| `src/include/nodes/execnodes.h` +| Extend `EState.es_with_func_container` + +| `src/include/parser/parse_node.h` +| Extend `ParseState.p_with_func_list` + +| `src/backend/oracle_parser/ora_gram.y` +| Extend `with_clause`; add `plsql_declarations` / `plsql_declaration` productions + +| `src/backend/parser/parse_cte.c` +| `transformWithClause()` calls `transformWithFuncDefs` + +| `src/backend/parser/parse_func.c` +| FuncExpr tagging logic; default-argument sync; `FUNC_MAX_ARGS` boundary check + +| `src/backend/parser/analyze.c` +| Propagate `withFuncDefs` into Query node + +| `src/backend/optimizer/plan/planner.c` +| Propagate `withFuncDefs` into PlannedStmt + +| `src/backend/optimizer/plan/setrefs.c` +| Skip reference resolution for WITH-clause functions + +| `src/backend/executor/execExpr.c` +| `ExecInitFunc()` handles WITH functions; three-level EState lookup chain + +| `src/backend/executor/execSRF.c` +| `init_sexpr` early-rejects WITH functions (cannot be used as table/SRF) + +| `src/backend/executor/execTuples.c` +| Explicitly exclude `FUNC_FROM_WITH_CLAUSE` from subproc return-type change path + +| `src/backend/utils/fmgr/funcapi.c` +| `get_internal_function_result_type` early-rejects WITH functions + +| `src/backend/commands/explain.c` +| EXPLAIN output: `WITH Function:` / `WITH Procedure:` signatures; VERBOSE Body output + +| `src/backend/commands/explain.c` (MERGE path) +| `IvytransformMergeStmt` sets `qry->withFuncDefs` (fixes T16 MERGE crash) + +| `src/pl/plisql/src/pl_handler.c` +| `buildWithFuncContainer`, `plisql_get_with_func`, `buildParamListForFunc` (typmod fix), + WITH-function compile error callback + +| `src/pl/plisql/src/pl_comp.c` +| `plisql_parser_setup` gates `p_with_func_list` exposure via `is_with_clause_func` flag + (prevents scope leakage) + +| `src/pl/plisql/src/plisql.h` +| Add `PLiSQL_function.is_with_clause_func` field + +| `src/oracle_fe_utils/ora_psqlscan.l` +| psql client scanner: recognize `WITH FUNCTION/PROCEDURE` and `EXPLAIN WITH ...`; + handle labeled END; handle nested BEGIN +|=== + +=== Automatically Generated Node Infrastructure Files + +PostgreSQL 16+ introduced a node-infrastructure code generator +(`src/backend/nodes/gen_node_support.pl`). After the `InlineFunctionDef` node is added +to the header, the build system automatically regenerates: + +|=== +| Auto-generated File | Contents + +| `copyfuncs.funcs.c` / `copyfuncs.switch.c` +| `_copyInlineFunctionDef` + +| `equalfuncs.funcs.c` / `equalfuncs.switch.c` +| `_equalInlineFunctionDef` + +| `outfuncs.funcs.c` / `outfuncs.switch.c` +| `_outInlineFunctionDef` + +| `readfuncs.funcs.c` / `readfuncs.switch.c` +| `_readInlineFunctionDef` + +| `nodetags.h` +| `T_InlineFunctionDef = 498` + +| `queryjumblefuncs.funcs.c` +| `_jumbleInlineFunctionDef` +|=== + +== Usage Examples + +=== Simplest Inline Function + +[source,sql] +---- +WITH + FUNCTION double_it(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n * 2; END; +SELECT double_it(5) FROM dual; +-- Output: 10 +---- + +=== Function Mixed with a CTE + +[source,sql] +---- +WITH + FUNCTION tax(amt NUMBER) RETURN NUMBER AS + BEGIN RETURN amt * 0.1; END; + orders AS (SELECT 100 AS amount) +SELECT amount, tax(amount) FROM orders; +-- Output: 100 | 10 +---- + +=== Multiple Inline Functions + +[source,sql] +---- +WITH + FUNCTION add1(n NUMBER) RETURN NUMBER AS BEGIN RETURN n+1; END; + FUNCTION mul2(n NUMBER) RETURN NUMBER AS BEGIN RETURN n*2; END; +SELECT mul2(add1(3)) FROM dual; +-- Output: 8 +---- + +=== Recursive Function + +[source,sql] +---- +WITH + FUNCTION factorial(n NUMBER) RETURN NUMBER AS + BEGIN + IF n <= 1 THEN RETURN 1; END IF; + RETURN n * factorial(n-1); + END; +SELECT factorial(5) FROM dual; +-- Output: 120 +---- + +=== Integration with DML + +[source,sql] +---- +-- Allowed: INSERT with inline function +WITH + FUNCTION get_bonus(sal NUMBER) RETURN NUMBER AS + BEGIN RETURN sal * 1.2; END; +INSERT INTO emp_bonus (empno, bonus) +SELECT empno, get_bonus(sal) FROM emp WHERE deptno = 10; +---- + +NOTE: Oracle does not allow WITH FUNCTION to precede UPDATE, DELETE, or MERGE. +IvorySQL enforces the same restriction; such usage raises `ERRCODE_FEATURE_NOT_SUPPORTED`. diff --git a/EN/modules/ROOT/pages/master/4.6.4.adoc b/EN/modules/ROOT/pages/master/containerization/docker_podman_deployment.adoc similarity index 90% rename from EN/modules/ROOT/pages/master/4.6.4.adoc rename to EN/modules/ROOT/pages/master/containerization/docker_podman_deployment.adoc index d3255e3..8ebda8f 100644 --- a/EN/modules/ROOT/pages/master/4.6.4.adoc +++ b/EN/modules/ROOT/pages/master/containerization/docker_podman_deployment.adoc @@ -4,6 +4,11 @@ = Docker & Podman deployment IvorySQL +[TIP] +==== +In addition to Docker Hub, the IvorySQL community also provides a Harbor registry for users in China. Replace the image prefix used in this chapter with `registry.highgo.com/`, e.g. `registry.highgo.com/ivorysql/ivorysql:5.0-ubi8`. +==== + == Running IvorySQL in docker ** Get IvorySQL image from Docker Hub diff --git a/EN/modules/ROOT/pages/master/4.6.3.adoc b/EN/modules/ROOT/pages/master/containerization/docker_swarm_compose_deployment.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/4.6.3.adoc rename to EN/modules/ROOT/pages/master/containerization/docker_swarm_compose_deployment.adoc diff --git a/EN/modules/ROOT/pages/master/4.6.1.adoc b/EN/modules/ROOT/pages/master/containerization/k8s_deployment.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/4.6.1.adoc rename to EN/modules/ROOT/pages/master/containerization/k8s_deployment.adoc diff --git a/EN/modules/ROOT/pages/master/4.6.2.adoc b/EN/modules/ROOT/pages/master/containerization/operator_deployment.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/4.6.2.adoc rename to EN/modules/ROOT/pages/master/containerization/operator_deployment.adoc diff --git a/EN/modules/ROOT/pages/master/8.adoc b/EN/modules/ROOT/pages/master/contribution/community_contribution_guide.adoc similarity index 95% rename from EN/modules/ROOT/pages/master/8.adoc rename to EN/modules/ROOT/pages/master/contribution/community_contribution_guide.adoc index 357d6cc..c1cb5df 100644 --- a/EN/modules/ROOT/pages/master/8.adoc +++ b/EN/modules/ROOT/pages/master/contribution/community_contribution_guide.adoc @@ -60,6 +60,8 @@ You can upload your modified bugs, new functions and other codes to your persona The IvorySQL community provides Chinese and English documents. English documents are saved in ... document repository, Chinese documents are saved in i18n document repository. You can contribute to one of them or both. +When you add or update a plugin compatibility document, you must also update the plugin summary table in xref:master/ecosystem_components/ecosystem_overview.adoc[ecosystem_overview.adoc]. Keep the plugin name, version, function description, use cases, and link to the detail page in sync so that the overview page and the detail page stay consistent. + ==== Test IvorySQL and Report Bugs GitHub: https://github.com/IvorySQL/IvorySQL @@ -98,7 +100,7 @@ If the change you're working on touches functionality that is common between Pos ==== Patch submission -Once you are ready to share your work with the IvorySQL core team and the rest of the IvorySQL community, you should push all the commits to a branch in your own repository forked from the official IvorySQL and send us a pull request. +Once you are ready to share your work with the IvorySQL core team and the rest of the IvorySQL community, you should first create or claim the corresponding GitHub Issue, then push all the commits to a branch in your own repository forked from the official IvorySQL and send us a pull request. Every pull request must correspond to at least one Issue so that the change request, discussion, review, and merge result remain traceable. ==== Patch review @@ -197,6 +199,8 @@ Click submit new issue button. WELL DONE! 2 Click the fork button in the upper right corner, Wait for the fork to finish +Before you start coding or editing documents, make sure there is a corresponding GitHub Issue for the change. You can create a new Issue or claim an existing one, but do not submit a PR without an associated Issue. + === Second: Clone the warehouse to local ``` @@ -237,17 +241,15 @@ Git push -u origin new-branch-name 2 Click Compare & pull request button and create a PR. +3 In the PR title or description, explicitly reference the corresponding Issue number, for example `Fixes #123` or `Refs #123`. + == **Submit PR** A PR submission should contain only one function or one bug. Prohibit submitting multiple functions at one time. -=== First:Create a Pull Request +Every GitHub PR must have a corresponding Issue. A PR without an associated Issue does not meet the community contribution process requirements. -1 Open your warehouse: https://github.com/$user/docs-cn[https://github.com/$user/IvorySQL] ($user is your GitHub ID) 。 - -2 Click Compare & pull request button. - -=== Second:Fill in PR information +=== Fill in PR information ``` Fix test @@ -259,7 +261,9 @@ leave a comment Give a detailed description of the submission function ``` -=== Third:Submit PR +Reference the related Issue in the PR description and keep the Issue number visible to reviewers, for example `Fixes #123`. + +=== Submit PR Click Create pull request button. WELL DONE! diff --git a/EN/modules/ROOT/pages/master/10.adoc b/EN/modules/ROOT/pages/master/contribution/faq.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/10.adoc rename to EN/modules/ROOT/pages/master/contribution/faq.adoc diff --git a/EN/modules/ROOT/pages/master/cpu_arch_adp.adoc b/EN/modules/ROOT/pages/master/cpu_os_adaptation/cpu_architecture_adaptation.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/cpu_arch_adp.adoc rename to EN/modules/ROOT/pages/master/cpu_os_adaptation/cpu_architecture_adaptation.adoc diff --git a/EN/modules/ROOT/pages/master/os_arch_adp.adoc b/EN/modules/ROOT/pages/master/cpu_os_adaptation/os_architecture_adaptation.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/os_arch_adp.adoc rename to EN/modules/ROOT/pages/master/cpu_os_adaptation/os_architecture_adaptation.adoc diff --git a/EN/modules/ROOT/pages/master/4.3.adoc b/EN/modules/ROOT/pages/master/developer_guide.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/4.3.adoc rename to EN/modules/ROOT/pages/master/developer_guide.adoc diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/age.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/age.adoc new file mode 100644 index 0000000..3897317 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/age.adoc @@ -0,0 +1,172 @@ +:sectnums: +:sectnumlevels: 5 + += Apache AGE + +== Overview + +Apache AGE is a PostgreSQL extension that provides graph database capabilities to relational databases. AGE stands for *Adaptive Graph Engine*, which brings graph database functionality to PostgreSQL, allowing users to use both relational and graph models in the same database. + +Apache AGE is a top-level project of the Apache Software Foundation and fully supports the openCypher query language (the graph query language used by Neo4j). + +*Core Features:* + +[cols="1,2"] +|=== +| Feature | Description + +| openCypher Support +| Full support for openCypher query language, the industry standard for graph queries + +| Hybrid Database +| Use both relational and graph data models in the same database + +| ACID Transactions +| Full ACID transaction support inherited from PostgreSQL + +| SQL Integration +| Seamless integration of Cypher graph queries with SQL queries + +| Property Graph Model +| Support for property graphs with vertices and edges with attributes + +| Graph Traversal +| Efficient graph traversal and pattern matching capabilities + +| Free and Open Source +| Apache 2.0 License, fully open source +|=== + +== Use Cases + +* Social network analysis (friend relationships, follower relationships, influence analysis) +* Knowledge graph construction and reasoning +* Fraud detection (financial transaction network analysis) +* Recommendation systems (relationship chain-based recommendations) +* Network and IT infrastructure management +* Route planning and logistics optimization +* Access control and permission management + +== Installation + +[TIP] +Source installation was tested on Ubuntu 24.04. + +=== Dependencies + +[literal] +---- +# Ubuntu / Debian +sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libxslt-dev libssl-dev libxml2-utils xsltproc ccache libssl-dev pkg-config + +# Install AGE dependencies +sudo apt install python3 python3-pip python3-dev +pip3 install antlr4-runtime4 +---- + +=== Install from Source + +[literal] +---- +# Download Apache AGE 1.7.0 source package +wget https://github.com/apache/age/releases/download/PG18%2Fv1.7.0-rc0/apache-age-1.7.0-src.tar.gz + +# Extract +tar -xzf apache-age-1.7.0-src.tar.gz +cd apache-age-1.7.0-src + +# Compile and install +make install + +# Or specify IvorySQL installation path +make install PG_CONFIG=/usr/ivory-5/bin/pg_config +---- + +=== Verify Installation + +[literal] +---- +# Check AGE extension +ls /usr/ivory-5/lib/age--*.so +---- + +== Configuration + +=== Modify IvorySQL Configuration + +Edit `postgresql.conf` or `ivorysql.conf`: + +[literal] +---- +# Preload AGE extension (recommended) +shared_preload_libraries = 'age' + +# Or load at database level (no restart required) +# shared_preload_libraries = '' +---- + +[literal] +---- +# Restart IvorySQL for configuration to take effect +# Using systemd +sudo systemctl restart ivorysql-5 + +# Or manually restart +pg_ctl restart -D /usr/ivory-5/data +---- + +=== Create Extension + +Connect to IvorySQL and create the AGE extension: + +[literal] +---- +# Connect to database +psql -U postgres -d postgres + +# Create AGE extension +CREATE EXTENSION age; + +# Verify installation +SELECT * FROM pg_extension WHERE extname = 'age'; +---- + +== Usage + +[literal] +---- +To create a graph, use the create_graph function located in the ag_catalog namespace. + +SELECT create_graph('graph_name'); + +To create a single vertex with label and properties, use the CREATE clause. + +SELECT * +FROM cypher('graph_name', $$ + CREATE (:label {property:"Node A"}) +$$) as (v agtype); + +SELECT * +FROM cypher('graph_name', $$ + CREATE (:label {property:"Node B"}) +$$) as (v agtype); + +To create an edge between two nodes and set its properties: + +SELECT * +FROM cypher('graph_name', $$ + MATCH (a:label), (b:label) + WHERE a.property = 'Node A' AND b.property = 'Node B' + CREATE (a)-[e:RELTYPE {property:a.property + '<->' + b.property}]->(b) + RETURN e +$$) as (e agtype); + +And to query the connected nodes: + +SELECT * from cypher('graph_name', $$ + MATCH (V)-[R]-(V2) + RETURN V,R,V2 +$$) as (V agtype, R agtype, V2 agtype); + +---- + diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc new file mode 100644 index 0000000..4e3270c --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc @@ -0,0 +1,49 @@ +:sectnums: +:sectnumlevels: 5 + + +[discrete] +== IvorySQL Ecosystem Plugin Compatibility List + +IvorySQL, as an advanced open-source database compatible with Oracle and based on PostgreSQL, has powerful extension capabilities and supports a rich ecosystem of plugins. These plugins can help users enhance database functionality in different scenarios, including geospatial information processing, vector retrieval, full-text search, data definition extraction, and path planning. The following is a list of major plugins currently officially compatible with and supported by IvorySQL: + ++ + +[cols="1,2,1,3,3"] +|==== +|*Index*|*Plugin Name*|*Version*|*Function Description*|*Use Cases* +|*1*| xref:master/ecosystem_components/postgis.adoc[postgis] | 3.5.4 | Provides geospatial data support for IvorySQL, including spatial indexes, spatial functions, and geographic object storage | Geographic Information Systems (GIS), map services, location data analysis +|*2*| xref:master/ecosystem_components/pgvector.adoc[pgvector] | 0.8.1 | Supports vector similarity search, can be used to store and retrieve high-dimensional vector data| AI applications, image retrieval, recommendation systems, semantic search +|*3*| xref:master/ecosystem_components/pgddl.adoc[pgddl (DDL Extractor)] | 0.31 | Extracts DDL (Data Definition Language) statements from databases, facilitating version management and migration | Database version control, CI/CD integration, structure comparison and synchronization +|*4*| xref:master/ecosystem_components/pg_cron.adoc[pg_cron]​ | 1.6.0 | Provides database-internal scheduled task scheduling functionality, supports regular SQL statement execution | Data cleanup, regular statistics, automated maintenance tasks +|*5*| xref:master/ecosystem_components/pgsql_http.adoc[pgsql-http]​ | 1.7.0 | Allows HTTP requests to be initiated in SQL, interacting with external web services | Data collection, API integration, microservice calls +|*6*| xref:master/ecosystem_components/plpgsql_check.adoc[plpgsql_check] | 2.8 | Provides static analysis functionality for PL/pgSQL code, can detect potential errors during development | Stored procedure development, code quality checking, debugging and optimization +|*7*| xref:master/ecosystem_components/pgroonga.adoc[pgroonga] | 4.0.4 | Provides full-text search functionality for non-English languages, meeting the needs of high-performance applications | Full-text search capabilities for languages like Chinese, Japanese, and Korean +|*8*| xref:master/ecosystem_components/pgaudit.adoc[pgaudit] | 18.0 | Provides fine-grained auditing, recording database operation logs to support security auditing and compliance checks | Database security auditing, compliance checks, audit report generation +|*9*| xref:master/ecosystem_components/pgrouting.adoc[pgrouting] | 3.8.0 | Provides routing computation for geospatial data, supporting multiple algorithms and data formats | Geospatial analysis, route planning, logistics optimization +|*10*| xref:master/ecosystem_components/system_stats.adoc[system_stats] | 3.2 | Provide functions for accessing system-level statistics. | system monitor +|*11*| xref:master/ecosystem_components/wal2json.adoc[wal2json] | 2.6 | converts database Write-Ahead Log (WAL) changes into structured JSON format | it is primarily used for change data capture (CDC), real-time data replication, event-driven microservices, cache invalidation, and cross-database synchronization +|*12*| xref:master/ecosystem_components/pg_stat_monitor.adoc[pg_stat_monitor] | 2.3.1 | Collects performance statistics and provides query performance insights in a single view and graphically in histogram. | Performance monitoring +|*13*| xref:master/ecosystem_components/pg_ai_query.adoc[pg_ai_query] | 0.1.1 | AI-driven natural language to SQL extension supporting multiple LLMs | AI-assisted querying, natural language database interaction +|*14*| xref:master/ecosystem_components/pg_partman.adoc[pg_partman] | 5.2 | Automates the creation, maintenance, and cleanup of native partition subtables. | Large-Scale Data Storage Management +|*15*| xref:master/ecosystem_components/pgbouncer.adoc[pgbouncer] | 1.25.1 | Lightweight connection pooler for PostgreSQL | High-concurrency connection management, connection reuse, reduced database connection overhead +|*16*| xref:master/ecosystem_components/pg_curl.adoc[pg_curl] | 2.4 | A libcurl-based network transfer extension supporting HTTP/HTTPS, FTP, SMTP, IMAP, and 20+ other protocols, enabling network data transfer operations directly in SQL | REST API integration, email sending, file transfer, external system notifications +|*17*| xref:master/ecosystem_components/pg_textsearch.adoc[pg_textsearch] | 0.6.1 | enhances PostgreSQL with full-text search capabilities, enabling fast text indexing, tokenization, and efficient full-text querying | document and content retrieval +|*18*| xref:master/ecosystem_components/pg_hint_plan.adoc[pg_hint_plan] | PG18 | Controls execution plans via SQL comment hints, optimizing query performance without modifying SQL logic | Query performance optimization, execution plan tuning, database performance analysis +|*19*| xref:master/ecosystem_components/redis_fdw.adoc[redis_fdw] | PG18 | connects PostgreSQL with Redis, supporting standard SQL operations including SELECT, INSERT, UPDATE, and DELETE | unified SQL querying, lightweight data synchronization, transparent cache access, and cross-database data analysis +|*20*| xref:master/ecosystem_components/age.adoc[Apache AGE] | 1.7.0 | Provides graph database capabilities to IvorySQL, supports openCypher query language, enabling hybrid use of relational and graph databases | Social network analysis, knowledge graphs, fraud detection, recommendation systems, route planning +|*21*| xref:master/ecosystem_components/pg_show_plans.adoc[pg_show_plans] | 2.1 | Displays execution plans of all currently running SQL statements, supporting TEXT, JSON, and YAML output formats | Query performance diagnosis, real-time execution plan monitoring, slow query analysis +|*22*| xref:master/ecosystem_components/pg_bulkload.adoc[pg_bulkload] | 3.1.23 | Provides high-speed data loading tool for IvorySQL, which directly imports data into tables bypassing PostgreSQL shared buffers | initial loading of massive data, historical data archiving and cross-database migration +|*23*| xref:master/ecosystem_components/pg_bigm.adoc[pg_bigm] | 1.2 | Equips IvorySQL with bigram full-text search capability, supporting Chinese, Japanese and Korean texts to implement fuzzy retrieval and similarity query efficiently | text search for articles, commodities and addresses in CJK languages +|*24*| xref:master/ecosystem_components/pg_profile.adoc[pg_profile] | 4.11 | Collects statistics on resource-intensive database activities and generates historic workload reports based on delta analysis between two sample points, helping identify performance bottlenecks | Database performance analysis +|*25*| xref:master/ecosystem_components/pg_repack.adoc[pg_repack] | 1.5.3 | Rebuilds tables and indexes online with minimal locking, eliminating storage bloat and achieving effects similar to VACUUM FULL | Online table and index rebuild, storage bloat reclamation +|*26*| xref:master/ecosystem_components/pgdog.adoc[PgDog] | 0.1.45 | High-performance, open-source clustering middleware (proxy tool) designed specifically for PostgreSQL and written in Rust | Connection pooler, load balancer, distributed database +|*27*| xref:master/ecosystem_components/pg_readonly_en.adoc[pg_read_only] | 1.0.5 | Allows to set all PostgreSQL cluster databases read only. | System debugging、disaster recevory +|*28*| xref:master/ecosystem_components/zhparser_en.adoc[zhparser] | master branch | PostgreSQL extension for full-text search of Chinese language (Mandarin Chinese). It implements a Chinese language parser base on the | Search engine、keyword extraction +|*29*| xref:master/ecosystem_components/pgbackrest.adoc[pgBackRest] | 2.58.0 | pgBackRest is a reliable backup and restore solution for PostgreSQL that seamlessly scales up to the largest databases and workloads | Disaster recovery backup, large database backup, off-site/multi-tier disaster recovery +| *30* | xref:master/ecosystem_components/set_user.adoc[set_user] | REL4_2_0 | PostgreSQL security auditing extension with controlled role switching, supporting allowlists, enforced auditing, and blocking of high-risk operations | Controlled role switching, privilege management, audit logging +|==== + +These plugins have all been tested and adapted by the IvorySQL team to ensure stable operation in the IvorySQL environment. Users can select appropriate plugins based on business needs to further enhance the capabilities and flexibility of the database system. + +We will continue to expand and enrich the IvorySQL plugin ecosystem. Community developers are welcome to submit new plugin adaptation suggestions or code contributions. For more detailed usage methods and the latest compatible versions of each plugin, please refer to the corresponding documentation chapters for each plugin. diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_ai_query.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_ai_query.adoc new file mode 100644 index 0000000..7d972f2 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_ai_query.adoc @@ -0,0 +1,321 @@ +:sectnums: +:sectnumlevels: 5 + += pg_ai_query + +== Overview + +pg_ai_query is an AI-powered natural language to SQL extension for IvorySQL/PostgreSQL. It leverages Large Language Models (LLMs) to convert user's natural language descriptions directly into executable SQL query statements. It supports multiple AI models including OpenAI, Anthropic Claude, and Google Gemini. + +Project URL: + +License: Apache-2.0 + +Key features: + +* **Natural Language to SQL**: Convert plain text descriptions into valid SQL queries +* **Multi-model Support**: Supports gpt-4o-mini, gpt-4o, gpt-5, claude-3-haiku-20240307, claude-sonnet-4-5-20250929, claude-4.5-opus and other large models +* **System Table Protection**: Blocks access to `information_schema` and `pg_catalog` +* **Limited Scope**: Only operates on user tables +* **Configurable Limits**: Built-in row limit enforcement +* **API Key Security**: Secure handling of API credentials + +== Quick Start + +The following demonstration uses v0.1.1 as an example. + +=== Installation + +*Prerequisites* + +* PostgreSQL 14+ with development headers +* CMake 3.16+ +* C++20 compatible compiler +* API key from OpenAI, Anthropic, or Google (Gemini) + +*Install dependencies* + +[source,bash] +---- +sudo apt-get install libcurl4-openssl-dev +---- + +*Compile and install IvorySQL* + +If you need to compile IvorySQL from source, you can refer to the following configuration: + +[source,bash] +---- +./configure \ +--prefix=$PWD/inst \ +--enable-cassert \ +--enable-debug \ +--enable-tap-tests \ +--enable-rpath \ +--enable-nls \ +--enable-injection-points \ +--with-tcl \ +--with-python \ +--with-gssapi \ +--with-pam \ +--with-ldap \ +--with-openssl \ +--with-libedit-preferred \ +--with-uuid=e2fs \ +--with-ossp-uuid \ +--with-libxml \ +--with-libxslt \ +--with-perl \ +--with-icu \ +--with-libnuma +---- + +*Compile and install pg_ai_query* + +[source,bash] +---- +git clone --recurse-submodules https://github.com/benodiwal/pg_ai_query.git +cd pg_ai_query +mkdir build && cd build +export PATH="$HOME/works/repo/ivorysql/IvorySQL/inst/bin:$PATH" +cmake .. -DCMAKE_INSTALL_PREFIX=$HOME/works/repo/ivorysql/IvorySQL/inst +make && sudo make install +---- + +*Create extension* + +[source,sql] +---- +CREATE EXTENSION pg_ai_query; +---- + +=== Configuration + +Create a `~/.pg_ai.config` configuration file in your home directory: + +[source,ini] +---- +[general] +log_level = "INFO" +enable_logging = false + +[query] +enforce_limit = true +default_limit = 1000 + +[response] +show_explanation = true +show_warnings = true +show_suggested_visualization = false +use_formatted_response = false + +[anthropic] +# Your Anthropic API key (if using Claude) +api_key = "******" + +# Default model to use (options: claude-sonnet-4-5-20250929) +default_model = "claude-sonnet-4-5-20250929" + +# Custom API endpoint (optional) - for Anthropic-compatible APIs +api_endpoint = "https://open.bigmodel.cn/api/anthropic" + +[prompts] +# Use file paths to read custom prompts +system_prompt = /home/highgo/.pg_ai.prompts +explain_system_prompt = /home/highgo/.pg_ai.explain.prompts +---- + +For more examples, please refer to: + +== Usage Examples + +=== Basic Usage + +[source,sql] +---- +SELECT generate_query('Find all users'); +---- + +Example output: + +---- +[INFO] Text generation successful - model: claude-sonnet-4-5-20250929, response_id: msg_20260209135507cc16362d5d324ccd + + generate_query +-------------------------------------------------------- + SELECT * FROM public.users LIMIT 1000; ++ + -- Explanation: + -- Retrieves all columns and rows from the users table. ++ + -- Warning: INFO: Applied LIMIT 1000 to prevent large result sets. Remove LIMIT if you need all data. ++ + -- Note: Row limit was automatically applied to this query for safety +(1 row) +---- + +Execute the query: + +[source,sql] +---- +SELECT * FROM public.users LIMIT 1000; +---- + +Output: + +---- + id | name | email | age | created_at | city +----+---------------+-------------------+-----+----------------------------+--------------- + 1 | Alice Johnson | alice@example.com | 28 | 2026-02-04 15:47:55.208111 | New York + 2 | Bob Smith | bob@example.com | 35 | 2026-02-04 15:47:55.208111 | San Francisco + 3 | Carol Davis | carol@example.com | 31 | 2026-02-04 15:47:55.208111 | Chicago + 4 | David Wilson | david@example.com | 27 | 2026-02-04 15:47:55.208111 | Seattle + 5 | Eva Brown | eva@example.com | 33 | 2026-02-04 15:47:55.208111 | Boston +(5 rows) +---- + +=== generate_query Examples + +*Generate test data* + +[source,sql] +---- +SELECT generate_query('Generate 100 user records and insert into users table'); +---- + +Output: + +---- +[INFO] Text generation successful - model: claude-sonnet-4-5-20250929, response_id: msg_2026021114092101601c5650864a2d + + generate_query +-------------------------------------------------------------------------------------------------------- + INSERT INTO public.users (name, email, age, city, status) + SELECT 'User_' || generate_series AS name, + 'user' || generate_series || '@example.com' AS email, + (18 + (generate_series % 50)) AS age, + (ARRAY['Beijing','Shanghai','Guangzhou','Shenzhen','Hangzhou'])[1 + (generate_series % 5)] AS city, + 'active' AS status + FROM generate_series(1, 100); ++ + -- Explanation: + -- Generates 100 simulated user records and inserts them into the users table. Data includes auto-generated names, unique emails, random ages (18-67), random cities, and default status. ++ + -- Warnings: + -- 1. INFO: Relies on DEFAULT auto-increment setting for id column in users table, id is not manually inserted. + -- 2. INFO: Uses generate_series function to generate sequence data, which is a PostgreSQL/IvorySQL feature. + -- 3. WARN: Ensure users table is empty or id sequence does not conflict before running, otherwise duplicate inserts may occur. + -- 4. WARN: Email format is simple simulation, actual environment may require more complex logic or duplicate checking. +(1 row) +---- + +*Case-insensitive query* + +[source,sql] +---- +SELECT generate_query('Show users from beijing, beijing is non-Case insensitive'); +---- + +Output: + +---- +[INFO] Text generation successful - model: claude-sonnet-4-5-20250929, response_id: msg_20260211142845878f5f1a5a2f44a7 + + generate_query +----------------------------------------- + SELECT id, name, email, age, created_at, city, status + FROM public.users + WHERE LOWER(city) = LOWER('beijing') LIMIT 100; ++ + -- Explanation: + -- Selects all user details for users located in Beijing, performing a case-insensitive match on the city column. ++ + -- Warnings: + -- 1. INFO: Using LOWER() on both sides ensures case-insensitive matching but may prevent the database from using a standard index on the city column if one exists. + -- 2. INFO: Row limit of 100 applied to prevent large result sets. ++ + -- Note: Row limit was automatically applied to this query for safety +(1 row) +---- + +=== explain_query Examples + +[source,sql] +---- +SELECT explain_query('SELECT * FROM orders WHERE user_id = 12'); +---- + +Output: + +---- +[INFO] Text generation successful - model: claude-sonnet-4-5-20250929, response_id: msg_20260211175909d47a6871bcca4897 + + explain_query +-------------------------------------------------------------------------------------------------------------- + 1. Query Overview ++ + - This query aims to retrieve all records from the orders table where user_id equals 12 (SELECT *). + - This is a typical query that filters data based on a specific field (user_id). ++ + 2. Performance Summary ++ + - Total execution time: 0.021 ms + - Planning time: 0.430 ms + - Total cost: 18.12 + - Rows returned: 0 rows (Actual Rows: 0) + - Rows scanned: 0 rows (Rows Removed by Filter: 0) ++ + 3. Execution Plan Analysis ++ + - Key step: Sequential Scan + - The database performed a full table scan on the orders table. + - The planner estimated finding 3 rows, but actual execution returned 0 rows. + - Filter condition: orders.user_id = 12, which means the database must read every row in the table to check this condition. ++ + 4. Performance Issues ++ + - Full table scan risk: Although the current table data volume is small (execution time is only 0.021ms), using Seq Scan (sequential scan) means the database is not using an index. If the orders table grows to millions of rows over time, this query approach will become extremely slow (high I/O consumption). + - Missing index: The plan shows no index was used to locate rows where user_id = 12, indicating that a necessary B-Tree index may be missing on the user_id column. ++ + 5. Optimization Recommendations ++ + - Primary recommendation: Create an index on the user_id column to avoid full table scans. This will transform the query from O(N) (scanning all rows) to O(log N) (index lookup). + - SQL optimization example: ++ + CREATE INDEX idx_orders_user_id ON orders(user_id); ++ + 6. Index Recommendations ++ + - Recommended index: Create a B-Tree index on the user_id column of the orders table. + - Reason: The query condition is based on equality comparison (=) of user_id. After creating the index, IvorySQL (PostgreSQL) will be able to use the index to quickly locate data, significantly reducing query time and resource consumption, especially when data volume is large. +(1 row) +---- + +== Best Practices + +=== Prompt Writing Guidelines + +* **Use English**: While AI supports multiple languages, English works best +* **Understand your database structure**: The more you understand your database structure, the more accurate the generated queries will be +* **Refine iteratively**: Start broad, then add details step by step to improve results +* **Be explicit**: If you know specific tables or columns, mention them in your prompts - this helps AI generate more precise queries + +=== Error Handling Examples + +When tables referenced in the query do not exist, the system returns an error message: + +[source,sql] +---- +SELECT generate_query('List all products and prices'); +---- + +Error output: + +---- +[INFO] Text generation successful - model: claude-sonnet-4-5-20250929, response_id: msg_20260209135642777cbc5c82ca4a85 + +ERROR: Query generation failed: Cannot generate query. Referenced table(s) for 'products' or 'goods' do not exist in the database. Available tables: public.orders, public.student_scores, public.users, sys.dual +---- + +In this case, the AI informs you of the available table list to help adjust your query. + diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_bigm.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_bigm.adoc new file mode 100644 index 0000000..d29e795 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_bigm.adoc @@ -0,0 +1,65 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_bigm + +== Overview +pg_bigm provides bigram-based full-text search capabilities for IvorySQL. It is optimized for Chinese, Japanese, and Korean (CJK) text, enabling fuzzy search and similarity queries. + +== Installation +The IvorySQL installation package already includes the pg_bigm extension. If you installed IvorySQL using the installation package, you typically do not need to manually install pg_bigm. For other installation methods, refer to the source code installation steps below. + +[TIP] +The source code installation environment is Ubuntu 24.04 (x86_64), with IvorySQL 5 or later already installed at `/usr/ivory-5`. + +=== Source Code Installation +Download the pg_bigm v1.2 source code from https://github.com/pgbigm/pg_bigm/releases/tag/v1.2-20250903. + +** Install pg_bigm +[literal] +---- +# Extract the source code and enter its directory +cd pg_bigm-1.2-20250903 + +# Build the code, setting the PG_CONFIG environment variable to the path of pg_config, e.g., /usr/ivory-5/bin/pg_config +make USE_PGXS=1 PG_CONFIG=/usr/ivory-5/bin/pg_config +sudo make USE_PGXS=1 PG_CONFIG=/usr/ivory-5/bin/pg_config install +---- + +== Create Extension +Connect to the database via psql and execute the following commands: +[literal] +---- +ivorysql=# CREATE EXTENSION pg_bigm; +CREATE EXTENSION + +ivorysql=# SELECT * FROM pg_available_extensions WHERE name = 'pg_bigm'; + name | default_version | installed_version | comment +---------+-----------------+-------------------+------------------------------------------------------------------ + pg_bigm | 1.2 | 1.2 | text similarity measurement and index searching based on bigrams +(1 row) +---- + +== Usage +[literal] +---- +# First, create the index +CREATE TABLE pg_tools (tool text, description text); + +INSERT INTO pg_tools VALUES ('pg_hint_plan', 'Tool that allows a user to specify an optimizer HINT to PostgreSQL'); +INSERT INTO pg_tools VALUES ('pg_dbms_stats', 'Tool that allows a user to stabilize planner statistics in PostgreSQL'); +INSERT INTO pg_tools VALUES ('pg_bigm', 'Tool that provides 2-gram full text search capability in PostgreSQL'); +INSERT INTO pg_tools VALUES ('pg_trgm', 'Tool that provides 3-gram full text search capability in PostgreSQL'); + +CREATE INDEX pg_tools_idx ON pg_tools USING gin (description gin_bigm_ops); + +# Execute full-text search +SELECT * FROM pg_tools WHERE description LIKE '%search%'; + + tool | description +---------+--------------------------------------------------------------------- + pg_bigm | Tool that provides 2-gram full text search capability in PostgreSQL + pg_trgm | Tool that provides 3-gram full text search capability in PostgreSQL +(2 rows) +---- diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_bulkload.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_bulkload.adoc new file mode 100644 index 0000000..a35e5c7 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_bulkload.adoc @@ -0,0 +1,81 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_bulkload + +== Overview +pg_bulkload is a high-performance bulk data loading extension for PostgreSQL. It bypasses conventional read/write mechanisms to significantly improve the efficiency of importing text data into the database. It supports data filtering, error tolerance, parallel loading, and checkpoint recovery, making it suitable for scenarios such as large-scale historical data migration, data warehouse batch synchronization, and offline log ingestion. Compared to the native COPY command, pg_bulkload delivers a notable improvement in import speed. + +== Installation +The IvorySQL installation package already includes the pg_bulkload extension. If you installed IvorySQL using the installation package, you typically do not need to manually install pg_bulkload. For other installation methods, refer to the source code installation steps below. + +[TIP] +The source code installation environment is Ubuntu 24.04 (x86_64), with IvorySQL 5 or later already installed at `/usr/ivory-5`. + +=== Source Code Installation +Download the pg_bulkload v3.1.23 source code from https://github.com/ossc-db/pg_bulkload/releases/tag/VERSION3_1_23. + +** Install Dependencies +[literal] +---- +sudo apt install gawk +---- + +** Install pg_bulkload +[literal] +---- +# Extract the source code and enter its directory +cd pg_bulkload-VERSION3_1_23 + +# Modify the Makefile to accommodate IvorySQL's non-PIE static libraries + + LDFLAGS+=-Wl,--build-id ++ ++# Workaround for non-PIE static libraries (e.g., IvorySQL's libpgcommon.a) ++# Some distributions build libpgcommon.a without -fPIE, causing link failures ++# when the system defaults to PIE executables. ++ifdef DISABLE_PIE ++CFLAGS+=-no-pie ++LDFLAGS+=-no-pie ++endif + +# Build the code, setting the PG_CONFIG environment variable to the path of pg_config, e.g., /usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config clean +make PG_CONFIG=/usr/ivory-5/bin/pg_config DISABLE_PIE=1 +sudo make PG_CONFIG=/usr/ivory-5/bin/pg_config +---- + +== Create Extension +Connect to the database via psql and execute the following commands: +[literal] +---- +ivorysql=# CREATE extension pg_bulkload; +CREATE EXTENSION + +ivorysql=# SELECT * FROM pg_available_extensions WHERE name = 'pg_bulkload'; + name | default_version | installed_version | comment +-------------+-----------------+-------------------+----------------------------------------------------------------- + pg_bulkload | 3.1.23 | 3.1.23 | pg_bulkload is a high speed data loading utility for PostgreSQL +(1 row) +---- + +== Usage +[literal] +---- +ivorysql=# create database testdb; +CREATE DATABASE + +ivorysql=# \c testdb +You are now connected to database "testdb" as user "highgo". + +testdb=# create table tb_asher (id int,name text); +CREATE TABLE +testdb=# \q + +# Generate a sample CSV file +seq 100000| awk '{print $0"|asher"}' > bulk_asher.txt + +# Load the data from bulk_asher.txt into the tb_asher table in the testdb database +/usr/ivory-5/bin/pg_bulkload -i ./bulk_asher.txt -O tb_asher -l ./tb_asher_output.log -P ./tb_asher_bad.txt -o "TYPE=CSV" -o "DELIMITER=|" -d testdb -U highgo -h 127.0.0.1 +---- diff --git a/EN/modules/ROOT/pages/master/5.4.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_cron.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/5.4.adoc rename to EN/modules/ROOT/pages/master/ecosystem_components/pg_cron.adoc diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_curl.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_curl.adoc new file mode 100644 index 0000000..aeb492e --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_curl.adoc @@ -0,0 +1,198 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_curl + +== Overview + +pg_curl is a PostgreSQL extension based on libcurl that brings cURL's powerful data transfer capabilities directly into the database, allowing users to perform network protocol data transfer operations through SQL function calls without relying on external programs or middleware. + +Unlike lightweight extensions that only support HTTP, pg_curl fully encapsulates the libcurl easy interface API and supports more than twenty protocols including DICT, FILE, FTP, FTPS, GOPHER, HTTP, HTTPS, IMAP, IMAPS, LDAP, LDAPS, MQTT, POP3, POP3S, RTMP, RTMPS, RTSP, SCP, SFTP, SMB, SMBS, SMTP, SMTPS, TELNET, TFTP, WS, and WSS, providing extremely rich network interaction capabilities. + +Typical use cases include: sending HTTP notifications to external systems from triggers or stored procedures; sending emails directly from the database; uploading or downloading files via FTP/SFTP; calling third-party REST APIs and writing results back to database tables; and integrating with MQTT message brokers, LDAP directories, and other backend services in data processing workflows. + +pg_curl is licensed under the MIT License. Project repository: https://github.com/RekGRpth/pg_curl + +== Features + +* *Multi-protocol support*: Based on libcurl, supports more than twenty protocols including HTTP/HTTPS, FTP/FTPS, SMTP/SMTPS, IMAP, POP3, SCP, SFTP, MQTT, and LDAP, covering the vast majority of network integration scenarios. +* *Full HTTP method support*: Supports standard HTTP methods including GET, POST (URL-encoded, JSON, multipart/form-data), PUT, DELETE, and PATCH. Custom methods can be specified via `curl_easy_setopt_customrequest`. +* *Flexible request construction*: Provides fine-grained `curl_easy_setopt_*` functions to precisely control request headers, authentication, timeouts, proxies, TLS certificates, cookies, and more. +* *Multiple authentication methods*: Supports Basic Auth (username/password), Bearer Token, NTLM, Digest, OAuth, and other authentication mechanisms. +* *File transfer*: Supports uploading (via `curl_easy_setopt_readdata`) and downloading files over FTP/FTPS/SFTP/SCP, with upload data passed directly as `bytea` type. +* *Email sending*: Send emails directly from the database over SMTP/SMTPS, with support for setting sender, recipient, mail headers, and MIME body. +* *Response parsing*: Provides `curl_easy_getinfo_data_in()`, `curl_easy_getinfo_header_in()`, and other functions to retrieve response body and headers. Headers can be parsed into a key-value table using regular expressions. +* *Error handling*: `curl_easy_getinfo_errcode()`, `curl_easy_getinfo_errdesc()`, and `curl_easy_getinfo_errbuf()` provide complete error codes and descriptions for debugging. +* *Timeout and interruption*: By registering a progress callback with libcurl that periodically checks PostgreSQL's cancellation flag (`QueryCancelPending`), requests can be interrupted by `statement_timeout`, `pg_cancel_backend()`, and similar mechanisms, preventing long-running requests from blocking connections. +* *URL encoding utilities*: Built-in `curl_easy_escape()` and `curl_easy_unescape()` functions for URL encoding and decoding in SQL. + +== Installation + +[TIP] +The following example environment is Ubuntu 24.04 (x86_64) with IvorySQL 5 or later installed at `/usr/local/ivorysql/ivorysql-5`. + +=== Install Dependencies + +pg_curl depends on the system libcurl development library. Ensure it is installed before building: + +[source,shell] +---- +sudo apt install libcurl4-openssl-dev +---- + +=== Build and Install from Source + +Clone the source code from https://github.com/RekGRpth/pg_curl and run the build: + +[source,shell] +---- +git clone https://github.com/RekGRpth/pg_curl.git +cd pg_curl +# Ensure pg_config is accessible, or specify it explicitly via PG_CONFIG +PG_CONFIG=/usr/local/ivorysql/ivorysql-5/bin/pg_config make +PG_CONFIG=/usr/local/ivorysql/ivorysql-5/bin/pg_config sudo make install +---- + +After a successful installation, `pg_curl.so` and the corresponding SQL scripts will be placed in IvorySQL's extension directory. + +== Create Extension and Verify Version + +Connect to the database using psql (either PG mode port 5432 or Oracle mode port 1521), then run: + +[source,sql] +---- +CREATE EXTENSION pg_curl; + +SELECT name, default_version, installed_version, comment + FROM pg_available_extensions + WHERE name = 'pg_curl'; +---- + +Expected output: + +[source,text] +---- + name | default_version | installed_version | comment +---------+-----------------+-------------------+---------------------------------------------------------------------------------------------------------------------------------------- + pg_curl | 2.4 | 2.4 | PostgreSQL cURL allows most curl actions, including data transfer with URL syntax via HTTP, HTTPS, FTP, FTPS, GOPHER, TFTP, SCP, ... +(1 row) +---- + +== Usage + +pg_curl follows a "configure, execute, retrieve" pattern. The core workflow is: + +. Call `curl_easy_reset()` to initialize (or reset) the current cURL session. +. Call `curl_easy_setopt_*`, `curl_header_append`, `curl_postfield_append`, and other functions to configure request parameters. +. Call `curl_easy_perform()` to execute the request. +. Call `curl_easy_getinfo_*` functions to retrieve the response or error information. + +=== HTTP GET + +[source,sql] +---- +BEGIN; +SELECT curl_easy_reset(); +SELECT curl_easy_setopt_url('https://httpbin.org/get?'); +SELECT curl_url_append('key1', 'value1'); +SELECT curl_url_append('key2', 'hello world'); -- automatically URL-encoded +SELECT curl_easy_perform(); +SELECT convert_from(curl_easy_getinfo_data_in(), 'utf-8'); +END; +---- + +=== HTTP POST (JSON) + +Form POST (`curl_postfield_append`) and multipart POST (`curl_mime_data`) follow the same pattern — simply replace the data-setting function. + +[source,sql] +---- +BEGIN; +SELECT curl_easy_reset(); +SELECT curl_easy_setopt_postfields(convert_to('{"name":"IvorySQL"}', 'utf-8')); +SELECT curl_easy_setopt_url('https://httpbin.org/post'); +SELECT curl_header_append('Content-Type', 'application/json; charset=utf-8'); +SELECT curl_easy_perform(); +SELECT convert_from(curl_easy_getinfo_data_in(), 'utf-8'); +END; +---- + +=== Authentication and Request Headers + +Basic Auth uses `curl_easy_setopt_username` / `curl_easy_setopt_password`. Bearer Token and other custom headers are added via `curl_header_append`: + +[source,sql] +---- +BEGIN; +SELECT curl_easy_reset(); +SELECT curl_easy_setopt_url('https://httpbin.org/bearer'); +SELECT curl_header_append('Authorization', 'Bearer '); +SELECT curl_easy_perform(); +SELECT convert_from(curl_easy_getinfo_data_in(), 'utf-8')::jsonb; +END; +---- + +=== Error Handling and Timeouts + +After `curl_easy_perform()` returns, check the execution status with the following functions (`errcode` of 0 indicates success): + +[source,sql] +---- +BEGIN; +SELECT curl_easy_reset(); +SELECT curl_easy_setopt_url('https://httpbin.org/get'); +SELECT curl_easy_perform(); +SELECT + curl_easy_getinfo_errcode() AS errcode, + curl_easy_getinfo_errdesc() AS errdesc; +END; +---- + +pg_curl checks PostgreSQL's cancellation flag via a libcurl progress callback. When `statement_timeout` fires, the in-progress request is immediately aborted and the current transaction is rolled back: + +[source,sql] +---- +BEGIN; +SET LOCAL statement_timeout = '3s'; +SELECT curl_easy_reset(); +SELECT curl_easy_setopt_url('https://httpbin.org/delay/10'); +SELECT curl_easy_perform(); -- raises query_canceled after timeout, transaction rolls back +END; +---- + +== IvorySQL Adaptation Notes + +=== Dual-Port Compatibility + +IvorySQL uses a dual-port architecture supporting both PostgreSQL mode (default port *5432*) and Oracle-compatible mode (default port *1521*): + +* Port *5432* (PG mode): accepts standard PostgreSQL SQL syntax. +* Port *1521* (Oracle mode): accepts Oracle-compatible SQL syntax. + +pg_curl works normally in both modes. `CREATE EXTENSION pg_curl` succeeds on both port 1521 and port 5432. + +=== Oracle Mode Notes + +When using pg_curl in Oracle mode (port 1521): + +* All pg_curl functions are invoked via plain `SELECT` statements (e.g. `SELECT curl_easy_reset()`), which are valid in Oracle mode. The Oracle-style `SELECT curl_easy_reset() FROM DUAL` syntax also works. +* The `ivorysql.enable_emptystring_to_NULL` parameter affects how empty strings are handled. The regression tests set `SET ivorysql.enable_emptystring_to_NULL = 'off'` to ensure empty-value parameters (such as empty form fields) are passed as expected. Configure this parameter according to your production requirements. + +=== Using pg_curl in PL/iSQL + +pg_curl can be called from IvorySQL's PL/iSQL (the Oracle-compatible procedural language) to push external notifications automatically when data changes. PL/iSQL uses Oracle-style `IS` syntax for procedure declarations, and function return values are discarded by assigning them to a local variable: + +[source,sql] +---- +-- Call pg_curl inside a PL/iSQL stored procedure (Oracle mode, IS syntax) +CREATE OR REPLACE PROCEDURE notify_external(p_payload IN VARCHAR2) IS + v_ok BOOLEAN; +BEGIN + v_ok := curl_easy_reset(); + v_ok := curl_easy_setopt_postfields(convert_to(p_payload, 'utf-8')); + v_ok := curl_easy_setopt_url('https://hooks.example.com/notify'); + v_ok := curl_header_append('Content-Type', 'application/json'); + v_ok := curl_easy_perform(); +END; +---- diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_hint_plan.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_hint_plan.adoc new file mode 100644 index 0000000..73889fa --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_hint_plan.adoc @@ -0,0 +1,211 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_hint_plan + +== Overview + +pg_hint_plan is an extension that allows controlling PostgreSQL/IvorySQL execution plans using SQL comment hints. It optimizes query performance without modifying SQL logic and works in both PostgreSQL and Oracle compatibility modes. + + +== Installation + +[TIP] +The source installation environment is Ubuntu 24.04(x86_64), with IvorySQL 5 or above installed at /usr/ivory-5 + +=== Install from Source + +[literal] +---- +# Clone PG18 branch source code + +git clone --branch PG18 https://github.com/ossc-db/pg_hint_plan.git +cd pg_hint_plan + +# Compile and install the extension +make PG_CONFIG=/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config install + +---- + +=== Modify Database Configuration File + +Modify the ivorysql.conf file to add pg_hint_plan to shared_preload_libraries: + +[literal] +---- +shared_preload_libraries = 'gb18030_2022, liboracle_parser, ivorysql_ora, pg_hint_plan' +---- + +Restart the database for the configuration to take effect. + +=== Load Extension + +[literal] +---- +postgres=# LOAD 'pg_hint_plan'; +LOAD +---- + +== Hint Types + +pg_hint_plan provides various types of hints to control different aspects of query execution plans. + +=== Scan Method Hints + +Hints specifying table scan methods: + +* `SeqScan(table)` - Sequential scan +* `IndexScan(table[ index])` - Index scan, optionally specifying index name +* `BitmapScan(table[ index])` - Bitmap scan, optionally specifying index name +* `TidScan(table)` - TID scan + +Negative form hints (prohibit using a specific scan method): + +* `NoSeqScan(table)` +* `NoIndexScan(table)` +* `NoBitmapScan(table)` +* `NoTidScan(table)` + +=== Join Method Hints + +Hints specifying table join methods: + +* `HashJoin(table table[ table...])` - Hash join +* `NestedLoop(table table[ table...])` - Nested loop join +* `MergeJoin(table table[ table...])` - Merge join + +Negative form hints: + +* `NoHashJoin(table table)` +* `NoNestedLoop(table table)` +* `NoMergeJoin(table table)` + +=== Join Order Hints + +* `Leading(table table[ table...])` - Specify join order, tables are joined in sequence + +=== Parallel Execution Hints + +* `Parallel(table count[ workers])` - Set number of parallel worker processes + * `count` - Whether to use parallel (0 means no, 1 means yes) + * `workers` - Number of parallel worker processes (optional, defaults to planner-calculated value) + +=== Other Hints + +* `Set(enable_*)` - Set GUC parameters +* `Rows(table table[ table...] correction)` - Correct rows estimate + +== Usage Examples + +=== Basic Usage Example + +Using HashJoin and SeqScan hints: + +[literal] +---- +postgres=# /*+ + HashJoin(a b) + SeqScan(a) +*/ +EXPLAIN SELECT * +FROM pgbench_branches b +JOIN pgbench_accounts a ON b.bid = a.bid +ORDER BY a.aid; +---- + +=== Complex Multi-Hint Example + +Combining multiple hints to control execution plan for complex queries: + +[literal] +---- +postgres=# /*+ + NestedLoop(t1 t2) + IndexScan(t2 t2_pkey) + MergeJoin(t1 t2 t3) + Leading(t1 t2 t3) +*/ +EXPLAIN SELECT * FROM table1 t1 +JOIN table2 t2 ON t1.id = t2.id +JOIN table3 t3 ON t2.id = t3.id; +---- + +=== Parallel Execution Example + +Setting parallel degree and join methods for multiple tables: + +[literal] +---- +postgres=# /*+ + Parallel(t1 3) + Parallel(t2 5) + HashJoin(t1 t2) +*/ +EXPLAIN SELECT * FROM large_table1 t1 +JOIN large_table2 t2 ON t1.id = t2.id; +---- + +=== Specify Index Scan Example + +Force scanning using a specific index: + +[literal] +---- +postgres=# /*+ + IndexScan(employees emp_name_idx) +*/ +EXPLAIN SELECT * FROM employees +WHERE last_name = 'SMITH'; +---- + +== Hint Table (Optional Feature) + +pg_hint_plan provides an optional hint table feature for persistently storing hints. + +=== Create Hint Table + +[literal] +---- +postgres=# CREATE TABLE hint_plan.hints ( + id serial PRIMARY KEY, + norm_query_string text NOT NULL, + application_name text, + hints text NOT NULL, + CONSTRAINT hint_plan_hints_norm_query_string_key UNIQUE (norm_query_string, application_name) +); +---- + +=== Insert Hints + +[literal] +---- +postgres=# INSERT INTO hint_plan.hints (norm_query_string, application_name, hints) +VALUES ( + 'SELECT * FROM table1 WHERE id = ?;', + 'psql', + 'SeqScan(table1)' +); +---- + +Through the hint table, hints can be automatically applied to specific queries without modifying SQL statements. + +== IvorySQL Compatibility + +=== Oracle Compatibility Mode + +pg_hint_plan works in both PostgreSQL and Oracle compatibility modes. The hint syntax remains consistent and is compatible with IvorySQL-specific data types (VARCHAR2, NUMBER, etc.). + +=== Oracle Mode Example + +[literal] +---- +-- Connect to port 1521 (Oracle mode) +postgres=# /*+ + IndexScan(employees emp_name_idx) +*/ +SELECT * FROM employees WHERE last_name = 'SMITH'; +---- + +In Oracle compatibility mode, hint usage is identical to PostgreSQL mode and can normally control query execution plans containing IvorySQL-specific data types and syntax. diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_partman.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_partman.adoc new file mode 100644 index 0000000..9270a14 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_partman.adoc @@ -0,0 +1,195 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_partman + +== Overview +pg_partman is used to simplify and automate the management of PostgreSQL's native declarative partitioned tables. It provides functionality for automatic creation, maintenance, and cleanup of partitions, especially for time-based and integer-based range partitioning. + +PostgreSQL supports 3 types of partitioning: range partitioning, list partitioning, and hash partitioning. Among these, the pg_partman extension does not support hash partitioning and has limited support for list partitioning. It is primarily designed for range partitioning based on time (such as day, week, month) or numeric ranges (such as auto-increment IDs). + +Note: Since pg_partman is a PostgreSQL extension specifically designed for PG native partition management, it is not supported under Oracle compatibility mode. + +Project URL: + +Version: 5.2-STABLE + +License: PostgreSQL License + +== Installation and Enablement + +=== Prerequisites +IvorySQL 5.0 is installed, and the pg_config command can be successfully executed from the command line. + +=== Source Installation +Get the source code from https://github.com/pgpartman/pg_partman: +[literal] +---- +git clone https://github.com/pgpartman/pg_partman.git +cd pg_partman +git switch 5.2-STABLE + +make && sudo make install +---- + +=== Enable Extension +[literal] +---- +-- Log in to the database +CREATE SCHEMA partman; +CREATE EXTENSION pg_partman SCHEMA partman; +---- + +[NOTE] +After pg_partman is installed and enabled, it does not automatically create a SCHEMA. If not specifically specified, PostgreSQL will install the extension objects into the first valid SCHEMA in the current search_path (usually public). It is recommended to install it into a dedicated SCHEMA. + +== Usage Workflow +pg_partman itself does not create partitioned parent tables. Its main function is to automatically create and maintain partition child tables based on existing partitioned tables. Therefore, before using pg_partman, you must first manually create a parent table with partitioning enabled. After the parent table is created, register it with pg_partman for management. During registration, you will specify the partitioning strategy and other parameters. Upon successful registration, a template table named template_[schema]_[table_name] will be created in the database. This template table does not store data but serves as the template for all child partitions. After registration, you must create and clean up partitions by manually calling or automatically calling run_maintenance()/run_maintenance_proc() through scripts. + +Below are two use cases. For other usage requirements, please refer to the https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md[pg_partman official documentation]. + +== Case 1: Time-based Partitioning +Scenario: There is a logs table that generates large amounts of data daily. You want to automatically create child partitions by day and retain only the last 30 days of data, automatically deleting older partitions. + +=== Manually Create Partitioned Parent Table +[literal] +---- +CREATE TABLE public.logs ( + id BIGSERIAL, + log_time TIMESTAMPTZ NOT NULL DEFAULT now(), + message TEXT +) PARTITION BY RANGE (log_time); +---- + +=== Register Partitioned Parent Table with pg_partman +Register the logs table with daily partitioning, pre-create 3 future partitions starting from the specified date, and retain the last 30 days: +[literal] +---- +SELECT partman.create_parent( + p_parent_table => 'public.logs', -- Partitioned parent table to manage, must include SCHEMA explicitly + p_control => 'log_time', -- Partition column + p_interval => '1 day', -- Partition by day + p_type => 'range', -- Use range partitioning + p_premake => 3, -- Pre-create 3 future partitions + p_start_partition => '2025-12-24', -- Starting partition + p_default_table => false -- Whether to create a default partition +); +---- + +The above SQL will create daily partitions starting from 2025-12-24 and pre-create 3 future partitions based on the current server time. Partition naming uses the partition table's start time as a suffix. + +[literal] +---- +INSERT INTO public.logs (log_time) VALUES ('2025-12-28 00:01:00'); +---- + +=== Configure Retention (Optional) +[literal] +---- +UPDATE partman.part_config +SET retention = '30 days' +WHERE parent_table = 'public.logs'; +---- + +=== Manual Maintenance +[literal] +---- +select partman.run_maintenance(); +---- +Or +[literal] +---- +CALL partman.run_maintenance_proc(); +---- + +[NOTE] +==== +When calling run_maintenance(), new partition tables are automatically pre-generated based on the partitioning strategy. The pre-generation baseline is the data in the table's partition column. If the most recently inserted data has a log_time of 20251228, then three child partitions (logs_p20251229/logs_p20251230/logs_p20251231) will be pre-generated based on this baseline. + +Additionally, run_maintenance() also executes the cleanup mechanism. The cleanup mechanism is based on server time. In the example above, child tables older than 30 days from the current server time will be removed from the parent table's records, but not actually deleted to avoid accidental data loss. At this point, using \d+ logs to view the partition information of the parent table, you won't see the tables removed more than 30 days ago. However, using \dt to view all tables in the database, you can see that the removed child partitions still exist. These child tables can be manually deleted. +==== + +=== Automatic Maintenance +Using the operating system's crontab: + +Create a shell script /usr/local/bin/partman_maintenance.sh: +[literal] +---- +cd /usr/local/bin +vi partman_maintenance.sh +---- +Script content: +[literal] +---- +#!/bin/bash +psql -U [db_username] -d [db_name] -c "CALL partman.run_maintenance_proc();" +---- + +Grant execute permission: +[literal] +---- +chmod +x partman_maintenance.sh +---- + +Configure execution schedule: +[literal] +---- +crontab -e +# Add a line to execute once daily at 1 AM +0 1 * * * /usr/local/bin/partman_maintenance.sh +---- + +== Case 2: Auto-increment ID-based Partitioning +Scenario: Create an orders table with a new partition for every 10,000 IDs. + +=== Manually Create Partitioned Parent Table +[literal] +---- +CREATE TABLE orders ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, -- Partition key + amount NUMERIC +); +---- + +=== Register Partitioned Parent Table with pg_partman +[literal] +---- +SELECT partman.create_parent( + p_parent_table => 'public.orders', -- Partitioned parent table to manage, must include SCHEMA + p_control => 'id', -- Partition column + p_interval => '10000', -- One partition per 10,000 records + p_type => 'range', -- Use range partitioning + p_premake => 3, -- Pre-create 3 future partitions + p_start_partition => '0', -- Starting partition + p_default_table => false -- Whether to create a default partition +); +---- + +=== Manual Maintenance +[literal] +---- +CALL partman.run_maintenance_proc(); +---- + +== Common Operations + +=== Get Partitioned Parent Tables Managed by pg_partman +[literal] +---- +SELECT parent_table FROM partman.part_config; +---- + +=== Remove a Partitioned Parent Table from pg_partman Management +Delete maintenance information: +[literal] +---- +DELETE FROM partman.part_config +WHERE parent_table = 'table_name'; +---- + +Delete the template: +[literal] +---- +DROP TABLE template_[schema]_[table_name]; +---- diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_profile.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_profile.adoc new file mode 100644 index 0000000..16d1269 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_profile.adoc @@ -0,0 +1,106 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_profile + +== Overview +pg_profile is an extension tool for PostgreSQL database performance analysis. It is mainly used to collect resource-intensive activities in a target database, helping users gain in-depth insight into database runtime status, identify performance bottlenecks, and perform optimization. Its report is essentially a "delta analysis between two sample points", deriving the incremental load within the sampling interval, with sample point sequence numbers starting from 1. + +pg_profile hard-depends on two extensions: dblink and plpgsql (it is written entirely in SQL and PL/pgSQL and requires no external libraries or software). Additionally, it is recommended to install the pg_stat_statements extension to obtain SQL statement-level statistics in the reports. The version of pg_profile is strongly tied to the version of PostgreSQL. For the specific version support, please refer to the README in the official pg_profile github repository. + +Both the PG mode and Oracle compatibility mode of IvorySQL have been adapted for pg_profile. + +Project address: + +License: PostgreSQL License + +== Installation and Enabling + +=== Source Code Compilation + +Perform source code compilation and installation: +[literal] +---- +# Build and install pg_profile and its dependent extensions +make -C contrib/dblink install +make -C contrib/pg_stat_statements install +make -C contrib/pg_profile install +---- + +The installation artifacts are `pg_profile.control` and `pg_profile--4.11.sql`. + +=== Modify Configuration +The pg_stat_statements that pg_profile depends on must be loaded at server startup via `shared_preload_libraries`. + +Edit `ivorysql.conf` and append `pg_stat_statements` to the end of the `shared_preload_libraries` configuration item: +[literal] +---- +# ivorysql.conf +shared_preload_libraries = 'liboracle_parser, ivorysql_ora, pg_stat_statements' +---- + +=== Restart the Service +[literal] +---- +pg_ctl -D $PGDATA restart +---- + +=== Install the Extension +The commands are identical in both PG mode and Oracle mode sessions: +[literal] +---- +CREATE SCHEMA profile; +CREATE SCHEMA dblink; +CREATE SCHEMA statements; +CREATE EXTENSION dblink SCHEMA dblink; +CREATE EXTENSION pg_stat_statements SCHEMA statements; +CREATE EXTENSION pg_profile SCHEMA profile; + +SELECT extname, extversion FROM pg_extension WHERE extname = 'pg_profile'; + extname | extversion +------------+------------ + pg_profile | 4.11 +---- + +== Usage Workflow + +=== Create Samples +[literal] +---- +-- First sampling +SELECT * FROM profile.take_sample(); + server | result | elapsed +--------+--------+------------- + local | OK | 00:00:01.34 + +-- ...after the business workload runs for a while, sample again +SELECT * FROM profile.take_sample(); +---- + +In production environments, periodic sampling via scheduled tasks is typically used (e.g., once every 30 minutes), which can be combined with pg_cron or an external crontab: +[literal] +---- +*/30 * * * * psql -c 'SELECT profile.take_sample()' >/dev/null +---- + +=== View Samples +[literal] +---- +SELECT sample, sample_time, sizes_collected FROM profile.show_samples(); + sample | sample_time | sizes_collected +--------+------------------------+----------------- + 1 | 2026-06-12 09:00:00+00 | t + 2 | 2026-06-12 09:30:00+00 | t +---- + +=== Generate Reports +[literal] +---- +-- Generate the workload report between sample 1 and sample 2 (HTML text) +\o report_1_2.html +SELECT profile.get_report(1, 2); +\o +---- + +The above functions are invoked and produce identical results in an Oracle mode session (Oracle port/1521). diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_readonly_en.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_readonly_en.adoc new file mode 100644 index 0000000..19f1d38 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_readonly_en.adoc @@ -0,0 +1,94 @@ +:sectnums: +:sectnumlevels: 5 + += pg_readonly + +== Overview +pg_readonly is a plugin that can set all PostgreSQL databases to read-only mode. + +pg_readonly does not have specific GUC parameters. It relies on a global flag in memory to manage the read-only state and provides SQL functions to set or query this global flag. +This global flag is cluster-level: either all databases in the cluster are read-only, or all are read-write. + +Read-only mode is implemented by filtering SQL statements. +[literal] +---- +SELECT statements without write-operation functions are allowed; +DML (INSERT, UPDATE, DELETE) and DDL statements including TRUNCATE are completely disallowed; +DCL statements GRANT and REVOKE are also prohibited; +---- + +== Installation + +[TIP] +The source installation environment is Ubuntu 24.04 (x86_64), with IvorySQL already installed at /path-to/ivorysql + +=== Source Installation + +[literal] +---- +# Download the 1.0.6 source package 1.0.6.tar.gz from https://github.com/pierreforstmann/pg_readonly/releases/tag/1.0.6 +tar xvf 1.0.6.tar.gz +cd pg_readonly-1.0.6 + +# Compile and install +make PG_CONFIF=/path-to/ivorysql/bin/pg_config +make PG_CONFIF=/path-to/ivorysql/bin/pg_config install +---- + +=== Configure Preloaded Libraries +Modify the ivorysql.conf file in the data directory to append pg_readonly to shared_preload_libraries. +shared_preload_libraries = 'pg_readonly' + +== Create Extension and Verify pg_readonly Version + +Connect to the database using psql and execute the following commands: +[literal] +---- +ivorysql=# CREATE EXTENSION pg_readonly; +CREATE EXTENSION + +ivorysql=# SELECT * FROM pg_available_extensions WHERE name = 'pg_readonly'; + name | default_version | installed_version | location | comment +-------------+-----------------+-------------------+----------+---------------------------- + pg_readonly | 1.0.6 | 1.0.6 | $system | cluster database read only +(1 row) +---- + +== Usage + +=== Check Individual Functions + +[literal] +---- +-- Query the current cluster read-only status +select get_cluster_readonly(); + get_cluster_readonly +---------------------- + f +(1 row) + +-- Set the current cluster to read-only mode +select set_cluster_readonly(); + set_cluster_readonly +---------------------- + t +(1 row) + +-- Read-only cluster only allows SELECT statements +select * from t; + x | y +---+--- +(0 rows) + +update t set x=33 where y='abc'; +ERROR: cannot execute UPDATE in a read-only transaction + +select 1 into tmp; +ERROR: cannot execute UPDATE in a read-only transaction + +create table tmp(c text); +ERROR: cannot execute UPDATE in a read-only transaction +---- + +For more detailed usage and advanced features, please refer to the https://github.com/pierreforstmann/pg_readonly . + diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_repack.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_repack.adoc new file mode 100644 index 0000000..9a60328 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_repack.adoc @@ -0,0 +1,102 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_repack + +== Overview +pg_repack is an extension for PostgreSQL used to rebuild tables and indexes online and eliminate storage bloat. It can achieve effects similar to `VACUUM FULL` (reclaiming space), while barely blocking business read/write operations. + +Both the PG mode and Oracle compatibility mode of IvorySQL have been adapted for pg_repack. + +Github: + +Documentation: + +License: BSD-3-Clause License + +== Principle +`VACUUM FULL` lacks a mechanism for capturing concurrent changes during the rewrite. It holds an `ACCESS EXCLUSIVE` lock on the entire table for the whole duration, during which the table cannot be read or written, and the lock duration grows with the table size. pg_repack compresses the strong lock into two brief moments at the beginning and end by means of "shadow table + triggers + file node swap", thereby enabling online rebuild. + +=== Change Capture and Data Copy +* First, install row-level triggers on the original table to record subsequent `INSERT` / `UPDATE` / `DELETE` operations into a log table (`repack.log_`) — this step requires a brief `ACCESS EXCLUSIVE` lock to atomically install the triggers, which is released immediately after installation; + +* Create a shadow table with the same structure as the original table (`repack.table_`), and bulk-copy the original table's data under a consistent MVCC snapshot; this phase holds only `SHARE UPDATE EXCLUSIVE`, so the original table continues to allow normal reads and writes; + +* Rebuild indexes on the shadow table, then replay the log so that the shadow table catches up with the latest state of the original table. + +=== Swap and Cleanup +* Acquire a brief `ACCESS EXCLUSIVE` lock, replay any remaining log entries, then swap the physical file nodes (`relfilenode`) of the original table and the shadow table at the system catalog level — the table's OID remains unchanged, and the swap is instantaneous; + +* Drop the triggers and log tables, then drop the shadow table (which now points to the old heap, so dropping it reclaims the old physical files); finally, run `ANALYZE` on the table. + +[NOTE] +==== +Prerequisites for use: the target table must have a primary key or a non-null unique key; the operation requires approximately twice the table size of temporary disk space during the process. +==== + +== Installation and Enabling +IvorySQL 5 or above is already installed in the environment. pg_repack consists of two parts: the server-side extension (`pg_repack.so` and SQL scripts) and the client-side command-line tool `pg_repack`. + +It is assumed that IvorySQL 5 is currently installed and the `pg_config` tool is available. + +=== Source Code Installation +Pull the pg_repack source code: +[literal] +---- +git clone --branch ver_1.5.3 https://github.com/reorg/pg_repack.git +---- + +Perform compilation and installation: +[literal] +---- +cd pg_repack +make +make install +---- + +=== Create the Extension +[literal] +---- +[ivorysql@localhost ivorysql]$ psql +psql (18.0) +Type "help" for help. + +ivorysql=# CREATE EXTENSION pg_repack; +CREATE EXTENSION +---- + +[NOTE] +==== +`CREATE EXTENSION pg_repack` must be executed by a superuser; the client tool `pg_repack` also requires connecting and running as a superuser by default. +==== + +== Usage +pg_repack is driven through the command-line client. The connection parameters are the same as those of psql (`-h` host, `-p` port, `-U` user, `-d` database, or use the `PGHOST` / `PGPORT` / `PGUSER` environment variables). + +=== Rebuild All Eligible Tables in the Entire Database +[literal] +---- +pg_repack -d ivorysql +---- + +=== Rebuild a Specified Table +The schema.table form is recommended: +[literal] +---- +pg_repack -d ivorysql -t public.big_table +---- + +=== Rebuild Indexes Only +[literal] +---- +pg_repack -d ivorysql -t big_table --only-indexes # rebuild all indexes of this table +pg_repack -d ivorysql -i public.big_table_idx # rebuild a single index +---- + +=== Dry-Run Mode +Only shows the actions that would be performed, without actually rebuilding: +[literal] +---- +pg_repack -d ivorysql -t big_table --dry-run +---- diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_show_plans.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_show_plans.adoc new file mode 100644 index 0000000..3c1e56a --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_show_plans.adoc @@ -0,0 +1,111 @@ +:sectnums: +:sectnumlevels: 5 + += pg_show_plans + +== Overview + +pg_show_plans is a PostgreSQL extension that shows the query plans of all currently running SQL statements. Query plans can be displayed in several formats, including `TEXT`, `JSON`, and `YAML`. + +NOTE: This extension creates a hash table within shared memory. The hash table is not resizable — no new plans can be added once it has been filled up. + +== Installation + +[TIP] +The source installation environment is Ubuntu 24.04 (x86_64), with IvorySQL 5 or above installed at `/usr/ivory-5`. + +=== Install from Source + +Make sure `pg_config` is available on your PATH. + +[source,bash] +---- +git clone https://github.com/cybertec-postgresql/pg_show_plans.git +cd pg_show_plans + +make PG_CONFIG=/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config install +---- + +=== Configure shared_preload_libraries + +Add `pg_show_plans` to the `shared_preload_libraries` parameter in `ivorysql.conf`: + +[source,conf] +---- +shared_preload_libraries = 'liboracle_parser, ivorysql_ora, pg_show_plans' +---- + +Restart the IvorySQL instance to apply the change: + +[source,bash] +---- +pg_ctl restart -D data +---- + +=== Create the Extension + +After the server has restarted, connect to the target database and create the extension: + +[source,sql] +---- +CREATE EXTENSION pg_show_plans; +---- + +== Usage + +=== View Current Query Plans + +Use the `pg_show_plans` view to inspect the execution plans of all currently running SQL statements: + +[source,sql] +---- +SELECT * FROM pg_show_plans; +---- + +[source,text] +---- + pid | level | userid | dbid | plan +-------+-------+--------+-------+----------------------------------------------------------------------- + 11473 | 0 | 10 | 16384 | Function Scan on pg_show_plans (cost=0.00..10.00 rows=1000 width=56) + 11504 | 0 | 10 | 16384 | Function Scan on print_item (cost=0.25..10.25 rows=1000 width=524) + 11504 | 1 | 10 | 16384 | Result (cost=0.00..0.01 rows=1 width=4) +(3 rows) +---- + +=== View Query Plans with SQL Text + +Use the `pg_show_plans_q` view to see query plans together with the corresponding SQL statement. This view joins `pg_show_plans` with `pg_stat_activity`: + +[source,sql] +---- +SELECT * FROM pg_show_plans_q; +---- + +[source,text] +---- +-[ RECORD 1 ]------------------------------------------------------------------------------------ +pid | 11473 +level | 0 +plan | Sort (cost=72.08..74.58 rows=1000 width=80) + | Sort Key: pg_show_plans.pid, pg_show_plans.level + | -> Hash Left Join (cost=2.25..22.25 rows=1000 width=80) + | Hash Cond: (pg_show_plans.pid = s.pid) + | Join Filter: (pg_show_plans.level = 0) + | -> Function Scan on pg_show_plans (cost=0.00..10.00 rows=1000 width=48) + | -> Hash (cost=1.00..1.00 rows=100 width=44) + | -> Function Scan on pg_stat_get_activity s (cost=0.00..1.00 rows=100 width=44) +query | SELECT p.pid, p.level, p.plan, a.query FROM pg_show_plans p + | LEFT JOIN pg_stat_activity a + | ON p.pid = a.pid AND p.level = 0 ORDER BY p.pid, p.level; +-[ RECORD 2 ]------------------------------------------------------------------------------------ +pid | 11517 +level | 0 +plan | Function Scan on print_item (cost=0.25..10.25 rows=1000 width=524) +query | SELECT * FROM print_item(1,20); +-[ RECORD 3 ]------------------------------------------------------------------------------------ +pid | 11517 +level | 1 +plan | Result (cost=0.00..0.01 rows=1 width=4) +query | +---- diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_stat_monitor.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_stat_monitor.adoc new file mode 100644 index 0000000..1b7722c --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_stat_monitor.adoc @@ -0,0 +1,98 @@ +:sectnums: +:sectnumlevels: 5 + += pg_stat_monitor + +== Overview + +pg_stat_monitor is a Query Performance Monitoring tool for PostgreSQL. pg_stat_monitor collects performance statistics and provides query performance insights in a single view and graphically in histogram. + +These insights allow database users to understand query origins, execution, planning statistics and details, query information, and metadata. This significantly improves observability, enabling users to debug and tune query performance. + +== Installation + +The IvorySQL installation package already includes the pg_stat_monitor plugin. If you installed IvorySQL using the installation package, you typically don't need to manually install pg_stat_monitor. If you installed IvorySQL from source code, you can proceed to install the pg_stat_monitor plugin also from source. The IvorySQL community provides step-by-step instructions for source code installation： + +To build pg_stat_monitor from source code, you require the following: + +* git +* make +* gcc +* pg_config + +You can download the source code of the latest release of pg_stat_monitor from https://github.com/Percona/pg_stat_monitor/releases[the releases page on GitHub] or using git: + +[source,bash] +---- +git clone https://github.com/percona/pg_stat_monitor.git +---- + +Compile and install the extension. Assuming IvorySQL is already installed in the environment with the installation path at /usr/ivory-5. + +[source,bash] +---- +cd pg_stat_monitor +export PG_CONFIG=/usr/ivory-5/bin/pg_config +make USE_PGXS=1 +make USE_PGXS=1 install +---- + +== Load the module + +Load pg_stat_monitor at the start time by adding it to the shared_preload_libraries configuration parameter. This is because pg_stat_monitor requires additional shared memory. + +modify the shared_preload_libraries parameter： + +[source,conf] +---- +shared_preload_libraries = 'pg_stat_monitor'; +---- + +NOTE: If you’ve added other modules to the shared_preload_libraries parameter (for example, pg_stat_statements), list all of them separated by commas for the ALTER SYSTEM command. + +[source,conf] +---- +shared_preload_libraries = 'liboracle_parser, ivorysql_ora, pg_stat_statements, pg_stat_monitor'; +---- + +Start or restart the IvorySQL instance to apply the changes. + +[source,bash] +---- +pg_ctl restart -D data +---- + +After you have added pg_stat_monitor to the shared_preload_libraries, it starts collecting statistics data for all existing databases. To access this data, you need to create the view on every database that you wish to monitor. + +== Create the extension view + +Create the extension view with the user that has the privileges of a superuser or a database owner. Connect to psql as a superuser for a database and run the CREATE EXTENSION command: + +[source,sql] +---- +CREATE EXTENSION pg_stat_monitor; +---- + +After the setup is complete, you can see the stats collected by pg_stat_monitor. + +== Usage + +To obtain query execution time information as an example, connect to the database and execute the following SQL： + +[source,sql] +---- +SELECT userid, total_exec_time, min_exec_time, max_exec_time, mean_exec_time, query FROM pg_stat_monitor; +---- + +[source,text] +---- +userid | total_exec_time | min_exec_time | max_exec_time | mean_exec_time | query +--------+-----------------+---------------+---------------+----------------+---------------------------------------------------------------------------------------------- + 10 | 1.532168 | 0.749108 | 0.78306 | 0.766084 | SELECT userid, datname, queryid, substr(query,$1, $2) AS query, calls FROM pg_stat_monitor + 10 | 0.755857 | 0.755857 | 0.755857 | 0.755857 | SELECT application_name, client_ip, substr(query,$1,$2) as query FROM pg_stat_monitor + 10 | 0 | 0 | 0 | 0 | SELECT userid, total_time, min_time, max_time, mean_time, query FROM pg_stat_monitor; + 10 | 0 | 0 | 0 | 0 | SELECT userid, total_exec_time, min_time, max_time, mean_time, query FROM pg_stat_monitor; +(4 rows) +---- + +For more information on using pg_stat_monitor, please refer to pg_stat_monitor https://docs.percona.com/pg-stat-monitor/user_guide.html[ Official documentation] \ No newline at end of file diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_textsearch.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_textsearch.adoc new file mode 100644 index 0000000..40c880a --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_textsearch.adoc @@ -0,0 +1,88 @@ + +:sectnums: +:sectnumlevels: 5 + += pg_textsearch + +== Overview +pg_textsearch is a PostgreSQL extension developed by the Timescale team. It is designed to provide high-performance, modern full-text search capabilities for Postgres, and is optimized for AI workloads and hybrid search. + +== Installation + +[TIP] +The source code installation environment is Ubuntu 24.04 (x86_64), in which IvorySQL 5 or a later version has been installed. The installation path is /usr/ivory-5. + +=== Source Code Installation + +[literal] +---- +# download source code package from: https://github.com/timescale/pg_textsearch/archive/refs/tags/v0.6.1.tar.gz + +tar xzvf v0.6.1.tar.gz +cd pg_textsearch-0.6.1 + +# compile and install the extension +make PG_CONFIG=/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config install +---- + +[TIP] +If there is error "xlocale.h: No such file or directory" during compilation, user should remove the +line of "#define HAVE_XLOCALE_H 1" from file /usr/ivory-5/include/postgresql/server/pg_config.h. + +=== Modify the configuration file + +Modify the ivorysql.conf file to add pg_textsearch into shared_preload_libraries. +[literal] +---- +shared_preload_libraries = 'gb18030_2022, liboracle_parser, ivorysql_ora, pg_textsearch' +---- + +Then restart the database. + +=== Create Extension + +[literal] +---- +postgres=# create extension pg_textsearch; +WARNING: pg_textsearch v0.6.1 is a prerelease. Do not use in production. +CREATE EXTENSION +---- + +== Use + +Create a table with text content: +[literal] +---- +postgres=# CREATE TABLE documents (id bigserial PRIMARY KEY, content text); +CREATE TABLE + +postgres=# INSERT INTO documents (content) VALUES + ('PostgreSQL is a powerful database system'), + ('BM25 is an effective ranking function'), + ('Full text search with custom scoring'); +INSERT 0 3 +---- + +Create a pg_textsearch index on the text column: +[literal] +---- +postgres=# CREATE INDEX docs_idx ON documents USING bm25(content) WITH (text_config='english'); +NOTICE: BM25 index build started for relation docs_idx +NOTICE: Using text search configuration: english +NOTICE: Using index options: k1=1.20, b=0.75 +NOTICE: BM25 index build completed: 3 documents, avg_length=4.33 +CREATE INDEX +---- + +Get the most relevant documents using the <@> operator: +[literal] +---- +postgres=# SELECT * FROM documents ORDER BY content <@> 'database system' LIMIT 5; + id | content +----+------------------------------------------ + 1 | PostgreSQL is a powerful database system + 2 | BM25 is an effective ranking function + 3 | Full text search with custom scoring +(3 rows) +---- diff --git a/EN/modules/ROOT/pages/master/5.8.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pgaudit.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/5.8.adoc rename to EN/modules/ROOT/pages/master/ecosystem_components/pgaudit.adoc diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pgbackrest.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pgbackrest.adoc new file mode 100644 index 0000000..39a9807 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pgbackrest.adoc @@ -0,0 +1,499 @@ +:sectnums: +:sectnumlevels: 5 + += pgBackRest + +== Overview + +pgBackRest is a reliable backup and restore solution for PostgreSQL. At its core, it is built on PostgreSQL physical-level backups (files in the data directory) and the WAL archive management mechanism. The project is notable for its high quality: 100% test coverage, high-quality code, and minimal dependencies. + +NOTE: pgBackRest introduced a CRC check of the pg_control file in v2.48.0. Since IvorySQL modifies the pg_control file layout, the pg-version-force option must be used in the stanza configuration to specify the PostgreSQL version corresponding to IvorySQL. + +NOTE: pgBackRest works with IvorySQL 5.x without modification, but this is merely a coincidence. In practice, it is still recommended to add the pg-version-force option. + +Project URL: + +Version: v2.58.0 + +License: MIT License + +== Installation + +[TIP] +The official website recommends installing from a package manager. + +[TIP] +The source build was tested on Ubuntu 26.04. + +=== Dependencies + +[source,bash] +---- +sudo apt update && \ +sudo apt-get install python3-setuptools meson gcc \ + libpq-dev libssl-dev libxml2-dev \ + pkg-config liblz4-dev libzstd-dev \ + libbz2-dev libz-dev libyaml-dev \ + libssh2-1-dev +---- + +=== Building from Source + +[source,bash] +---- +cd ~ +mkdir -p build +wget -q -O - \ + https://github.com/pgbackrest/pgbackrest/archive/release/2.58.0.tar.gz | \ + tar zx -C build +meson setup build/pgbackrest build/pgbackrest-release-2.58.0 +ninja -C build/pgbackrest + +# After compilation, the pgbackrest executable will be generated under build/src +./build/src/pgbackrest --version +---- + +=== Verifying the Installation + +NOTE: pgBackRest itself is a single executable; simply pass in a configuration file when using it. This document is for usage instructions only — no additional installation steps are required. + +== Configuration + +NOTE: pgBackRest supports multiple backup repositories. For details, see: + +NOTE: In the demonstration in this document, the PG database and the backup repository are on the same host, and the backup repository is a local file system. + +Configuration is required in two places: + +[cols="1,2"] +|=== +| File name | Description + +| postgresql.conf +| Configures `PostgreSQL`: add `archive_command` to enable `WAL` backup + +| pgbackrest.conf +| Configuration for `pgBackRest` itself, including the source database, backup repository, etc. +|=== + +Append to the end of `postgresql.conf`: + +NOTE: This is an example only; for actual steps, refer to the later sections of this document. + +[source,conf] +---- +port = 6551 +ivorysql.port = 6553 +listen_addresses = '127.0.0.1' +unix_socket_directories = '/tmp/.pgbr_sock_1000_6551' +archive_mode = on +archive_command = '/path/to/pgbackrest --config=/path/to/pgbackrest.conf --stanza=ivory archive-push %p' +log_min_messages = info +---- + +Create `pgbackrest.conf`: + +NOTE: This is an example only; for actual steps, refer to the later sections of this document. + +[source,conf] +---- +[global] +# With multiple repos configured, WAL is fully redundant and backups must be run manually +repo1-path=/tmp/pgbackrest_ivorysql/repo +# Retention period for full backups +repo1-retention-full=9999 +# Log files +log-path=/tmp/pgbackrest_ivorysql/log +# Console log level +log-level-console=info +# Log level for the log file +log-level-file=detail +# Do not wait for the next checkpoint +start-fast=y + +[$STANZA] +# Configuring multiple pg entries is for primary+replica setups; the numbering does not indicate primary/standby roles, which are detected automatically +pg1-path=/tmp/pgbackrest_ivorysql/pg1/data +pg1-port=6551 +pg1-socket-path='/tmp/.pgbr_sock_1000_6551' +pg-version-force=18 +---- + +== Usage + +=== pgBackRest Commands + +[source,bash] +---- +# Usage +pgbackrest [options] [command] +---- + +List of `pgBackRest` commands: + +[source,bash] +---- +# Add or modify a backup annotation; annotations are recorded in JSON form in the backup.info file +annotate + +# Fetch a WAL segment from the backup repository; invoked by PostgreSQL via restore_command. +# When running pgbackrest restore, it is automatically written into postgresql.auto.conf: +# "restore_command = 'pgbackrest --stanza=ivory archive-get %f "%p"'" +archive-get + +# Push a WAL segment into the backup repository — pushes the WAL segment PostgreSQL has just finished writing to the pgbackrest repository. +# Together with archive-get (one in, one out), it forms the foundation of PITR. +# Must be manually written into postgresql.conf: "archive_command = 'pgbackrest --stanza=ivory archive-push %p'" +archive-push + +# Back up the PG database: take a consistent backup of the entire data directory and write it to the repository. +backup + +# Check the configuration, verify repository information, and test the archiving pipeline (executes pg_switch_wal) +check + +# Delete expired backups and WAL archives that are no longer needed +expire + +# Display help information +help + +# Get information about backups +info + +# For debugging: fetch any file from the repository and write it to stdout or a local file. +repo-get + +# List files and directories in the repository via the storage driver +repo-ls + +# Restore a backup from the repository into a startable data directory, and automatically configure subsequent WAL replay. +restore + +# Run the transport service for multi-host deployments (TLS mode): moves backup data between hosts. Must run on both the PG and Repo hosts. +server + +# pgBackRest server liveness-check command +server-ping + +# Initialization command: create the stanza metadata skeleton for a cluster in the repository +stanza-create + +# Delete a stanza +stanza-delete + +# After a major database version upgrade, the stanza must be upgraded accordingly +stanza-upgrade + +# Does not start any process/service; its only job is to delete the stop file left by the stop command, allowing pgbackrest operations to resume. +start + +# Create a stop file; backup, archive-push, expire, stanza-create/upgrade, and remote requests will refuse to run when they detect this file at startup. +stop + +# Check whether existing backups and WAL are intact +verify + +# Get the current version +version +---- + +List of `pgBackRest` internal commands. These commands should not be used in production scenarios, as they may cause data corruption: + +[source,bash] +---- +# Dump the manifest of a backup in readable format +manifest + +# Write a file into the repository (encrypted automatically); the reverse of repo-get +repo-put + +# Delete repository files/directories (--recurse) +repo-rm +---- + +=== Performing Operations + +TIP: Since the process is fairly complex, the commands have been parameterized and helper functions have been created to make testing easier. Please run the commands below within the same terminal session. + +Variable preparation: + +[source,bash] +---- +# Adjust as appropriate for your environment: IVORY_PG_VERSION_MAJOR, IVORY_BIN, PGBACKREST_BIN; the rest can be left at defaults +IVORY_PG_VERSION_MAJOR=18 +IVORY_BIN="/usr/local/ivorysql/bin" +PGBACKREST_BIN="/home/lct/repos/pgbackrest/build/src/pgbackrest" +TEST_BASE="/tmp/pgbackrest_ivorysql_regress" +PG_PORT="6551" +STANDBY_PORT="6552" +ORA_PORT="6553" +STANDBY_ORA_PORT="6554" +DB_MODE="oracle" +STANZA="ivory" + +PG1_DATA="$TEST_BASE/pg1/data" +PG2_DATA="$TEST_BASE/pg2/data" +REPO_PATH="$TEST_BASE/repo" +# Unix socket paths have a 107-byte limit, so use a fixed short directory under /tmp +SOCK_PATH="${SOCK_PATH:-/tmp/.pgbr_sock_$(id -u)_$PG_PORT}" +LOG_PATH="$TEST_BASE/log" +CONF="$TEST_BASE/pgbackrest.conf" +RUN_LOG="$TEST_BASE/regression.log" +TEST_DB="regress" +RESTORE_POINT="pgb_regress_rp" + +sql() +{ + local port="$1" db="$2" query="$3" + "$IVORY_BIN/psql" -h "$SOCK_PATH" -p "$port" -d "$db" -U "$(id -un)" \ + -v ON_ERROR_STOP=1 -Atc "$query" 2>>"$RUN_LOG" +} +switch_wal_and_wait() +{ + local wal i + # First generate a WAL record, to avoid pg_switch_wal() being a no-op (which would not trigger archiving) when the current segment is empty + sql "$ORA_PORT" postgres "SELECT pg_create_restore_point('pgb_wal_sync')" >/dev/null || return 1 + wal=$(sql "$ORA_PORT" postgres "SELECT pg_walfile_name(pg_switch_wal())") || return 1 + for i in $(seq 1 60); do + local archived + archived=$(sql "$ORA_PORT" postgres "SELECT last_archived_wal FROM pg_stat_archiver") + [[ -n "$archived" && ! "$archived" < "$wal" ]] && return 0 + sleep 1 + done + return 1 +} +---- + +Preparation before execution: + +[source,bash] +---- +rm -rf "$SOCK_PATH" +mkdir -p "$PG1_DATA" "$REPO_PATH" "$SOCK_PATH" "$LOG_PATH" +chmod 700 "$PG1_DATA" "$SOCK_PATH" +: >"$RUN_LOG" +export LD_LIBRARY_PATH="$(dirname "$IVORY_BIN")/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + +# Confirm the versions +${IVORY_BIN}/pg_ctl --version +${PGBACKREST_BIN} --version +---- + +Initialize the primary database and configuration: + +[source,bash] +---- +# Initialize, using oracle mode +"$IVORY_BIN/initdb" -D "$PG1_DATA" -m "$DB_MODE" -E UTF8 --no-locale -A trust -U "$(id -un)" + +# Configure IvorySQL +cat >>"$PG1_DATA/postgresql.conf" <"$CONF" < 100;"; + +# Perform the restore +"$IVORY_BIN/pg_ctl" -D "$PG1_DATA" -w -t 60 -m fast stop +"$PGBACKREST_BIN" --config="$CONF" --stanza="$STANZA" \ + --delta --type=name --target="$RESTORE_POINT" \ + --target-action=promote restore; + +# Verify; this md5 output should match the one above +"$IVORY_BIN/pg_ctl" -D "$PG1_DATA" -l "$LOG_PATH/pg1.log" -w start +sql "$ORA_PORT" "$TEST_DB" \ + "SELECT md5(string_agg(id::text || ':' || name, ',' ORDER BY id)) FROM t_ora;"; +---- diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pgbouncer.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pgbouncer.adoc new file mode 100644 index 0000000..4a923bc --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pgbouncer.adoc @@ -0,0 +1,221 @@ +:sectnums: +:sectnumlevels: 5 + += PgBouncer + +== Overview + +PgBouncer is a lightweight connection pooling middleware that sits between the application layer and the database. It reduces connection overhead, protects database resources, and improves application concurrency performance by reusing backend connections. + +PgBouncer uses the standard PostgreSQL communication protocol, which IvorySQL fully supports. + +*Three Connection Pooling Modes:* + +[cols="1,2,2"] +|=== +| Mode | Description | Use Case + +| session +| Client exclusively holds one backend connection for the duration of the session +| Required for in-process COMMIT and full session features + +| transaction +| Connection is returned to the pool after each transaction +| Most commonly used, highest connection reuse rate + +| statement +| Connection is returned after each statement +| Most restrictive, does not support explicit transactions +|=== + +== Installation + +[TIP] +Source installation was tested on Ubuntu 24.04. + +=== Dependencies + +[literal] +---- +# Ubuntu / Debian +sudo apt install libevent-dev libssl-dev pkg-config + +# RHEL / Rocky Linux +sudo dnf install libevent-devel openssl-devel pkgconfig +---- + +=== Source Installation + +[literal] +---- +wget https://github.com/pgbouncer/pgbouncer/archive/refs/tags/pgbouncer_1_25_1.zip +unzip pgbouncer_1_25_1.zip +cd pgbouncer_1_25_1 + +./autogen.sh + +./configure \ + --prefix=/usr/ivory-5 \ + --with-openssl \ + --with-pam + +make -j4 +make install +cp pgbouncer /usr/ivory-5/bin/ +---- + +=== Verify Installation + +[literal] +---- +pgbouncer --version +# PgBouncer 1.25.1 +# libevent 2.1.12-stable +# tls: OpenSSL 3.0.2 15 Mar 2022 +---- + +== Configuration + +=== Connecting to IvorySQL PG Mode (Port 5432) + +Create `/etc/pgbouncer/pgbouncer.ini`: + +[literal] +---- +[databases] +postgres = host=127.0.0.1 port=5432 dbname=postgres + +[pgbouncer] +listen_addr = 127.0.0.1 +listen_port = 6432 +auth_type = trust +auth_file = /etc/pgbouncer/userlist.txt +pool_mode = transaction +max_client_conn = 200 +default_pool_size = 20 +logfile = /var/log/pgbouncer/pgbouncer.log +pidfile = /var/run/pgbouncer/pgbouncer.pid +---- + +=== Connecting to IvorySQL Oracle Compatible Mode (Port 1521) + +[literal] +---- +[databases] +postgres = host=127.0.0.1 port=1521 dbname=postgres + +[pgbouncer] +listen_addr = 127.0.0.1 +listen_port = 2521 +auth_type = trust +auth_file = /etc/pgbouncer/userlist.txt + +# Oracle compatible mode recommends session mode +pool_mode = session + +max_client_conn = 200 +default_pool_size = 20 +logfile = /var/log/pgbouncer/pgbouncer_oracle.log +pidfile = /var/run/pgbouncer/pgbouncer_oracle.pid +---- + +=== User Authentication File + +[literal] +---- +# /etc/pgbouncer/userlist.txt +# Format: "username" "password" (trust mode can leave password empty) +"postgres" "" +"app_user" "app_password" +---- + +=== Start and Stop + +[literal] +---- +# Foreground mode (for debugging) +pgbouncer /etc/pgbouncer/pgbouncer.ini + +# Daemon mode +pgbouncer -d /etc/pgbouncer/pgbouncer.ini + +# Reload configuration (without interrupting connections) +kill -HUP $(cat /var/run/pgbouncer/pgbouncer.pid) + +# Stop +kill -INT $(cat /var/run/pgbouncer/pgbouncer.pid) +---- + +== Usage + +=== Client Connection + +Connecting through PgBouncer uses the exact same syntax as connecting directly to IvorySQL, only the port changes: + +[literal] +---- +# PG mode (via PgBouncer) +psql -U postgres -p 6432 -d postgres + +# Oracle compatible mode (via PgBouncer) +psql -U postgres -p 2521 -d postgres +---- + +=== Admin Console + +PgBouncer provides a built-in administration database: + +[literal] +---- +psql -U postgres -p 6432 -d pgbouncer +---- + +[literal] +---- +-- View pool status +SHOW POOLS; + +-- View statistics +SHOW STATS; + +-- View client connections +SHOW CLIENTS; + +-- View backend connections +SHOW SERVERS; + +-- Reload configuration +RELOAD; +---- + +=== Oracle Compatible Mode + +[literal] +---- +-- Verify Oracle mode is active +SHOW ivorysql.compatible_mode; +-- ivorysql.compatible_mode +-- -------------------------- +-- oracle + +-- Oracle data types and functions +CREATE TABLE bouncer_test ( + id NUMBER(10) PRIMARY KEY, + name VARCHAR2(100), + hired DATE DEFAULT SYSDATE +); + +INSERT INTO bouncer_test VALUES (1, 'Alice', SYSDATE); + +SELECT id, + NVL(name, 'N/A') AS name, + DECODE(id, 1, 'CEO', 'Staff') AS title, + TO_CHAR(hired, 'YYYY-MM-DD') AS hire_date +FROM bouncer_test; + +-- Oracle sequences +CREATE SEQUENCE ora_seq START WITH 100 INCREMENT BY 10; +SELECT ora_seq.NEXTVAL FROM DUAL; -- 100 +SELECT ora_seq.NEXTVAL FROM DUAL; -- 110 +SELECT ora_seq.CURRVAL FROM DUAL; -- 110 +---- \ No newline at end of file diff --git a/EN/modules/ROOT/pages/master/5.3.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pgddl.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/5.3.adoc rename to EN/modules/ROOT/pages/master/ecosystem_components/pgddl.adoc diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pgdog.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pgdog.adoc new file mode 100644 index 0000000..fd944fe --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pgdog.adoc @@ -0,0 +1,278 @@ +:sectnums: +:sectnumlevels: 5 + += PgDog + +== Overview +PgDog is a high-performance, open-source clustering middleware (proxy tool) designed specifically for PostgreSQL and written in Rust. It integrates automatic sharding, connection pooling, and load balancing, enabling developers to achieve horizontal scaling and high-availability management of PostgreSQL databases without modifying any application code. + +NOTE: PgDog uses PostgreSQL's native pg_query module to parse statements, so it currently does not support Oracle compatibility mode. + +Project URL: + +Version: v0.1.45 + +Open-source license: AGPL-3.0 License + +== Installation + +[TIP] +The source build was tested on Ubuntu 26.04. + +=== Dependencies + +[source,bash] +---- +sudo apt update && \ +sudo apt install -y cmake clang curl pkg-config \ + libssl-dev git build-essential mold rustup \ + docker +---- + +[TIP] +The instructions in this document require two IvorySQL database instances, which can be quickly set up using the docker-compose file provided in this document. To install docker-compose, refer to: + +=== Building from Source + +[source,bash] +---- +wget https://github.com/pgdogdev/pgdog/archive/refs/tags/v0.1.45.tar.gz +tar -zxf v0.1.45.tar.gz +cd pgdog-0.1.45 + +# After compilation, the executable will be generated in the target/release directory +cargo build --release +---- + +=== Verifying the Installation + +[source,bash] +---- +# At the time this document was written, PgDog is under rapid iterative development, so the output of the command below is incomplete +./target/release/pgdog --version +# Output: PgDog v +---- + +== Configuration + +This document configures two shards and uses PgDog's automatic sharding feature as an example. + +PgDog is configured through two files: + +[cols="1,2"] +|=== +| File Name | Description + +| pgdog.toml +| Contains basic configuration information such as PgDog's port settings and the backend PostgreSQL service configuration + +| users.toml +| The username and password for accessing PgDog are configured here +|=== + + +Create `pgdog.toml`: + +[source,toml] +---- +[general] +host = "0.0.0.0" +port = 6432 +default_pool_size = 10 + +# ---- ivory_shard: shard across two IvorySQL backends ---- +# host, port, and database_name need to be modified according to your actual setup if you are not using the environment built with the docker-compose provided in this document +[[databases]] +name = "ivory_shard" +host = "ivory-shard0" +port = 5432 +database_name = "testdb" +user = "ivorysql" # if not provide, using name in users.toml +password = "ivorysql" # if not provide, using password in users.toml +shard = 0 + +[[databases]] +name = "ivory_shard" +host = "ivory-shard1" +port = 5432 +database_name = "testdb" +shard = 1 + +# ---- Shard key declaration ---- +# The configuration must match the actual table structure +[[sharded_tables]] +database = "ivory_shard" +name = "orders" +column = "customer_id" +data_type = "bigint" +---- + +Create `users.toml`: + +[source,toml] +---- +[admin] +name = "admin" +user = "admin" +password = "pgdog" + +[[users]] +name = "ivorysql" +password = "ivorysql" +database = "ivory_shard" +pool_size = 10 +---- + +Create `docker-compose.shard.yml`: + +[TIP] +Please modify the `volumes` field according to the actual location of your configuration files. + +[source,dockerfile] +---- +# Sharding test topology (standalone compose): 2 shard backends. +# ivory-shard0 IvorySQL 5.4 (pg) host:5443 +# ivory-shard1 IvorySQL 5.4 (pg) host:5444 + +x-ivory: &ivory + image: registry.highgo.com/ivorysql/ivorysql:5.4-bookworm + environment: &ivoryenv + MODE: pg + IVORYSQL_USER: ivorysql + IVORYSQL_PASSWORD: ivorysql + IVORYSQL_DB: testdb + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ivorysql -d testdb"] + interval: 5s + timeout: 3s + retries: 30 + +services: + ivory-shard0: + <<: *ivory + container_name: ivory-shard0 + ports: ["5443:5432"] + ivory-shard1: + <<: *ivory + container_name: ivory-shard1 + ports: ["5444:5432"] +---- + +== Usage + +=== Starting PgDog + +[source,bash] +---- +./target/release/pgdog --config ./pgdog.toml --users ./users.toml +---- + + +=== Administration Console + +PgDog provides a built-in administration database. The username and password are configured via the `+[[admin]]+` field in `users.toml`. + +[source,bash] +---- +psql "postgres://admin:pgdog@localhost:6433/admin" +---- + +[source,sql] +---- +-- View client connections and real-time statistics +SHOW CLIENTS + +-- View PostgreSQL connections initiated by PgDog +SHOW SERVERS + +-- View connection pool information +SHOW POOLS + +-- View the configuration currently loaded from pgdog.toml +SHOW CONFIG + +-- View connection pool statistics +SHOW STATS + +-- List of PgDog processes running on the same network. Requires service discovery to be enabled +SHOW PEERS + +-- Reload the configuration from disk. For which options can be changed at runtime, refer to pgdog.toml and users.toml +RELOAD + +-- Recreate all server connections using the existing configuration +RECONNECT + +-- Pause all connection pools. Clients will wait for a connection until the pools resume. Useful for performing a graceful restart of the PostgreSQL server +PAUSE + +-- Resume all connection pools. Clients can acquire connections again +RESUME + +-- List the prepared statements currently in the cache +SHOW PREPARED + +-- List the statements currently in the AST cache used for query routing +SHOW QUERY_CACHE + +-- Pause all queries in order to synchronize configuration changes across multiple PgDog instances +MAINTENANCE + +-- Show the PostgreSQL replication status for each database, including replica lag +SHOW REPLICATION +---- + +=== Connecting to PgDog + +[source,bash] +---- +psql "postgres://ivorysql:ivorysql@localhost:6433/ivory_shard" +---- + +=== Performing Operations + +[TIP] +Make sure the shard backends do not have an `orders` table; PgDog will create it automatically. + +[source,sql] +---- +-- Create the table +CREATE TABLE orders ( + order_id bigint, + customer_id bigint, + amount numeric(10,2), + PRIMARY KEY (order_id, customer_id) +); + +-- Insert data: these rows will be inserted into the two shard backends respectively +-- BUG: the generate_series function cannot be used here, because PgDog currently passes this function through transparently +INSERT INTO orders values(1, 1, 1); +INSERT INTO orders values(2, 2, 2); +INSERT INTO orders values(3, 3, 3); +INSERT INTO orders values(4, 4, 4); +INSERT INTO orders values(5, 5, 5); +INSERT INTO orders values(6, 6, 6); +INSERT INTO orders values(7, 7, 7); + +-- Query the data +SELECT * FROM orders; +---- + +=== Connecting to Each Shard Backend to Verify the Data + +[TIP] +Here you will see that the entries in the two shard backends are not evenly distributed. This is because PgDog extracts the value of the `customer_id` field configured in `+[[sharded_tables]]+` and applies HASH-based sharding to it. + +[source,bash] +---- +# Shard 1 +psql "postgres://ivorysql:ivorysql@localhost:5443/testdb", +# Shard 2 +psql "postgres://ivorysql:ivorysql@localhost:5444/testdb", +---- + +Run the query on each shard backend to verify the data +[source,sql] +---- +SELECT * FROM orders; +---- diff --git a/EN/modules/ROOT/pages/master/5.7.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pgroonga.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/5.7.adoc rename to EN/modules/ROOT/pages/master/ecosystem_components/pgroonga.adoc diff --git a/EN/modules/ROOT/pages/master/5.9.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pgrouting.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/5.9.adoc rename to EN/modules/ROOT/pages/master/ecosystem_components/pgrouting.adoc diff --git a/EN/modules/ROOT/pages/master/5.5.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pgsql_http.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/5.5.adoc rename to EN/modules/ROOT/pages/master/ecosystem_components/pgsql_http.adoc diff --git a/EN/modules/ROOT/pages/master/5.2.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pgvector.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/5.2.adoc rename to EN/modules/ROOT/pages/master/ecosystem_components/pgvector.adoc diff --git a/EN/modules/ROOT/pages/master/5.6.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/plpgsql_check.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/5.6.adoc rename to EN/modules/ROOT/pages/master/ecosystem_components/plpgsql_check.adoc diff --git a/EN/modules/ROOT/pages/master/5.1.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/postgis.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/5.1.adoc rename to EN/modules/ROOT/pages/master/ecosystem_components/postgis.adoc diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/redis_fdw.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/redis_fdw.adoc new file mode 100644 index 0000000..e7f6deb --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/redis_fdw.adoc @@ -0,0 +1,108 @@ + +:sectnums: +:sectnumlevels: 5 + += redis_fdw + +== Overview +redis_fdw enables the connection between PostgreSQL and the Redis key-value database, supporting operations such as SELECT, INSERT, UPDATE, and DELETE. It is compatible with PostgreSQL 10+ and Redis versions around 6.0, and can handle various data types including hashes and lists. It works in both PostgreSQL and Oracle compatibility modes.. + +== Installation + +[TIP] +The source code installation environment is Ubuntu 24.04 (x86_64), in which IvorySQL 5 or a later version has been installed. The installation path is /usr/ivory-5. + +=== Source Code Installation + +[literal] +---- +# Install Hiredis lib +wget https://github.com/redis/hiredis/archive/refs/tags/v1.3.0.tar.gz +tar xzvf v1.3.0.tar.gz +cd hiredis-1.3.0 +make +sudo make install +sudo ldconfig + +# Install local Redis server +sudo apt update +sudo apt install -y redis-server + +# Local Redis server is running on 127.0.0.1:6379 + +# Create an administrator user using the redis-cli command: +# (Temporarily created users will be lost after Redis restarts.) +# User: highgo, password: Admin@123 +ACL SETUSER highgo on >Admin@123 ~* +@all + +# Download source code of redis_fdw +git clone https://github.com/pg-redis-fdw/redis_fdw.git -b REL_18_STABLE +cd redis_fdw + +# Compile and install redis_fdw extension +make PG_CONFIG=/usr/ivory-5/bin/pg_config +sudo make install PG_CONFIG=/usr/ivory-5/bin/pg_config + +---- + +[TIP] +If there is error "xlocale.h: No such file or directory" during compilation, user should remove the line of "#define HAVE_XLOCALE_H 1" from file /usr/ivory-5/include/postgresql/server/pg_config.h. + +=== Create extension + +[literal] +---- +postgres=# create extension redis_fdw; +CREATE EXTENSION +---- + +== Usage + +[literal] +---- +// Create a foreign server with appropriate configuration +postgres=# CREATE SERVER redis_server +postgres-# FOREIGN DATA WRAPPER redis_fdw +postgres-# OPTIONS ( address '127.0.0.1', port '6379'); +CREATE SERVER +---- + +[literal] +---- +// User mapping +postgres=# CREATE USER MAPPING FOR highgo +postgres-# SERVER redis_server +postgres-# OPTIONS (password 'Admin@123'); +CREATE USER MAPPING +---- + +[literal] +---- +// Create a simple table +postgres=# CREATE FOREIGN TABLE redis_db0 ( +postgres(# key text, +postgres(# val text +postgres(# ) +postgres-# SERVER redis_server +postgres-# OPTIONS ( +postgres(# database '0' +postgres(# ); +CREATE FOREIGN TABLE +---- + +[literal] +---- +// Insert data +postgres=# insert into redis_db0 values('k2', 'v2'); +INSERT 0 1 +---- + +[literal] +---- +// Read the data +postgres=# select * from redis_db0; + key | val +-----+----- + k2 | v2 +(1 row) +---- \ No newline at end of file diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/set_user.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/set_user.adoc new file mode 100644 index 0000000..7eeb7d6 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/set_user.adoc @@ -0,0 +1,250 @@ +:sectnums: +:sectnumlevels: 5 + += set_user + +== Overview +set_user is a PostgreSQL security auditing extension maintained by the pgaudit project that enhances the native user-switching capability. It supports switching between ordinary users as well as controlled escalation to superuser; configurable allowlists restrict which users may be switched to, all switching operations are recorded in the audit log, full SQL logging is enforced when switching to superuser, and high-risk operations such as modifying database configuration or invoking system commands are blocked. It standardizes temporary privilege escalation. Deployment requires preloading the plugin and restarting the database, then creating the extension before use. + +Project page: + +Version: REL4_2_0 + +License: PostgreSQL License + +== How It Works + +PostgreSQL's native `SET ROLE` / `SET SESSION AUTHORIZATION` has two security weaknesses: first, after escalating privileges a user can casually run `SET log_statement = 'none'` to turn off logging, or `RESET ROLE` to quietly switch back, leaving no audit trail; second, to let DBAs perform superuser work, they typically must be allowed to log in directly as a superuser account, granting excessive and uncontrolled privileges. + +set_user's approach is "don't forbid privilege escalation, but make every escalation fully traceable and non-repudiable." Once deployed, all superuser accounts can be set to NOLOGIN; DBAs log in with ordinary accounts and call set_user_u('postgres') when escalation is needed. Throughout the window from escalation until reset_user() restores the original identity: the user switch is written to the log, log_statement is forcibly set to all so that every SQL statement is persisted, and the log prefix automatically gains an AUDIT tag for easy filtering and alerting; at the same time, every channel that could undermine auditing or escape the identity — ALTER SYSTEM, COPY PROGRAM, SET log_statement, SET ROLE, and the set_config() backdoor — is blocked. Because session_user always remains the real login user, "who did what, when, and under which identity" is plainly visible in the logs. + +Implementation-wise, it is a C extension loaded via shared_preload_libraries and relies on three kernel mechanisms: ProcessUtility_hook to intercept dangerous statements, object_access_hook to block function-level backdoors, and transaction commit callbacks to guarantee transactional safety of the switch. Access control uses a dual-gate design — at the SQL layer, GRANT EXECUTE determines who may call the functions, while at the configuration layer, allowlists (superuser_allowlist, etc.) can be hot-adjusted at any time to tighten control. In addition, it provides a token-locked set_user(user, token) to prevent escape when a connection pooler holds connections on behalf of clients, and the irreversible set_session_auth() for permanent privilege drop before handing over a connection. + +set_user cannot prevent a superuser from acting maliciously — its role is to ensure that no privileged operation can evade auditing. Combined with a log alerting system, it forms a key component of a least-privilege operations framework for PostgreSQL. + +== Usage Notes + +[NOTE] +`set_user` must be added to `shared_preload_libraries`. If other plugins implement `post-execution hooks`, `set_user` must be listed before them. + +[IMPORTANT] +Superuser accounts (e.g., postgres) should be set to NOLOGIN; DBAs log in with ordinary accounts and call `set_user_u('postgres')` when escalation is needed. + +** Configuration options + +[cols="1,1,1,1", options="header"] +|=== +| Option | Description | Example | Function +| set_user.block_alter_system | `on` (default) or `off` | `on` | Block ALTER SYSTEM commands +| set_user.block_copy_program | `on` (default) or `off` | `on` | Block COPY PROGRAM commands +| set_user.block_log_statement | `on` (default) or `off` | `on` | Block changes to log_statement +| set_user.nosuperuser_target_allowlist | Wildcard '*' (default) or string | 'dba1, dba2, +admin_group' | List of target users that set_user() may switch to +| set_user.superuser_allowlist | Wildcard '*' (default) or string | 'dba1, dba2, +admin_group' | List of users allowed to call set_user_u() for escalation +| set_user.superuser_audit_tag | String | 'AUDIT' | Log prefix tag +| set_user.exit_on_error | `on` (default) or `off` | `on` | Whether to exit the current session on error +|=== + + +** Functions + +[cols="1,1,1,1", options="header"] +|=== +| Function | Default privilege | Parameters | Description +| `set_user(text)` | REMOVE FROM PUBLIC | Target non-superuser username | Switch the current session's user identity to the specified non-superuser +| `set_user(text, text)` | REMOVE FROM PUBLIC | Target non-superuser username, random token | Same as above, but requires a random token for added security +| `set_user_u(text)` | REMOVE FROM PUBLIC | Target superuser username | Switch the current session's user identity to the specified superuser +| `reset_user()` | GRANT TO PUBLIC | None | Restore the current session's user identity to the original user +| `reset_user(text)` | GRANT TO PUBLIC | Random token | Token-protected restore; the matching token must be supplied +| `set_session_auth(text)` | REMOVE FROM PUBLIC | Target non-superuser username | Similar to `SET SESSION AUTHORIZATION`, but can only switch to an ordinary user and cannot be reverted +|=== + +== Installation + +[TIP] +The source build was tested on Ubuntu 26.04. IvorySQL must be installed; for installation steps see: + +=== Building from Source + +** Set the IvorySQL installation path +[source,bash] +---- +export IVY_BIN_DIR=/usr/local/ivorysql/bin +export TEST_DB_DIR=/tmp/test_db +---- + +** Get the source code +[source,bash] +---- +git clone https://github.com/pgaudit/set_user.git +cd set_user +git checkout REL4_2_0 +---- + +** Build and install +[source,bash] +---- +make USE_PGXS=1 PG_CONFIG=${IVY_BIN_DIR}/pg_config clean +make USE_PGXS=1 PG_CONFIG=${IVY_BIN_DIR}/pg_config +make USE_PGXS=1 PG_CONFIG=${IVY_BIN_DIR}/pg_config install +---- + +** Configuration +[source,ini] +---- +shared_preload_libraries = 'set_user' +---- + +=== Creating a Test Database + +[NOTE] +`set_user` must be added to `shared_preload_libraries`. + +** Initialize the database +[source,bash] +---- +# Initialize the database +${IVY_BIN_DIR}/initdb -U postgres -D ${TEST_DB_DIR} + +# Add set_user to shared_preload_libraries (this can also be done manually) +sed -i "s/\(shared_preload_libraries = '.*\)'/\1, set_user'/g" ${TEST_DB_DIR}/ivorysql.conf + +# Change the PG port and the Oracle-compatible port to avoid conflicts +cat << EOF >> ${TEST_DB_DIR}/ivorysql.conf +ivorysql.port = 1522 +port = 5433 +EOF + +# Start the test database +${IVY_BIN_DIR}/pg_ctl -D ${TEST_DB_DIR} -l ${TEST_DB_DIR}/logfile start +---- + +=== Usage Test + +** Connect to the database + +[source,bash] +---- +PGHOST=127.0.0.1 PGPORT=1522 PGUSER=postgres $IVY_BIN_DIR/psql +---- + +** Test examples + +[TIP] +The default configuration is used here, i.e., the allowlist is '*'. + +[source,sql] +---- +CREATE EXTENSION set_user; +LOAD 'set_user'; + +--- Create two users +CREATE USER dba; +CREATE USER bob; + +-- Allow dba to execute set_user() +GRANT EXECUTE ON FUNCTION set_user(text) TO dba; +GRANT EXECUTE ON FUNCTION set_user(text,text) TO dba; +GRANT EXECUTE ON FUNCTION set_user_u(text) TO dba; + +-- Switch to user dba to simulate a DBA login +SET SESSION AUTHORIZATION dba; +SELECT SESSION_USER, CURRENT_USER; +--- session_user | current_user +--- --------------+-------------- +--- dba | dba +--- (1 row) + +--- Escalate privileges: using set_user() here fails because the target user is a superuser +SELECT set_user('postgres'); +--- ERROR: switching to superuser not allowed +--- HINT: Use 'set_user_u' to escalate. + +--- Escalate privileges: using set_user_u() here succeeds +SELECT set_user_u('postgres'); +--- set_user_u +--- ------------ +--- OK +--- (1 row) + +--- Check the user state after escalation +SELECT SESSION_USER, CURRENT_USER; +--- session_user | current_user +--- --------------+-------------- +--- dba | postgres +--- (1 row) + +--- Test a repeated set_user; it fails +SELECT set_user('bob'); +--- ERROR: must reset previous user prior to setting again + +--- While escalated, ALTER SYSTEM fails +ALTER SYSTEM SET wal_level = minimal; +--- ERROR: ALTER SYSTEM blocked by set_user config + +--- While escalated, COPY PROGRAM fails +COPY (select 42) TO PROGRAM 'cat'; +--- ERROR: COPY PROGRAM blocked by set_user config + +--- While escalated, SET log_statement fails +SET log_statement = 'none'; +--- ERROR: "SET log_statement" blocked by set_user config + +--- Restore the original user +SELECT reset_user(); +SELECT SESSION_USER, CURRENT_USER; +--- session_user | current_user +--- --------------+-------------- +--- dba | dba +--- (1 row) + +--- Switch to the ordinary user bob +SELECT set_user('bob'); +SELECT SESSION_USER, CURRENT_USER; +--- session_user | current_user +--- --------------+-------------- +--- dba | bob +--- (1 row) + +--- Restore the original user +SELECT reset_user(); +SELECT SESSION_USER, CURRENT_USER; +--- session_user | current_user +--- --------------+-------------- +--- dba | dba +--- (1 row) +---- + +Inspecting `logfile` shows that every operation performed after escalation was recorded, with the `AUDIT` tag in the log prefix. Each user switch has a corresponding line: `LOG: XXX transitioning to YYY`. + +[source,log] +---- +2026-07-17 13:55:34.905 CST [2670610] LOG: Role dba transitioning to Superuser Role postgres +2026-07-17 13:55:34.905 CST [2670610] STATEMENT: SELECT set_user_u('postgres'); +2026-07-16 16:01:21.193 CST [2644289] AUDIT: LOG: statement: SELECT SESSION_USER, CURRENT_USER; +2026-07-16 16:01:43.825 CST [2644289] AUDIT: LOG: statement: SELECT set_user('bob'); +2026-07-16 16:01:43.829 CST [2644289] AUDIT: ERROR: must reset previous user prior to setting again +2026-07-16 16:01:43.829 CST [2644289] AUDIT: STATEMENT: SELECT set_user('bob'); +2026-07-16 16:02:20.177 CST [2644289] AUDIT: LOG: statement: ALTER SYSTEM SET wal_level = minimal; +2026-07-16 16:02:20.177 CST [2644289] AUDIT: ERROR: ALTER SYSTEM blocked by set_user config +2026-07-16 16:02:20.177 CST [2644289] AUDIT: STATEMENT: ALTER SYSTEM SET wal_level = minimal; +2026-07-17 13:56:26.924 CST [2670610] STATEMENT: SELECT set_user_u('postgres'); +2026-07-17 14:01:12.017 CST [2670610] AUDIT: LOG: statement: select reset_user(); +2026-07-17 14:01:12.017 CST [2670610] AUDIT: LOG: Superuser Role postgres transitioning to Role dba +2026-07-17 14:18:42.884 CST [2671784] LOG: Role dba transitioning to Role bob +2026-07-17 14:18:42.884 CST [2671784] STATEMENT: select set_user('bob'); +2026-07-17 14:18:53.301 CST [2671784] LOG: Role bob transitioning to Role dba +2026-07-17 14:18:53.301 CST [2671784] STATEMENT: select reset_user(); +2026-07-17 14:26:16.737 CST [2671784] LOG: Role dba transitioning to Role bob +2026-07-17 14:26:16.737 CST [2671784] STATEMENT: SELECT set_user('bob'); +---- + +=== Cleaning Up the Test Database + +[source,bash] +---- +# Stop the test database +${IVY_BIN_DIR}/pg_ctl -D ${TEST_DB_DIR} -l ${TEST_DB_DIR}/logfile stop + +# Remove the test database +rm -rf ${TEST_DB_DIR} +---- \ No newline at end of file diff --git a/EN/modules/ROOT/pages/master/5.10.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/system_stats.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/5.10.adoc rename to EN/modules/ROOT/pages/master/ecosystem_components/system_stats.adoc diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc new file mode 100644 index 0000000..2fb85dc --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc @@ -0,0 +1,135 @@ + +:sectnums: +:sectnumlevels: 5 + += wal2json + +== Overview +wal2json is an output plugin for logical decoding. It generates JSON object for each transaction. + +== Installation + +[TIP] +The source code installation environment is Ubuntu 24.04 (x86_64), in which IvorySQL 5 or a later version has been installed. The installation path is /usr/ivory-5. + +=== Source Code Installation + +[literal] +---- +# download source code package from: https://github.com/eulerto/wal2json/releases/tag/wal2json_2_6 + +unzip wal2json_2_6.zip +cd wal2json_2_6 + +# compile and install the extension +make PG_CONFIG=/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config install +---- + +[TIP] +If there is error "xlocale.h: No such file or directory" during compilation, user should remove the +line of "#define HAVE_XLOCALE_H 1" from file /usr/ivory-5/include/postgresql/server/pg_config.h. + +=== Modify the configuration file + +Modify the postgresql.conf file to set wal_level as "logical", and set max_replication_slots/max_wal_senders. +[literal] +---- +wal_level = logical +max_replication_slots = 10 +max_wal_senders = 10 +---- + +Make sure the following content to be in pg_hba.conf. +[literal] +---- + local replication all trust + host replication all 127.0.0.1/32 trust + host replication all ::1/128 trust +---- + +Then restart the database. + +== Use + +Open the first terminal and execute command: + +[literal] +---- +sudo -u ivorysql /usr/ivory-5/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json + +Start monitoring, output the changes in JSON format and in real time. +sudo -u ivorysql /usr/ivory-5/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - +---- + +Connect database in the second terminal: +[literal] +---- +/usr/ivory-5/bin/psql -d postgres -p 1521 +---- + +Execute the following SQL statement: +[literal] +---- +CREATE TABLE test_cdc (id int primary key, name varchar(50)); +INSERT INTO test_cdc VALUES (1, 'test1'); +UPDATE test_cdc SET name = 'test1_update' WHERE id = 1; +DELETE FROM test_cdc WHERE id = 1; +DROP TABLE test_cdc; +---- + +The following output will appear in the first terminal: +[literal] +---- +{ + "change": [ + ] +} +{ + "change": [ + { + "kind": "insert", + "schema": "public", + "table": "test_cdc", + "columnnames": ["id", "name"], + "columntypes": ["integer", "sys.oravarcharbyte(50)"], + "columnvalues": [1, "test1"] + } + ] +} +{ + "change": [ + { + "kind": "update", + "schema": "public", + "table": "test_cdc", + "columnnames": ["id", "name"], + "columntypes": ["integer", "sys.oravarcharbyte(50)"], + "columnvalues": [1, "test1_update"], + "oldkeys": { + "keynames": ["id"], + "keytypes": ["integer"], + "keyvalues": [1] + } + } + ] +} +{ + "change": [ + { + "kind": "delete", + "schema": "public", + "table": "test_cdc", + "oldkeys": { + "keynames": ["id"], + "keytypes": ["integer"], + "keyvalues": [1] + } + } + ] +} +{ + "change": [ + ] +} +---- diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/zhparser_en.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/zhparser_en.adoc new file mode 100644 index 0000000..4373792 --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/zhparser_en.adoc @@ -0,0 +1,92 @@ +:sectnums: +:sectnumlevels: 5 + += zhparser + +== Overview +zhparser is a PostgreSQL plugin for Chinese full-text search. It implements a Chinese parser based on SCWS (Simple Chinese Word Segmentation). + +== Installation + +[TIP] +The source code installation environment is Ubuntu 24.04 (x86_64), with IvorySQL already installed in the environment at the path /path-to/ivorysql + +=== Source Installation + +[literal] +---- +First, install the dependencies + +wget http://www.xunsearch.com/scws/down/scws-1.2.3.tar.bz2 +tar xf scws-1.2.3.tar.bz2 +cd scws-1.2.3 + +Compile and install scws +./configure +make +sudo make install + +# Use git to download the zhparser source code, using the master branch +git clone https://github.com/amutu/zhparser +cd zhparser + +# Compile and install +make PG_CONFIF=/path-to/ivorysql/bin/pg_config +make PG_CONFIF=/path-to/ivorysql/bin/pg_config install +---- + +== Create Extension and Full-Text Search Configuration, Bind Dictionary Rules to Token Pos Tags + +Connect to the database using psql, and execute the following commands: +[literal] +---- +-- create the extension +CREATE EXTENSION zhparser; + +-- make test configuration using parser +CREATE TEXT SEARCH CONFIGURATION testzhcfg (PARSER = zhparser); + +-- add token mapping +ALTER TEXT SEARCH CONFIGURATION testzhcfg ADD MAPPING FOR n,v,a,i,e,l WITH simple; +---- + +== Usage + +[literal] +---- +-- ts_parse +ivorysql=# SELECT * FROM ts_parse('zhparser', 'hello world! 2010年保障房建设在全国范围内获全面启动，从中央到地方纷纷加大 了保障房的建设和投入力度 。2011年，保障房进入了更大规模的建设阶段。住房城乡建设部党组书记、部长姜伟新去年底在全国住房城乡建设工作会议上表示，要继续推进保障性安居工程建设。'); + tokid | token +-------+---------- + 101 | hello + 101 | world + 117 | ! + 101 | 2010 + 113 | 年 + 118 | 保障 + 110 | 房建 + ...... + +-- test to_tsvector +ivorysql=# SELECT to_tsvector('testzhcfg','"今年保障房新开工数量虽然有所下调，但实际的年度在建规模以及竣工规模会超以往年份，相对应的对资金的需求也会 创历>史纪录。"陈国强说。在他看来，与2011年相比，2012年的保障房建设在资金配套上的压力将更为严峻。'); + + to_tsvector + +----------------------------------------------------------------------------------------------------------------------------------------------------- +----------------------------------------------------------------------------------------------------------------------------------------------------- +--------------------------- + '2011':27 '2012':29 '上':35 '下调':7 '严峻':37 '会':14 '会创':20 '保障':1,30 '压力':36 '史':21 '国强':24 '在建':10 '实际':8 '对应':17 '年份':16 '年 +':9 '开工':4 '房':2 '房建':31 '数量':5 '新':3 '有所':6 '相比':28 '看来':26 '竣工':12 '纪录':22 '规模':11,13 '设在':32 '说':25 '资金':18,33 '超':15 ' +套':34 '陈':23 '需求':19 +(1 row) + +-- test to_tsquery +ivorysql=# SELECT to_tsquery('testzhcfg', '保障房资金压力'); + to_tsquery +--------------------------------------- + '保障' <-> '房' <-> '资金' <-> '压力' +(1 row) + +---- + +For more detailed usage and advanced features, please refer to https://github.com/amutu/zhparser . diff --git a/EN/modules/ROOT/pages/master/6.5.adoc b/EN/modules/ROOT/pages/master/gb18030.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.5.adoc rename to EN/modules/ROOT/pages/master/gb18030.adoc diff --git a/EN/modules/ROOT/pages/master/3.3.adoc b/EN/modules/ROOT/pages/master/getting-started/daily_maintenance.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/3.3.adoc rename to EN/modules/ROOT/pages/master/getting-started/daily_maintenance.adoc diff --git a/EN/modules/ROOT/pages/master/3.2.adoc b/EN/modules/ROOT/pages/master/getting-started/daily_monitoring.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/3.2.adoc rename to EN/modules/ROOT/pages/master/getting-started/daily_monitoring.adoc diff --git a/EN/modules/ROOT/pages/master/3.1.adoc b/EN/modules/ROOT/pages/master/getting-started/quick_start.adoc similarity index 95% rename from EN/modules/ROOT/pages/master/3.1.adoc rename to EN/modules/ROOT/pages/master/getting-started/quick_start.adoc index 8a1c16e..478c19d 100644 --- a/EN/modules/ROOT/pages/master/3.1.adoc +++ b/EN/modules/ROOT/pages/master/getting-started/quick_start.adoc @@ -111,6 +111,8 @@ ivorysql 3238 1551 0 20:35 pts/0 00:00:00 grep --color=auto postgres $ docker pull ivorysql/ivorysql:5.0-ubi8 ``` +TIP: Users in China can also pull from the IvorySQL community Harbor registry for faster downloads, e.g. `docker pull registry.highgo.com/ivorysql/ivorysql:5.0-ubi8`. + ** Running IvorySQL ``` $ docker run --name ivorysql -p 5434:5432 -e IVORYSQL_PASSWORD=your_password -d ivorysql/ivorysql:5.0-ubi8 @@ -143,4 +145,4 @@ TIP: When running IvorySQL in Docker, additional parameters need to be added, li Now you can start your journey of IvorySQL! Enjoy! -To explore additional installation methods, please refer to the xref:v5.0/6.adoc[Installation]. \ No newline at end of file +To explore additional installation methods, please refer to the xref:master/installation_guide.adoc[Installation]. \ No newline at end of file diff --git a/EN/modules/ROOT/pages/master/4.1.adoc b/EN/modules/ROOT/pages/master/installation_guide.adoc similarity index 96% rename from EN/modules/ROOT/pages/master/4.1.adoc rename to EN/modules/ROOT/pages/master/installation_guide.adoc index aad96b6..e511ae6 100644 --- a/EN/modules/ROOT/pages/master/4.1.adoc +++ b/EN/modules/ROOT/pages/master/installation_guide.adoc @@ -15,7 +15,7 @@ The installation methods for IvorySQL include the following five: - <> -This chapter will provide detailed instructions on the installation, execution, and uninstallation processes for each method. For a quicker access to IvorySQL, please refer to xref:v5.0/3.adoc#quick-installation[Quick installation]. +This chapter will provide detailed instructions on the installation, execution, and uninstallation processes for each method. For a quicker access to IvorySQL, please refer to xref:master/getting-started/quick_start.adoc#quick-installation[Quick installation]. Before getting started, please create an user and grant it root privileges. All the installation steps will be performed by this user. Here we just name it 'ivorysql'. @@ -44,6 +44,8 @@ $ sudo dnf install -y ivorysql5-5.0 $ docker pull ivorysql/ivorysql:5.0-ubi8 ``` +TIP: Users in China can also pull from the IvorySQL community Harbor registry for faster downloads — replace the image reference above with `registry.highgo.com/ivorysql/ivorysql:5.0-ubi8`. + ** Run IvorySQL ``` $ docker run --name ivorysql -p 5434:5432 -e IVORYSQL_PASSWORD=your_password -d ivorysql/ivorysql:5.0-ubi8 @@ -90,7 +92,7 @@ IvorySQL then will be installed in the /usr/ivory-5/ directory. == Source code installation ** Installing dependencies ``` -$ sudo dnf install -y bison readline-devel zlib-devel openssl-devel +$ sudo dnf install -y bison readline-devel zlib-devel openssl-devel uuid-devel $ sudo dnf groupinstall -y 'Development Tools' ``` ** Getting source code diff --git a/EN/modules/ROOT/pages/master/4.5.adoc b/EN/modules/ROOT/pages/master/migration_guide.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/4.5.adoc rename to EN/modules/ROOT/pages/master/migration_guide.adoc diff --git a/EN/modules/ROOT/pages/master/4.4.adoc b/EN/modules/ROOT/pages/master/operation_guide.adoc similarity index 97% rename from EN/modules/ROOT/pages/master/4.4.adoc rename to EN/modules/ROOT/pages/master/operation_guide.adoc index f3a6392..b0c92fb 100644 --- a/EN/modules/ROOT/pages/master/4.4.adoc +++ b/EN/modules/ROOT/pages/master/operation_guide.adoc @@ -140,6 +140,112 @@ We can also create a fallback server using logical replication of an updated ver This upgrade method can be used with built-in logical replication tools and external logical replication systems such as pglogical, Slony, Londiste, and Bucardo. +== Use pg_upgrade to upgrade PostgreSQL to IvorySQL + +`pg_upgrade` supports upgrading native PostgreSQL clusters to IvorySQL. When the source cluster is native PostgreSQL and the target is an IvorySQL cluster, the `-g`（`--using-ora-pg`）parameter must be used. + +Upgrading from PostgreSQL to IvorySQL is considered a "cross-product" migration. It is recommended to complete comprehensive testing and validation before performing the operation in a production environment, and to develop a detailed rollback plan. + +=== Parameter description + +[cols="1,3"] +|=== +| parameter | description + +| `-g` / `--using-ora-pg` +| Indicates that the source cluster is PostgreSQL and the target is an IvorySQL cluster. When this option is enabled, `pg_upgrade` uniformly uses the PostgreSQL standard port when connecting to both clusters, ensuring proper handling of the upgrade. +|=== + +=== Upgrade steps + +==== Step 1: Initialize the new IvorySQL cluster + +[source,bash] +---- +# Use IvorySQL's initdb to initialize the new cluster +/opt/ivorysql/bin/initdb -D /data/ivorysql/data +---- + +==== Step 2: Stop the source PostgreSQL cluster + +[source,bash] +---- +/usr/lib/postgresql/16/bin/pg_ctl -D /data/pg/data stop +---- + +==== Step 3: Run the upgrade check (optional) + +Use `-c` to perform compatibility checks only without modifying any data: + +[source,bash] +---- +/opt/ivorysql/bin/pg_upgrade \ + -b /usr/lib/postgresql/16/bin \ # Source PG executable directory + -B /opt/ivorysql/bin \ # Target IvorySQL executable directory + -d /data/pg/data \ # Source PG data directory + -D /data/ivorysql/data \ # Target IvorySQL data directory + -g \ # Source is PostgreSQL, target is IvorySQL + -c # Check only, do not upgrade +---- + +==== Step 4: Perform the upgrade + +[source,bash] +---- +/opt/ivorysql/bin/pg_upgrade \ + -b /usr/lib/postgresql/16/bin \ + -B /opt/ivorysql/bin \ + -d /data/pg/data \ + -D /data/ivorysql/data \ + -g +---- + +==== Step 5: Start IvorySQL and verify + +[source,bash] +---- +/opt/ivorysql/bin/pg_ctl -D /data/ivorysql/data start + +# Verify if the database is functioning properly +/opt/ivorysql/bin/psql -p 5432 -c "SELECT version();" +---- + +==== Step 6: Clean up the old cluster + +After the upgrade is completed,`pg_upgrade` will generate a cleanup script: + +[source,bash] +---- +./delete_old_cluster.sh +---- + +=== Quick reference for common parameters + +[cols="1,2,2"] +|=== +| Parameters | Long options | Description + +| `-b` | `--old-bindir` | Source cluster executable directory +| `-B` | `--new-bindir` | Target cluster executable directory +| `-d` | `--old-datadir` | Source cluster data directory +| `-D` | `--new-datadir` | Target cluster data directory +| `-g` | `--using-ora-pg` | Upgrade from PostgreSQL to IvorySQL +| `-p` | `--old-port` | Source cluster port +| `-P` | `--new-port` | Target cluster PG port +| `-q` | `--old-oraport` | Source cluster Oracle port +| `-Q` | `--new-oraport` | Target cluster Oracle port +| `-j` | `--jobs` | Number of parallel processes +| `-k` | `--link` | Use hard links instead of file copying +| `-c` | `--check` | Check compatibility only, do not perform the upgrade +| `-v` | `--verbose` | Output detailed logs +|=== + +=== Precautions + +* The `-g` parameter is only used when the **source is a pure PostgreSQL cluster**、and the target is **IvorySQL** . If the source cluster is already IvorySQL, this parameter is not needed. +* Both the source and target clusters must be in a stopped state during the upgrade. +* Be sure to perform a complete backup of the source cluster before upgrading. + == Managing IvorySQL Versions IvorySQL is based on PostgreSQL and is updated at the same frequency as PostgreSQL, with one major release per year and one minor release per quarter. IvorySQL 5.0 is based on PostgreSQL 18.0, and all versions of IvorySQL are backward compatible.The relevant version features can be viewed by looking at https://www.ivorysql.org/en/releases-page/[Official Website]。 diff --git a/EN/modules/ROOT/pages/master/oracle_builtin_functions/rawtohex.adoc b/EN/modules/ROOT/pages/master/oracle_builtin_functions/rawtohex.adoc new file mode 100644 index 0000000..013eed7 --- /dev/null +++ b/EN/modules/ROOT/pages/master/oracle_builtin_functions/rawtohex.adoc @@ -0,0 +1,57 @@ + +:sectnums: +:sectnumlevels: 5 + + += **Feature Overview** + +IvorySQL provides compatibility with Oracle's built-in function ```RAWTOHEX('parameter')```, which is used to convert RAW to a character value containing its hexadecimal representation. + +== Implementation Principle + +The function pg_catalog.encode(bytea, 'hex') provided by PostgreSQL can directly convert binary data to hexadecimal. + +Given that the existing HEXTORAW function in the current system is implemented by wrapping pg_catalog.decode using SQL, the RAWTOHEX function developed this time will be implemented in the same way. That is, it will be a SQL function wrapping the PostgreSQL built-in function pg_catalog.encode, rather than implementing it via a C extension. + +The following four types need to be supported as input: raw, text, bytea, and varchar2. + +sys.raw is a domain type of bytea (typtype = 'd', typbasetype = bytea). PostgreSQL supports implicit conversion from a domain type to its base type, so RAWTOHEX(bytea) can automatically accept sys.raw as input. + +There is an IMPLICIT cast from sys.oravarcharchar (i.e., varchar2) to pg_catalog.text (defined in datatype--1.0.sql), so RAWTOHEX(text) can automatically accept varchar2 as input. + +Therefore, two overloaded versions (rather than four) will be defined. + +``` +sys.rawtohex(bytea) RETURNS varchar2 +sys.rawtohex(text) RETURNS varchar2 +``` + +The specific functionality is implemented in builtin_functions—1.0.sql. +```sql +/* support rawtohex function for oracle compatibility */ +CREATE OR REPLACE FUNCTION sys.rawtohex(bytea) +RETURNS varchar2 +AS $$ SELECT CASE WHEN pg_catalog.octet_length($1) > 0 THEN upper(pg_catalog.encode($1, 'hex'))::varchar2 END; $$ +LANGUAGE SQL +PARALLEL SAFE +STRICT +IMMUTABLE; + +CREATE OR REPLACE FUNCTION sys.rawtohex(text) +RETURNS varchar2 +AS $$ SELECT CASE WHEN pg_catalog.octet_length($1) > 0 THEN upper(pg_catalog.encode($1::bytea, 'hex'))::varchar2 END; $$ +LANGUAGE SQL +PARALLEL SAFE +STRICT +IMMUTABLE; +``` + +== RAWTOHEX use cases +[cols="8,2"] +|==== +|*SQL statement *|*return value* +|SELECT sys.rawtohex('\xDEADBEEF'::bytea); | DEADBEEF +|SELECT sys.rawtohex('\xFF'::raw); | FF +|SELECT sys.rawtohex('hello'::text); | 68656C6C6F +|SELECT sys.rawtohex('hello'::varchar2); | 68656C6C6F +|==== diff --git a/EN/modules/ROOT/pages/master/oracle_builtin_functions/stragg.adoc b/EN/modules/ROOT/pages/master/oracle_builtin_functions/stragg.adoc new file mode 100644 index 0000000..03cba3d --- /dev/null +++ b/EN/modules/ROOT/pages/master/oracle_builtin_functions/stragg.adoc @@ -0,0 +1,148 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += Implementation of the STRAGG Aggregate Function + +== Purpose + +IvorySQL adds the `sys.stragg(text)` aggregate function to the `contrib/ivorysql_ora` +extension, implementing string aggregation behavior compatible with Oracle's function of +the same name: concatenating all non-NULL values within a group into a single +comma-separated string. + +== Implementation Notes + +=== State Layout Design + +The STRAGG aggregate reuses the `StringInfo` state layout used by PostgreSQL's built-in +`string_agg`, which allows direct reuse of four built-in functions — +`string_agg_finalfn`, `string_agg_combine`, `string_agg_serialize`, and +`string_agg_deserialize` — without implementing separate finalize, parallel combine, or +serialization logic. + +The `StringInfo` state convention is as follows: + +[source,c] +---- +/* + * data = "," + val1 + "," + val2 + ... + * A leading comma is prepended before the first value so that + * the finalfn can strip it uniformly. + * cursor = 1 + * Stores the byte length of the leading delimiter (one byte for ","). + * string_agg_finalfn returns &data[cursor], automatically dropping + * the leading comma. + */ +---- + +=== Transition Function (`stragg_transfn`) + +The transition function is located in +`contrib/ivorysql_ora/src/builtin_functions/misc_functions.c`. + +[source,c] +---- +PG_FUNCTION_INFO_V1(stragg_transfn); + +Datum +stragg_transfn(PG_FUNCTION_ARGS) +{ + StringInfo state; + MemoryContext aggcontext; + MemoryContext oldcontext; + + if (!AggCheckCallContext(fcinfo, &aggcontext)) + elog(ERROR, "stragg_transfn called in non-aggregate context"); + + state = PG_ARGISNULL(0) ? NULL : (StringInfo) PG_GETARG_POINTER(0); + + /* Skip NULL input values, consistent with Oracle STRAGG behavior */ + if (!PG_ARGISNULL(1)) + { + text *value = PG_GETARG_TEXT_PP(1); + + if (state == NULL) + { + oldcontext = MemoryContextSwitchTo(aggcontext); + state = makeStringInfo(); + MemoryContextSwitchTo(oldcontext); + + /* First value: prepend delimiter and record its length in cursor */ + appendStringInfoChar(state, ','); + state->cursor = 1; + } + else + { + appendStringInfoChar(state, ','); + } + + appendBinaryStringInfo(state, VARDATA_ANY(value), VARSIZE_ANY_EXHDR(value)); + } + + if (state) + PG_RETURN_POINTER(state); + PG_RETURN_NULL(); +} +---- + +Key design points: + +* The state is allocated in the aggregate memory context (`aggcontext`) and lives for + the entire duration of the aggregation. +* The `StringInfo` internal buffer doubles in capacity on demand; append operations are + amortized O(1), giving an overall time complexity of O(N) — superior to the O(N²) of + a pure-SQL string-concatenation approach. +* NULL input is detected via `PG_ARGISNULL(1)` and silently skipped, leaving the + accumulated state unaffected. + +=== SQL Definitions + +The transition function and aggregate are defined in +`contrib/ivorysql_ora/src/builtin_functions/builtin_functions--1.0.sql`. + +[source,sql] +---- +CREATE FUNCTION sys.stragg_transfn(internal, text) +RETURNS internal +AS 'MODULE_PATHNAME', 'stragg_transfn' +LANGUAGE C +CALLED ON NULL INPUT +PARALLEL SAFE; + +CREATE AGGREGATE sys.stragg(text) ( + SFUNC = sys.stragg_transfn, + STYPE = internal, + FINALFUNC = string_agg_finalfn, + COMBINEFUNC = string_agg_combine, + SERIALFUNC = string_agg_serialize, + DESERIALFUNC = string_agg_deserialize, + PARALLEL = SAFE +); +---- + +`FINALFUNC`, `COMBINEFUNC`, `SERIALFUNC`, and `DESERIALFUNC` all reference PostgreSQL's +built-in `string_agg` family directly, because STRAGG uses exactly the same `StringInfo` +state format as `string_agg`. + +=== Parallel Aggregation Support + +By specifying `COMBINEFUNC = string_agg_combine` together with the serialize and +deserialize functions, STRAGG supports PostgreSQL's parallel aggregation execution path. +Each parallel worker maintains its own partial `StringInfo` state independently; the +leader process merges them via `string_agg_combine`. + +=== Regression Tests + +Test file: `contrib/ivorysql_ora/sql/ora_stragg.sql` +Expected output: `contrib/ivorysql_ora/expected/ora_stragg.out` + +`ora_stragg` has been added to the `ORA_REGRESS` list in the `Makefile` and can be run +with: + +[source,bash] +---- +cd contrib/ivorysql_ora +make installcheck +---- diff --git a/EN/modules/ROOT/pages/master/6.4.1.adoc b/EN/modules/ROOT/pages/master/oracle_builtin_functions/sys_context.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.4.1.adoc rename to EN/modules/ROOT/pages/master/oracle_builtin_functions/sys_context.adoc diff --git a/EN/modules/ROOT/pages/master/6.4.2.adoc b/EN/modules/ROOT/pages/master/oracle_builtin_functions/userenv.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/6.4.2.adoc rename to EN/modules/ROOT/pages/master/oracle_builtin_functions/userenv.adoc diff --git a/EN/modules/ROOT/pages/master/7.6.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/anonymous_block.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.6.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/anonymous_block.adoc diff --git a/EN/modules/ROOT/pages/master/7.8.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/builtin_types_functions.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.8.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/builtin_types_functions.adoc diff --git a/EN/modules/ROOT/pages/master/7.22.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_call_into.adoc similarity index 97% rename from EN/modules/ROOT/pages/master/7.22.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_call_into.adoc index 73d05b9..c477b1d 100644 --- a/EN/modules/ROOT/pages/master/7.22.adoc +++ b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_call_into.adoc @@ -115,6 +115,14 @@ ivorysql=# PRINT default_result; === Reference bind variables in parameters or INTO clauses [source,sql] ---- +-- Create functions stand_alone_func +CREATE OR REPLACE FUNCTION stand_alone_func(p_input NUMBER) +RETURN NUMBER +IS +BEGIN + RETURN p_input * 2; +END; +/ -- Set input bind variables ivorysql=# VARIABLE input_num NUMBER = 7; -- Call a function using bind variables as parameters diff --git a/EN/modules/ROOT/pages/master/7.3.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_case_conversion.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.3.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_case_conversion.adoc diff --git a/EN/modules/ROOT/pages/master/oracle_compatibility/compat_create_index_online.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_create_index_online.adoc new file mode 100644 index 0000000..86392c1 --- /dev/null +++ b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_create_index_online.adoc @@ -0,0 +1,116 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += ONLINE Parameter for CREATE INDEX + +== Purpose + +This document explains the behavior of the `ONLINE` parameter when creating an index in IvorySQL. This parameter is implemented to maintain compatibility with Oracle behavior. + +== Feature Description + +- When specified during index creation, the `ONLINE` parameter allows concurrent DML operations, similar to PostgreSQL's `CONCURRENTLY`, but cannot be used together with `CONCURRENTLY`. +- `ONLINE` must be supported in any position after the closing `)` of the column list and before the `WHERE` clause, regardless of the order relative to other attributes such as `TABLESPACE` and `PARALLEL` (if supported). +- `CREATE INDEX ... ONLINE` on a temporary table is automatically downgraded to a regular build (no error, consistent with `CONCURRENTLY` behavior). +- `CREATE INDEX ... ONLINE` on a partitioned table is **automatically downgraded to a regular build** (no error). + +== Test Cases + +=== Test Environment Setup + +[source,sql] +---- +-- Basic test table +CREATE TABLE tbl_ci_online ( + id NUMBER(10) PRIMARY KEY, + name VARCHAR2(100), + dept_id NUMBER(10), + salary NUMBER(10,2), + status VARCHAR2(20) +); +INSERT INTO tbl_ci_online + SELECT g, 'name'||g, MOD(g,20), g*100.0, CASE WHEN MOD(g,2)=0 THEN 'ACTIVE' ELSE 'INACTIVE' END + FROM generate_series(1, 1000) g; + +-- Unique index test table +CREATE TABLE tbl_ci_unique ( + id NUMBER(10) PRIMARY KEY, + email VARCHAR2(200) NOT NULL +); +INSERT INTO tbl_ci_unique SELECT g, 'user'||g||'@example.com' FROM generate_series(1, 200) g; + +-- Partitioned table +CREATE TABLE tbl_ci_part ( + id NUMBER(10), + region VARCHAR2(20) +) PARTITION BY RANGE (id); +CREATE TABLE tbl_ci_part_p1 PARTITION OF tbl_ci_part FOR VALUES FROM (1) TO (501); +CREATE TABLE tbl_ci_part_p2 PARTITION OF tbl_ci_part FOR VALUES FROM (501) TO (1001); +INSERT INTO tbl_ci_part SELECT g, CASE WHEN g<=500 THEN 'east' ELSE 'west' END + FROM generate_series(1, 1000) g; +---- + +=== Basic ONLINE Build + +[source,sql] +---- +-- Simplest form +CREATE INDEX idx_online_name ON tbl_ci_online (name) ONLINE; + +-- Verify the index was created and is valid +SELECT indisvalid FROM pg_index + WHERE indexrelid = 'idx_online_name'::regclass; +-- Expected: t + +-- Multi-column index +CREATE INDEX idx_online_multi ON tbl_ci_online (dept_id, salary) ONLINE; + +-- Expression index +CREATE INDEX idx_online_expr ON tbl_ci_online (lower(name)) ONLINE; +---- + +=== ONLINE on Partitioned Table + +[source,sql] +---- +-- ONLINE on parent of partitioned table +-- Note: ONLINE on a partitioned table is silently downgraded to a regular build, +-- consistent with Oracle behavior +CREATE INDEX idx_part_online ON tbl_ci_part (id) ONLINE; + +-- Verify the index was created and is valid (downgraded to regular build; +-- parent index and all partition child indexes are VALID) +SELECT c.relname, i.indisvalid +FROM pg_index i +JOIN pg_class c ON c.oid = i.indexrelid +WHERE c.relname LIKE '%idx_part_online%' +ORDER BY c.relname; +-- Expected: idx_part_online (parent) and both partition child indexes have indisvalid = t + +-- Partitioned table ONLINE + TABLESPACE +CREATE INDEX idx_part_online_tbs ON tbl_ci_part (region) ONLINE TABLESPACE pg_default; +---- + +=== CONCURRENTLY and ONLINE Are Mutually Exclusive + +[source,sql] +---- +-- CONCURRENTLY first, ONLINE after +CREATE INDEX CONCURRENTLY idx_both ON tbl_ci_online (name) ONLINE; +-- Expected: ERROR: cannot use both CONCURRENTLY and ONLINE + +-- Verify existing CONCURRENTLY syntax is unaffected +CREATE INDEX CONCURRENTLY idx_still_conc ON tbl_ci_online (dept_id); +-- Expected: success +---- + +=== Test Environment Cleanup + +[source,sql] +---- +DROP TABLE IF EXISTS tbl_ci_online CASCADE; +DROP TABLE IF EXISTS tbl_ci_part CASCADE; +DROP TABLE IF EXISTS tbl_ci_unique CASCADE; +---- diff --git a/EN/modules/ROOT/pages/master/7.21.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_empty_string_to_null.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.21.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_empty_string_to_null.adoc diff --git a/EN/modules/ROOT/pages/master/7.18.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_force_view.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.18.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_force_view.adoc diff --git a/EN/modules/ROOT/pages/master/7.7.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_function_procedure.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.7.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_function_procedure.adoc diff --git a/EN/modules/ROOT/pages/master/7.5.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_like_operator.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.5.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_like_operator.adoc diff --git a/EN/modules/ROOT/pages/master/7.19.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_nested_function.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.19.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_nested_function.adoc diff --git a/EN/modules/ROOT/pages/master/7.17.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_nls_parameter.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.17.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_nls_parameter.adoc diff --git a/EN/modules/ROOT/pages/master/7.15.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_out_parameter.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.15.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_out_parameter.adoc diff --git a/EN/modules/ROOT/pages/master/oracle_compatibility/compat_read_only_view.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_read_only_view.adoc new file mode 100644 index 0000000..3efce9f --- /dev/null +++ b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_read_only_view.adoc @@ -0,0 +1,168 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += Read Only View + +== Purpose + +This document describes the purpose of `WITH READ ONLY` views in IvorySQL, implementing read-only view functionality to maintain compatibility with Oracle behavior. + +== Functionality Description + +- `WITH READ ONLY`: When specified during view creation, this option prevents INSERT, UPDATE, DELETE, and MERGE operations on the view. +- Storage: `read_only=true` is stored in the view's `reloptions`. +- Mutually Exclusive: `WITH READ ONLY` and `WITH CHECK OPTION` are mutually exclusive and cannot be specified together. +- Force View Support: Can be used with `FORCE VIEW`; the read-only attribute takes effect after the view compiles successfully. +- CREATE OR REPLACE Behavior: If a view is recreated using `CREATE OR REPLACE VIEW` without specifying `WITH READ ONLY`, the read-only attribute is cleared. + +== Test Cases + +=== Creating a Read Only View + +[source,sql] +---- +-- Create base table +CREATE TABLE t_ro (a int, b text); +INSERT INTO t_ro VALUES (1, 'hello'), (2, 'world'); + +-- Create read-only view, SELECT succeeds +CREATE VIEW ro_view AS SELECT * FROM t_ro WITH READ ONLY; +SELECT * FROM ro_view ORDER BY a; +-- Expected output: +-- a | b +-- ---+------- +-- 1 | hello +-- 2 | world +---- + +=== Verify DML is Blocked + +[source,sql] +---- +-- INSERT is blocked +INSERT INTO ro_view VALUES (3, 'fail'); +-- Expected output: ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. + +-- UPDATE is blocked +UPDATE ro_view SET b = 'fail' WHERE a = 1; +-- Expected output: ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. + +-- DELETE is blocked +DELETE FROM ro_view WHERE a = 1; +-- Expected output: ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. +---- + +=== MERGE Command is Blocked + +[source,sql] +---- +-- MERGE with INSERT action +MERGE INTO ro_view USING (SELECT 4 AS a, 'merge_ins' AS b) AS src +ON (ro_view.a = src.a) +WHEN NOT MATCHED THEN INSERT VALUES (src.a, src.b); +-- Expected output: ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. + +-- MERGE with UPDATE action +MERGE INTO ro_view USING (SELECT 1 AS a, 'merge_upd' AS b) AS src +ON (ro_view.a = src.a) +WHEN MATCHED THEN UPDATE SET b = src.b; +-- Expected output: ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. +---- + +=== CREATE OR REPLACE Behavior + +[source,sql] +---- +-- Recreating view with WITH READ ONLY: DML is still blocked +CREATE OR REPLACE VIEW ro_view AS SELECT a, b FROM t_ro WITH READ ONLY; +INSERT INTO ro_view VALUES (3, 'fail'); +-- Expected output: ERROR: cannot modify view "ro_view" +-- HINT: The view is defined as read-only. + +-- Recreating view without WITH READ ONLY: read-only attribute is cleared, view becomes updatable +CREATE OR REPLACE VIEW ro_view AS SELECT a, b FROM t_ro; +INSERT INTO ro_view VALUES (3, 'now_writable'); +SELECT * FROM ro_view ORDER BY a; +-- Expected output: +-- a | b +-- ---+-------------- +-- 1 | hello +-- 2 | world +-- 3 | now_writable +---- + +=== FORCE VIEW with WITH READ ONLY + +[source,sql] +---- +-- Base table does not exist at creation time, view is a placeholder +CREATE FORCE VIEW force_ro_view AS SELECT * FROM nonexistent_for_ro WITH READ ONLY; +-- Expected output: WARNING: View created with compilation errors + +-- Create base table +CREATE TABLE nonexistent_for_ro (a int, b text); + +-- Explicitly compile view +ALTER VIEW force_ro_view COMPILE; + +-- After successful compilation, DML is blocked +INSERT INTO force_ro_view VALUES (1, 'fail'); +-- Expected output: ERROR: cannot modify view "force_ro_view" +-- HINT: The view is defined as read-only. +---- + +=== Recursive View with WITH READ ONLY + +[source,sql] +---- +CREATE RECURSIVE VIEW ro_recursive_view (a) AS + SELECT 1 + UNION ALL + SELECT a + 1 FROM ro_recursive_view WHERE a < 3 +WITH READ ONLY; + +SELECT * FROM ro_recursive_view ORDER BY a; +-- Expected output: +-- a +-- --- +-- 1 +-- 2 +-- 3 + +INSERT INTO ro_recursive_view VALUES (99); +-- Expected output: ERROR: cannot modify view "ro_recursive_view" +-- HINT: The view is defined as read-only. +---- + +=== Verify reloptions Storage + +[source,sql] +---- +SELECT relname, reloptions +FROM pg_class +WHERE relname IN ('ro_view', 'ro_recursive_view', 'force_ro_view') +ORDER BY relname; +-- Expected output: +-- relname | reloptions +-- -------------------+------------------ +-- force_ro_view | {read_only=true} +-- ro_recursive_view | {read_only=true} +-- ro_view | +---- + +=== Cleanup + +[source,sql] +---- +DROP VIEW IF EXISTS ro_view; +DROP VIEW IF EXISTS force_ro_view; +DROP TABLE t_ro; +DROP TABLE IF EXISTS nonexistent_for_ro; +---- diff --git a/EN/modules/ROOT/pages/master/7.14.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_rowid.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.14.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_rowid.adoc diff --git a/EN/modules/ROOT/pages/master/oracle_compatibility/compat_stragg.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_stragg.adoc new file mode 100644 index 0000000..8ad234f --- /dev/null +++ b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_stragg.adoc @@ -0,0 +1,179 @@ +:sectnums: +:sectnumlevels: 5 + +:imagesdir: ./_images + += STRAGG Aggregate Function + +== Purpose + +This document describes the `sys.stragg` aggregate function in IvorySQL, which +concatenates multiple string values within a group into a single comma-separated string +in a manner compatible with Oracle. + +== Functional Description + +* `sys.stragg(text)` is an aggregate function that joins all non-NULL text values within + a group using a comma (`,`) as the separator. +* NULL values are silently ignored; they do not appear in the result and do not produce + extra commas. +* When all values in a group are NULL, or when the group is an empty set, the function + returns `NULL` (not an empty string). +* Output order is not guaranteed. To obtain a deterministic order, use an `ORDER BY` + clause inside the aggregate call (a PostgreSQL aggregate extension). +* The function is defined in the `sys` schema and is available in both Oracle + compatibility mode (don't need sys prefix) and PG mode. + +== Syntax + +[source,sql] +---- +sys.stragg(expr [ORDER BY sort_expr [ASC | DESC] [, ...]]) +---- + +[cols="1,3"] +|=== +|Parameter |Description + +|`expr` +|Any expression that can be implicitly cast to `text`. NULL values are skipped. + +|`ORDER BY` +|Optional. Specifies the concatenation order within the group. If omitted, order is + indeterminate. +|=== + +Return type: `text` + +== Test Cases + +=== Test Environment Setup + +[source,sql] +---- +CREATE TABLE stragg_test (dept TEXT, name TEXT); +INSERT INTO stragg_test VALUES ('HR', 'Alice'); +INSERT INTO stragg_test VALUES ('HR', 'Bob'); +INSERT INTO stragg_test VALUES ('HR', 'Carol'); +INSERT INTO stragg_test VALUES ('IT', 'Dave'); +INSERT INTO stragg_test VALUES ('IT', 'Eve'); +---- + +=== Basic Aggregation + +[source,sql] +---- +-- Single-group aggregation with ORDER BY for deterministic output +SELECT sys.stragg(name ORDER BY name) FROM stragg_test WHERE dept = 'HR'; +-- Expected: Alice,Bob,Carol + +-- Aggregation with GROUP BY, grouped by department +SELECT dept, sys.stragg(name ORDER BY name) +FROM stragg_test +GROUP BY dept +ORDER BY dept; +-- Expected: +-- HR | Alice,Bob,Carol +-- IT | Dave,Eve +---- + +=== NULL Value Handling + +[source,sql] +---- +-- Insert a row with a NULL value +INSERT INTO stragg_test VALUES ('HR', NULL); + +-- NULL is ignored; result is the same as without the NULL row +SELECT sys.stragg(name ORDER BY name) FROM stragg_test WHERE dept = 'HR'; +-- Expected: Alice,Bob,Carol + +-- All-NULL input returns NULL +SELECT sys.stragg(name) IS NULL FROM stragg_test WHERE name IS NULL; +-- Expected: t +---- + +=== Empty Set Handling + +[source,sql] +---- +-- No matching rows: returns NULL +SELECT sys.stragg(name) IS NULL FROM stragg_test WHERE dept = 'NONE'; +-- Expected: t +---- + +=== Single Value + +[source,sql] +---- +-- Only one value in the group: result contains no comma +SELECT sys.stragg(name) FROM stragg_test WHERE dept = 'IT' AND name = 'Dave'; +-- Expected: Dave +---- + +=== Usage in PG Compatibility Mode + +[source,sql] +---- +SET ivorysql.compatible_mode = pg; +SELECT sys.stragg(name ORDER BY name) FROM stragg_test WHERE dept = 'HR'; +-- Expected: Alice,Bob,Carol +RESET ivorysql.compatible_mode; +---- + +=== Test Environment Cleanup + +[source,sql] +---- +DROP TABLE stragg_test; +---- + +== Behavioral Differences from Oracle + +[cols="1,2,2"] +|=== +|Scenario |Oracle STRAGG |IvorySQL sys.stragg + +|Empty set +|`NULL` +|`NULL` (compatible) + +|All-NULL group +|`NULL` +|`NULL` (compatible) + +|NULL values +|Ignored +|Ignored (compatible) + +|Concatenation order +|Indeterminate +|Indeterminate; can be fixed with an `ORDER BY` clause + +|`ORDER BY` clause +|Not supported (no official syntax) +|Supported (PostgreSQL aggregate extension syntax) +|=== + +== Comparison with LISTAGG + +[cols="1,2,2"] +|=== +|Feature |`LISTAGG` (Oracle / IvorySQL) |`STRAGG` (Oracle / IvorySQL) + +|Separator +|Any delimiter can be specified +|Fixed comma + +|`ORDER BY` +|Specified via `WITHIN GROUP (ORDER BY ...)` +|Order not guaranteed; IvorySQL supports PostgreSQL aggregate `ORDER BY` + +|Result length limit +|Oracle enforces a 4000-byte limit (IvorySQL checks via `ora_listagg_check`) +|No limit + +|Typical use case +|When precise control of delimiter and order is required +|Quick concatenation; migration of legacy Oracle code +|=== diff --git a/EN/modules/ROOT/pages/master/7.20.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_sys_guid.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.20.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_sys_guid.adoc diff --git a/EN/modules/ROOT/pages/master/7.16.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/compat_type_rowtype.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.16.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/compat_type_rowtype.adoc diff --git a/EN/modules/ROOT/pages/master/7.13.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/invisible_column.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.13.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/invisible_column.adoc diff --git a/EN/modules/ROOT/pages/master/7.12.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/package.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.12.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/package.adoc diff --git a/EN/modules/ROOT/pages/master/7.9.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/port_ip.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.9.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/port_ip.adoc diff --git a/EN/modules/ROOT/pages/master/7.11.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/sequence.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.11.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/sequence.adoc diff --git a/EN/modules/ROOT/pages/master/oracle_compatibility/with_function_procedure_en.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/with_function_procedure_en.adoc new file mode 100644 index 0000000..3e5ad53 --- /dev/null +++ b/EN/modules/ROOT/pages/master/oracle_compatibility/with_function_procedure_en.adoc @@ -0,0 +1,345 @@ +:sectnums: +:sectnumlevels: 5 + += WITH FUNCTION/PROCEDURE + +== Purpose + +This document explains the `WITH FUNCTION` and `WITH PROCEDURE` features in IvorySQL, +which implement Oracle-style inline PL/SQL functions and procedures inside SQL statements. + +`WITH FUNCTION/PROCEDURE` is Oracle's _Subquery Factoring with PL/SQL Declarations_ feature. +It allows PL/SQL functions and procedures to be defined directly inside the WITH clause +(Common Table Expression, CTE) of a SQL statement. + +== Feature Description + +=== Basic Syntax + +In Oracle-compatible mode (`compatible_db = ORA_PARSER`), the WITH clause supports the +following extended syntax: + +[source,sql] +---- +WITH + FUNCTION func_name ( [ param_list ] ) RETURN return_type + { IS | AS } + [ declare_section ] + BEGIN + statements + END [ func_name ] ; + + PROCEDURE proc_name ( [ param_list ] ) + { IS | AS } + [ declare_section ] + BEGIN + statements + END [ proc_name ] ; + + cte_name AS ( SELECT ... ) +SELECT ... +---- + +=== Key Characteristics + +- **Scope-limited**: Functions/procedures defined in the WITH clause are visible only within + the current SQL statement. +- **Mixed usage**: Function/procedure definitions can be interleaved with standard CTEs + (`AS (SELECT ...)`). +- **Full PL/SQL syntax**: The function body supports the complete PL/SQL syntax (`BEGIN...END`). +- **No system catalog writes**: Definitions are automatically destroyed after execution; + they are not persisted to `pg_proc`. +- **SELECT context only**: Valid in SELECT statements and INSERT...SELECT statements. + Using WITH FUNCTION before UPDATE, DELETE, or MERGE raises an error (consistent with + Oracle behavior). + +=== Parameter Modes + +Inline procedures support the standard Oracle parameter modes: + +- `IN` (default): input parameter +- `OUT`: output parameter +- `IN OUT`: bidirectional parameter + +Inline functions only support `IN` parameters. + +== Supported Statement Types + +WITH inline functions/procedures are only allowed in the following top-level statements: + +- `SELECT` statements (most common) +- `INSERT ... SELECT` statements (Oracle-compatible form; WITH FUNCTION appears after + `INSERT INTO`) + +The following statements are **not supported** (Oracle does not allow them; +`ERRCODE_FEATURE_NOT_SUPPORTED` is raised): + +- `WITH FUNCTION ... UPDATE ...`: WITH FUNCTION cannot precede UPDATE +- `WITH FUNCTION ... DELETE ...`: WITH FUNCTION cannot precede DELETE +- `WITH FUNCTION ... MERGE ...`: WITH FUNCTION cannot precede MERGE + +To share reusable logic inside DML statements, define a schema-level function with +`CREATE FUNCTION` instead. + +== Syntax Examples + +=== Simplest Inline Function + +[source,sql] +---- +WITH + FUNCTION double_it(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n * 2; END; +SELECT double_it(5) FROM dual; +-- Expected output: 10 +---- + +=== Function Mixed with a CTE + +[source,sql] +---- +WITH + FUNCTION tax(amt NUMBER) RETURN NUMBER AS + BEGIN RETURN amt * 0.1; END; + orders AS (SELECT 100 AS amount) +SELECT amount, tax(amount) FROM orders; +-- Expected output: 100 | 10 +---- + +=== Multiple Inline Functions + +[source,sql] +---- +WITH + FUNCTION add1(n NUMBER) RETURN NUMBER AS BEGIN RETURN n+1; END; + FUNCTION mul2(n NUMBER) RETURN NUMBER AS BEGIN RETURN n*2; END; +SELECT mul2(add1(3)) FROM dual; +-- Expected output: 8 +---- + +=== Recursive Function + +[source,sql] +---- +WITH + FUNCTION factorial(n NUMBER) RETURN NUMBER AS + BEGIN + IF n <= 1 THEN RETURN 1; END IF; + RETURN n * factorial(n-1); + END; +SELECT factorial(5) FROM dual; +-- Expected output: 120 +---- + +=== OUT Parameters (PROCEDURE Only) + +WITH FUNCTION does not allow `OUT` or `IN OUT` parameters (consistent with Oracle +ORA-06572 behavior). Only WITH PROCEDURE may declare `OUT` / `IN OUT` parameters. + +[source,sql] +---- +-- Correct: PROCEDURE may declare OUT parameters +WITH + PROCEDURE swap(val IN NUMBER, result OUT NUMBER) AS + BEGIN + result := val * 10; + END; +SELECT 1 FROM dual; -- the procedure is called by another subprogram inside the same + -- WITH block, not directly from a SELECT expression + +-- Error: FUNCTION does not allow OUT parameters +WITH + FUNCTION bad_func(val NUMBER, result OUT NUMBER) RETURN NUMBER AS + BEGIN RETURN val; END; +SELECT bad_func(5) FROM dual; +-- Expected output: ERROR: WITH FUNCTION "bad_func" cannot declare OUT or IN OUT parameters +---- + +=== Default Parameter Values + +[source,sql] +---- +WITH + FUNCTION calc(n NUMBER DEFAULT 10) RETURN NUMBER AS + BEGIN RETURN n * 2; END; +SELECT calc() FROM dual; +-- Expected output: 20 +---- + +=== Exception Handling + +[source,sql] +---- +WITH + FUNCTION safe_div(a NUMBER, b NUMBER) RETURN NUMBER AS + BEGIN + RETURN a / b; + EXCEPTION + WHEN OTHERS THEN RETURN NULL; + END; +SELECT safe_div(1, 0) FROM dual; +-- Expected output: NULL +---- + +=== Integration with DML + +[source,sql] +---- +-- Allowed: INSERT INTO ... WITH FUNCTION ... SELECT ... (Oracle-compatible form) +WITH + FUNCTION get_bonus(sal NUMBER) RETURN NUMBER AS + BEGIN RETURN sal * 1.2; END; +INSERT INTO emp_bonus (empno, bonus) +SELECT empno, get_bonus(sal) FROM emp WHERE deptno = 10; +---- + +NOTE: Oracle does not allow WITH FUNCTION to precede UPDATE, DELETE, or MERGE. +IvorySQL enforces the same restriction; such usage raises `ERRCODE_FEATURE_NOT_SUPPORTED`. + +== Scope and Visibility + +=== Scope Rules + +- The scope of an inline function/procedure is limited to the SQL statement in which it + is defined (including subqueries within that statement). +- Functions/procedures may reference each other regardless of declaration order (mutual + recursion is supported). +- An inline definition shadows any existing database function with the same name and + signature (WITH definition takes precedence). +- Defining two functions/procedures with the same name and parameter types in a single + WITH clause is not allowed. + +=== Visibility Inside Subqueries + +[source,sql] +---- +WITH + FUNCTION add_tax(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n * 1.1; END; +SELECT * FROM (SELECT add_tax(amount) AS total FROM orders); +---- + +== Relationship with Existing Features + +|=== +| Existing Feature | Relationship + +| PL/iSQL nested subprograms +| Directly reused: leverages the existing compile/execute infrastructure. + +| Standard CTE (`WITH...AS (SELECT ...)`) +| Coexists: can be mixed in the same WITH clause. + +| `RECURSIVE` CTE +| Coexists: WITH RECURSIVE and inline functions can appear together. + +| Oracle Package +| Similar: package procedures/functions are also registered temporarily at session level. + +| PL/iSQL `CREATE FUNCTION` +| Different: WITH inline functions are not persisted and are not written to the system catalog. +|=== + +== Error Handling + +=== Duplicate Definition + +[source,sql] +---- +WITH + FUNCTION dup(n NUMBER) RETURN NUMBER AS BEGIN RETURN n; END; + FUNCTION dup(n NUMBER) RETURN NUMBER AS BEGIN RETURN n * 2; END; +SELECT dup(1) FROM dual; +-- Expected output: ERROR: WITH clause function "dup" is defined more than once with the same argument types +---- + +=== Usage in PG_PARSER Mode + +[source,sql] +---- +-- Attempting to use WITH FUNCTION syntax in PG_PARSER mode +SET compatible_db = PG_PARSER; +WITH FUNCTION foo(n NUMBER) RETURN NUMBER AS BEGIN RETURN n; END; +SELECT foo(1); +-- Expected output: ERROR: syntax error at or near "FUNCTION" +---- + +=== Function Body Syntax Error + +[source,sql] +---- +WITH + FUNCTION broken(n NUMBER) RETURN NUMBER AS + BEGIN + RETRUN n; -- typo + END; +SELECT broken(1) FROM dual; +-- Expected output: ERROR: syntax error at or near "RETRUN" +---- + +=== Rejection of Table Function Usage + +[source,sql] +---- +WITH + FUNCTION get_rows(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n; END; +SELECT * FROM get_rows(5); +-- Expected output: ERROR: WITH clause function cannot be used as a table or set-returning function +---- + +=== Rejection of Qualified Names + +[source,sql] +---- +WITH + FUNCTION public.qual_func(n NUMBER) RETURN NUMBER IS + BEGIN RETURN n; END; +SELECT qual_func(1) FROM dual; +-- Expected output: ERROR: qualified name is not allowed in WITH FUNCTION declaration +---- + +== EXPLAIN Output + +=== Basic Output + +[source,sql] +---- +EXPLAIN WITH + FUNCTION add_one(n NUMBER) RETURN NUMBER AS BEGIN RETURN n + 1; END; +SELECT add_one(5) FROM dual; +-- Expected output includes: WITH Function: add_one(number) RETURN number +---- + +=== VERBOSE Mode + +[source,sql] +---- +EXPLAIN (VERBOSE ON) WITH + FUNCTION double(n NUMBER) RETURN NUMBER AS + BEGIN RETURN n * 2; END; +SELECT double(3) FROM dual; +-- Expected output includes: Body: BEGIN RETURN n * 2; END +---- + +== Restrictions and Constraints + +1. **Oracle parser mode only**: This feature is active only when `compatible_db = ORA_PARSER`. +2. **No system catalog writes**: Inline functions/procedures are registered dynamically during + statement execution and deregistered immediately after; nothing is written to `pg_proc`. +3. **Transaction-safe**: Registration and deregistration are fully transparent to transactions; + no WAL is generated. +4. **Concurrency-safe**: Each concurrent session has its own independent inline + function/procedure registration context. +5. **Scope-limited**: An inline function cannot be called outside the statement that defines it. +6. **No polymorphic parameters**: Declaring parameters with types such as `ANYELEMENT` will + fail at call time. +7. **Not usable as a table function**: `SELECT * FROM with_func(...)` is rejected. + +== Cleanup + +[source,sql] +---- +-- The scope of WITH FUNCTION/PROCEDURE ends automatically with the statement. +-- No manual cleanup is required. +---- diff --git a/EN/modules/ROOT/pages/master/7.10.adoc b/EN/modules/ROOT/pages/master/oracle_compatibility/xml_functions.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/7.10.adoc rename to EN/modules/ROOT/pages/master/oracle_compatibility/xml_functions.adoc diff --git a/EN/modules/ROOT/pages/master/1.adoc b/EN/modules/ROOT/pages/master/release_notes.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/1.adoc rename to EN/modules/ROOT/pages/master/release_notes.adoc diff --git a/EN/modules/ROOT/pages/master/9.adoc b/EN/modules/ROOT/pages/master/tools_reference.adoc similarity index 100% rename from EN/modules/ROOT/pages/master/9.adoc rename to EN/modules/ROOT/pages/master/tools_reference.adoc