From d2837b7194969e834fffe4f13070796fb1462fda Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Tue, 7 Jul 2026 07:10:27 +0000 Subject: [PATCH 1/3] core: land xqlc catalog + analyzer on sqlc's AST Port xqlc's core catalog (SQLite-backed sql_* catalog) and its dialect-neutral query analyzer into internal/core, repointing the analyzer from xqlc's copy of the AST onto sqlc's internal/sql/ast so there is a single AST. No converter and no second AST package. A smoke test drives the analyzer with sqlc's own PostgreSQL parser to prove the repointed analyzer resolves columns, types, star expansion, and aliases end-to-end against internal/sql/ast. This is the first step of merging xqlc back into sqlc as the future analysis core; ClickHouse will be the first engine wired onto it. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XTGxNHW6v1S1YyC9FDSgrK --- go.mod | 11 +- go.sum | 49 +++- internal/core/analysis.go | 81 ++++++ internal/core/analyzer/analyzer.go | 152 ++++++++++ internal/core/analyzer/analyzer_test.go | 107 +++++++ internal/core/analyzer/expr.go | 364 ++++++++++++++++++++++++ internal/core/analyzer/projection.go | 117 ++++++++ internal/core/analyzer/scope.go | 162 +++++++++++ internal/core/attribute.go | 220 ++++++++++++++ internal/core/cast.go | 77 +++++ internal/core/cast_test.go | 53 ++++ internal/core/catalog.go | 79 +++++ internal/core/class.go | 42 +++ internal/core/constraint.go | 17 ++ internal/core/dialect.go | 49 ++++ internal/core/dialect_test.go | 44 +++ internal/core/namespace.go | 22 ++ internal/core/operator.go | 76 +++++ internal/core/operator_test.go | 48 ++++ internal/core/proc.go | 154 ++++++++++ internal/core/proc_test.go | 69 +++++ internal/core/schema.sql | 156 ++++++++++ internal/core/types.go | 137 +++++++++ 23 files changed, 2283 insertions(+), 3 deletions(-) create mode 100644 internal/core/analysis.go create mode 100644 internal/core/analyzer/analyzer.go create mode 100644 internal/core/analyzer/analyzer_test.go create mode 100644 internal/core/analyzer/expr.go create mode 100644 internal/core/analyzer/projection.go create mode 100644 internal/core/analyzer/scope.go create mode 100644 internal/core/attribute.go create mode 100644 internal/core/cast.go create mode 100644 internal/core/cast_test.go create mode 100644 internal/core/catalog.go create mode 100644 internal/core/class.go create mode 100644 internal/core/constraint.go create mode 100644 internal/core/dialect.go create mode 100644 internal/core/dialect_test.go create mode 100644 internal/core/namespace.go create mode 100644 internal/core/operator.go create mode 100644 internal/core/operator_test.go create mode 100644 internal/core/proc.go create mode 100644 internal/core/proc_test.go create mode 100644 internal/core/schema.sql create mode 100644 internal/core/types.go diff --git a/go.mod b/go.mod index c89cfa38f8..d8633c12c2 100644 --- a/go.mod +++ b/go.mod @@ -30,27 +30,36 @@ require ( google.golang.org/grpc v1.81.1 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.48.1 ) require ( cel.dev/expr v0.25.1 // indirect filippo.io/edwards25519 v1.2.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/kr/text v0.2.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-sqlite3-wasm/v2 v2.6.35302 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/ncruces/julianday v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect + modernc.org/libc v1.70.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index 017fc2647b..3298bf1540 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/cubicdaiya/gonp v1.0.4/go.mod h1:iWGuP/7+JVTn02OWhRemVbMmG1DOUnmrGTYY github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= @@ -27,8 +29,12 @@ github.com/google/cel-go v0.28.1 h1:YWIwi77J4xIsYUwAF/iIuS6haffzIHS8yWI8glSbLWM= github.com/google/cel-go v0.28.1/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -47,16 +53,22 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/ncruces/go-sqlite3 v0.34.4 h1:bp8jd1o2CMvjkrrp1lOtHDu1TdzExNWACt+piQeMwzg= github.com/ncruces/go-sqlite3 v0.34.4/go.mod h1:tOyhDWnzlrzflKIGw177imjuO4ZEbfOS1NJ67B1BanQ= github.com/ncruces/go-sqlite3-wasm/v2 v2.6.35302 h1:IZCiInPIp6OhOc1skMDGOBMwMQuE1TK+QqVe35vd/ro= github.com/ncruces/go-sqlite3-wasm/v2 v2.6.35302/go.mod h1:ELHF6yqC51E0DiitfabHXl/aKKouCihugbhNT5a+yEY= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/pganalyze/pg_query_go/v6 v6.2.2 h1:O0L6zMC226R82RF3X5n0Ki6HjytDsoAzuzp4ATVAHNo= github.com/pganalyze/pg_query_go/v6 v6.2.2/go.mod h1:Cn6+j4870kJz3iYNsb0VsNG04vpSWgEvBwc590J4qD0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/riza-io/grpc-go v0.2.0 h1:2HxQKFVE7VuYstcJ8zqpN84VnAoJ4dCL6YFhJewNcHQ= github.com/riza-io/grpc-go v0.2.0/go.mod h1:2bDvR9KkKC3KhtlSHfR3dAXjUMT86kg4UfWFyVGWqi8= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -104,16 +116,21 @@ go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09 go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= -golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 h1:tu/dtnW1o3wfaxCOjSLn5IRX4YDcJrtlpzYkhHhGaC4= @@ -130,3 +147,31 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= +modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.32.0 h1:hjG66bI/kqIPX1b2yT6fr/jt+QedtP2fqojG2VrFuVw= +modernc.org/ccgo/v4 v4.32.0/go.mod h1:6F08EBCx5uQc38kMGl+0Nm0oWczoo1c7cgpzEry7Uc0= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw= +modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.48.1 h1:S85iToyU6cgeojybE2XJlSbcsvcWkQ6qqNXJHtW5hWA= +modernc.org/sqlite v1.48.1/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/core/analysis.go b/internal/core/analysis.go new file mode 100644 index 0000000000..2cb61cbf53 --- /dev/null +++ b/internal/core/analysis.go @@ -0,0 +1,81 @@ +package core + +// Command identifies the kind of statement that produced a PrepareResult. +// Only the four DML statements that can have a prepare-able shape are +// emitted; DDL and TCL produce an empty result with Command == "". +type Command string + +const ( + CommandSelect Command = "SELECT" + CommandInsert Command = "INSERT" + CommandUpdate Command = "UPDATE" + CommandDelete Command = "DELETE" +) + +// PrepareResult describes the output of preparing a SQL statement. +type PrepareResult struct { + Command Command `json:"command,omitempty"` + Columns []Column `json:"columns"` + Parameters []Parameter `json:"parameters"` +} + +// ColumnSource identifies the table column a result column or bind +// parameter is sourced from. All fields are optional; the struct is +// emitted only when at least one is populated. +// +// Schema / Table / Column are the *origin* identifiers (pre-alias) — +// they correspond to sqlite's `sqlite3_column_origin_name` and mysql's +// `org_table` / `org_name`. TableAlias is the name the query used to +// refer to the table; it lets codegen distinguish `t1` from `t2` in +// `SELECT t1.x, t2.x FROM t t1 JOIN t t2 ...`. +type ColumnSource struct { + Schema string `json:"schema,omitempty"` + Table string `json:"table,omitempty"` + TableAlias string `json:"table_alias,omitempty"` + Column string `json:"column,omitempty"` +} + +// Column describes a single output column from a prepared statement. +// +// SourceClassOID and SourceAttributeOID are populated when the column +// is a direct reference to a table column (i.e. it appears in +// sql_attribute); they are zero for computed/derived expressions like +// aggregates or arithmetic. Source is the human-readable resolution of +// those OIDs and is populated under the same conditions. +// +// DeclType, TypeLength, TypeScale, IsPrimaryKey, IsUnique, and +// IsAutoIncrement come from the resolved source attribute and are zero +// for computed expressions. +type Column struct { + Name string `json:"name"` + DataType string `json:"data_type"` + TypeOID int64 `json:"type_oid,omitempty"` + NotNull bool `json:"not_null"` + SourceClassOID int64 `json:"source_class_oid,omitempty"` + SourceAttributeOID int64 `json:"source_attribute_oid,omitempty"` + Source *ColumnSource `json:"source,omitempty"` + DeclType string `json:"decl_type,omitempty"` + TypeLength int `json:"type_length,omitempty"` + TypeScale int `json:"type_scale,omitempty"` + IsPrimaryKey bool `json:"is_primary_key,omitempty"` + IsUnique bool `json:"is_unique,omitempty"` + IsAutoIncrement bool `json:"is_auto_increment,omitempty"` +} + +// Parameter describes a bind parameter in a prepared statement. +// +// Number is the 1-based position of the parameter as it appeared in the +// source ($1, $2, ...). Name is populated for named-parameter dialects +// or when a sqlc-style "-- name:" annotation gave the param a name; it +// is empty otherwise. DataType / TypeOID / NotNull come from the +// resolved usage site (an operator overload, function argument, etc.). +// Source identifies the column the parameter binds against (e.g. +// `users.age` for `WHERE age > $1`), when one can be inferred. +type Parameter struct { + Number int `json:"number"` + Name string `json:"name,omitempty"` + DataType string `json:"data_type,omitempty"` + TypeOID int64 `json:"type_oid,omitempty"` + NotNull bool `json:"not_null"` + Source *ColumnSource `json:"source,omitempty"` +} diff --git a/internal/core/analyzer/analyzer.go b/internal/core/analyzer/analyzer.go new file mode 100644 index 0000000000..39da95585a --- /dev/null +++ b/internal/core/analyzer/analyzer.go @@ -0,0 +1,152 @@ +// Package analyzer implements a dialect-neutral SQL query analyzer that +// resolves names, types, operators, and parameters by querying the +// catalog (core.Catalog). It produces a core.PrepareResult. +// +// Scope is intentionally narrow in this iteration: single-relation SELECT +// queries, simple WHERE / GROUP BY / projection. JOINs, subqueries, +// CTEs, set ops, and DML RETURNING are intended follow-ups. +package analyzer + +import ( + "fmt" + + "github.com/sqlc-dev/sqlc/internal/core" + "github.com/sqlc-dev/sqlc/internal/sql/ast" +) + +// Prepare walks a parsed statement and produces a PrepareResult by +// querying the catalog for relations, types, operators, and casts. +// stmt can be a *ast.RawStmt (typical parser output) or an unwrapped +// statement node. +func Prepare(cat *core.Catalog, stmt ast.Node) (core.PrepareResult, error) { + if rs, ok := stmt.(*ast.RawStmt); ok { + stmt = rs.Stmt + } + a := &analyzer{ + cat: cat, + params: map[int]*core.Parameter{}, + } + switch s := stmt.(type) { + case *ast.SelectStmt: + if err := a.analyzeSelect(s); err != nil { + return core.PrepareResult{}, err + } + a.command = core.CommandSelect + default: + return core.PrepareResult{}, fmt.Errorf("analyzer: unsupported statement %T", stmt) + } + return a.result(), nil +} + +type analyzer struct { + cat *core.Catalog + scope *scope + columns []core.Column + params map[int]*core.Parameter + command core.Command +} + +func (a *analyzer) result() core.PrepareResult { + return core.PrepareResult{ + Command: a.command, + Columns: a.columns, + Parameters: orderedParams(a.params), + } +} + +func orderedParams(m map[int]*core.Parameter) []core.Parameter { + if len(m) == 0 { + return nil + } + maxN := 0 + for n := range m { + if n > maxN { + maxN = n + } + } + out := make([]core.Parameter, 0, len(m)) + for i := 1; i <= maxN; i++ { + if p, ok := m[i]; ok { + out = append(out, *p) + } + } + return out +} + +func (a *analyzer) analyzeSelect(s *ast.SelectStmt) error { + sc, err := a.buildScope(s.FromClause) + if err != nil { + return err + } + a.scope = sc + + // Join ON conditions get typed against the (already-assembled) + // scope so they can reference columns from either side. + for _, item := range listItems(s.FromClause) { + if err := a.typeJoinConditions(item); err != nil { + return fmt.Errorf("join: %w", err) + } + } + + if s.WhereClause != nil { + if _, err := a.typeExpr(s.WhereClause); err != nil { + return fmt.Errorf("where: %w", err) + } + } + if items := listItems(s.GroupClause); items != nil { + for _, g := range items { + if _, err := a.typeExpr(g); err != nil { + return fmt.Errorf("group by: %w", err) + } + } + } + if s.HavingClause != nil { + if _, err := a.typeExpr(s.HavingClause); err != nil { + return fmt.Errorf("having: %w", err) + } + } + + targets := listItems(s.TargetList) + if targets == nil { + return fmt.Errorf("select: empty target list") + } + for _, t := range targets { + rt, ok := t.(*ast.ResTarget) + if !ok { + continue + } + if err := a.projectTarget(rt); err != nil { + return err + } + } + return nil +} + +func listItems(l *ast.List) []ast.Node { + if l == nil { + return nil + } + return l.Items +} + +// typeJoinConditions walks a FROM-list item and types every join's ON +// expression. USING clauses are skipped — the columns they reference +// already exist in scope, no expression to type. +func (a *analyzer) typeJoinConditions(item ast.Node) error { + je, ok := item.(*ast.JoinExpr) + if !ok { + return nil + } + if err := a.typeJoinConditions(je.Larg); err != nil { + return err + } + if err := a.typeJoinConditions(je.Rarg); err != nil { + return err + } + if je.Quals != nil { + if _, err := a.typeExpr(je.Quals); err != nil { + return fmt.Errorf("ON: %w", err) + } + } + return nil +} diff --git a/internal/core/analyzer/analyzer_test.go b/internal/core/analyzer/analyzer_test.go new file mode 100644 index 0000000000..b7300921b7 --- /dev/null +++ b/internal/core/analyzer/analyzer_test.go @@ -0,0 +1,107 @@ +package analyzer_test + +import ( + "strings" + "testing" + + "github.com/sqlc-dev/sqlc/internal/core" + "github.com/sqlc-dev/sqlc/internal/core/analyzer" + "github.com/sqlc-dev/sqlc/internal/engine/postgresql" +) + +// seedUsers builds a minimal catalog with a single "users" table so the +// analyzer has something to resolve against. It deliberately avoids any +// per-dialect seed: the point is to exercise the dialect-neutral +// analyzer directly on sqlc's ast. +func seedUsers(t *testing.T) *core.Catalog { + t.Helper() + cat, err := core.New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + + ns, err := cat.NamespaceOID("public") + if err != nil { + t.Fatal(err) + } + int4, err := cat.CreateType("int4", 4) + if err != nil { + t.Fatal(err) + } + text, err := cat.CreateType("text", -1) + if err != nil { + t.Fatal(err) + } + users, err := cat.CreateClass(ns, "users", "r") + if err != nil { + t.Fatal(err) + } + if err := cat.CreateAttribute(users, "id", int4, true, false, 1); err != nil { + t.Fatal(err) + } + if err := cat.CreateAttribute(users, "name", text, true, false, 2); err != nil { + t.Fatal(err) + } + return cat +} + +// prepare parses query with sqlc's own PostgreSQL parser — which emits +// exactly the internal/sql/ast the analyzer was repointed onto — and +// runs the analyzer against cat. +func prepare(t *testing.T, cat *core.Catalog, query string) core.PrepareResult { + t.Helper() + stmts, err := postgresql.NewParser().Parse(strings.NewReader(query)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(stmts) != 1 { + t.Fatalf("expected 1 stmt, got %d", len(stmts)) + } + res, err := analyzer.Prepare(cat, stmts[0].Raw) + if err != nil { + t.Fatalf("analyze: %v", err) + } + return res +} + +func TestPrepareSelectColumns(t *testing.T) { + cat := seedUsers(t) + res := prepare(t, cat, "SELECT id, name FROM users") + + if len(res.Columns) != 2 { + t.Fatalf("got %d cols, want 2: %+v", len(res.Columns), res.Columns) + } + if res.Columns[0].Name != "id" || res.Columns[0].DataType != "int4" || !res.Columns[0].NotNull { + t.Errorf("col 0: %+v", res.Columns[0]) + } + if res.Columns[1].Name != "name" || res.Columns[1].DataType != "text" || !res.Columns[1].NotNull { + t.Errorf("col 1: %+v", res.Columns[1]) + } + for i, c := range res.Columns { + if c.SourceClassOID == 0 || c.SourceAttributeOID == 0 { + t.Errorf("col %d %s missing source binding: %+v", i, c.Name, c) + } + } +} + +func TestPrepareSelectStar(t *testing.T) { + cat := seedUsers(t) + res := prepare(t, cat, "SELECT * FROM users") + + if len(res.Columns) != 2 { + t.Fatalf("got %d cols, want 2: %+v", len(res.Columns), res.Columns) + } + if res.Columns[0].Name != "id" || res.Columns[1].Name != "name" { + t.Errorf("got %q, %q; want id, name", res.Columns[0].Name, res.Columns[1].Name) + } +} + +func TestPrepareAlias(t *testing.T) { + cat := seedUsers(t) + res := prepare(t, cat, "SELECT id AS user_id FROM users u") + + if len(res.Columns) != 1 || res.Columns[0].Name != "user_id" { + t.Fatalf("got %+v", res.Columns) + } +} diff --git a/internal/core/analyzer/expr.go b/internal/core/analyzer/expr.go new file mode 100644 index 0000000000..fdf9739554 --- /dev/null +++ b/internal/core/analyzer/expr.go @@ -0,0 +1,364 @@ +package analyzer + +import ( + "fmt" + "strings" + + "github.com/sqlc-dev/sqlc/internal/core" + "github.com/sqlc-dev/sqlc/internal/sql/ast" +) + +// exprType is the analyzer's resolved type for an expression: the +// catalog type OID, nullability, and (when present) the source +// attribute for direct column refs. +type exprType struct { + typeOID int64 + nullable bool + sourceClassOID int64 + sourceAttributeOID int64 + // sourceTableAlias is the user-visible name of the relation the + // source attribute came from — i.e. the FROM-list alias, or the + // table name when no alias was given. Empty for computed + // expressions. + sourceTableAlias string +} + +// typeExpr recursively types an expression, side-effecting parameter +// inference along the way. Returns the expression's exprType. +func (a *analyzer) typeExpr(n ast.Node) (exprType, error) { + switch e := n.(type) { + case nil: + return exprType{}, nil + + case *ast.TODO: + // Parser emits TODO for fields it didn't fully translate (e.g. an + // absent WHERE clause shows up as TODO rather than nil). Treat as no-op. + return exprType{}, nil + + case *ast.A_Const: + return a.typeConst(e) + + case *ast.ColumnRef: + return a.typeColumnRef(e) + + case *ast.ParamRef: + return a.typeParamRef(e) + + case *ast.A_Expr: + return a.typeAExpr(e) + + case *ast.BoolExpr: + return a.typeBoolExpr(e) + + case *ast.FuncCall: + return a.typeFuncCall(e) + + case *ast.TypeCast: + return a.typeTypeCast(e) + + case *ast.NullTest: + // IS [NOT] NULL always returns boolean, regardless of operand type. + if _, err := a.typeExpr(e.Arg); err != nil { + return exprType{}, err + } + return a.boolType(false /* not nullable */) + } + return exprType{}, fmt.Errorf("typeExpr: unsupported %T", n) +} + +func (a *analyzer) typeConst(c *ast.A_Const) (exprType, error) { + switch v := c.Val.(type) { + case *ast.Integer: + _ = v + oid, err := a.cat.TypeOID("int4") + if err != nil { + return exprType{}, err + } + return exprType{typeOID: oid}, nil + case *ast.Float: + oid, err := a.cat.TypeOID("numeric") + if err != nil { + return exprType{}, err + } + return exprType{typeOID: oid}, nil + case *ast.String: + // PG bare string literals start as `unknown` but coerce contextually; + // for simplicity we type them as text and rely on casts later. + oid, err := a.cat.TypeOID("text") + if err != nil { + return exprType{}, err + } + return exprType{typeOID: oid}, nil + case *ast.Boolean: + return a.boolType(false) + case nil: + // NULL literal + return exprType{nullable: true}, nil + } + return exprType{}, fmt.Errorf("typeConst: unsupported %T", c.Val) +} + +func (a *analyzer) boolType(nullable bool) (exprType, error) { + oid, err := a.cat.TypeOID("bool") + if err != nil { + return exprType{}, err + } + return exprType{typeOID: oid, nullable: nullable}, nil +} + +func (a *analyzer) typeColumnRef(c *ast.ColumnRef) (exprType, error) { + parts := flattenFields(c.Fields) + if len(parts) == 0 { + return exprType{}, fmt.Errorf("column ref: empty") + } + relation := "" + column := parts[0] + if len(parts) >= 2 { + relation = parts[0] + column = parts[1] + } + rel, col, ok, err := a.scope.resolveColumn(relation, column) + if err != nil { + return exprType{}, err + } + if !ok { + if relation != "" { + return exprType{}, fmt.Errorf("unknown column %q.%q", relation, column) + } + return exprType{}, fmt.Errorf("unknown column %q", column) + } + return exprType{ + typeOID: col.typeOID, + nullable: !col.notNull, + sourceClassOID: rel.classOID, + sourceAttributeOID: col.attOID, + sourceTableAlias: rel.alias, + }, nil +} + +// flattenFields converts a ColumnRef.Fields list into a slice of strings, +// stopping at any *A_Star. +func flattenFields(fields *ast.List) []string { + if fields == nil { + return nil + } + out := make([]string, 0, len(fields.Items)) + for _, item := range fields.Items { + switch v := item.(type) { + case *ast.String: + out = append(out, v.Str) + case *ast.A_Star: + out = append(out, "*") + return out + } + } + return out +} + +func (a *analyzer) typeParamRef(p *ast.ParamRef) (exprType, error) { + cur, ok := a.params[p.Number] + if !ok { + cur = &core.Parameter{Number: p.Number} + a.params[p.Number] = cur + } + return exprType{typeOID: cur.TypeOID, nullable: !cur.NotNull}, nil +} + +// inferParam updates a previously-seen parameter's type from its +// usage context (typically the other side of a binary operator). +// Only fills in when not already set. +func (a *analyzer) inferParam(number int, t exprType) { + cur, ok := a.params[number] + if !ok { + cur = &core.Parameter{Number: number} + a.params[number] = cur + } + if cur.TypeOID == 0 && t.typeOID != 0 { + cur.TypeOID = t.typeOID + if name, err := a.cat.TypeName(t.typeOID); err == nil { + cur.DataType = name + } + cur.NotNull = !t.nullable + } + // Record the source column the parameter binds against (e.g. + // `WHERE age > $1` → users.age). Only set when not yet known so + // the first usage wins; subsequent appearances of the same param + // against a different column don't clobber it. + if cur.Source == nil && t.sourceAttributeOID != 0 { + ad, err := a.cat.LookupAttribute(t.sourceAttributeOID) + if err == nil { + cur.Source = &core.ColumnSource{ + Schema: ad.Schema, + Table: ad.Table, + TableAlias: t.sourceTableAlias, + Column: ad.Column, + } + } + } +} + +// typeAExpr handles binary and unary expressions. For now we only +// implement OP_OP (the standard binary/unary operator case); other +// kinds error out for visibility. +func (a *analyzer) typeAExpr(e *ast.A_Expr) (exprType, error) { + if e.Kind != ast.A_Expr_Kind_OP { + return exprType{}, fmt.Errorf("a_expr: unsupported kind %v", e.Kind) + } + opName := opNameFromList(e.Name) + if opName == "" { + return exprType{}, fmt.Errorf("a_expr: unnamed operator") + } + + leftT, err := a.typeExpr(e.Lexpr) + if err != nil { + return exprType{}, err + } + rightT, err := a.typeExpr(e.Rexpr) + if err != nil { + return exprType{}, err + } + + // Cross-infer parameter types from the non-param side. + if pr, ok := e.Lexpr.(*ast.ParamRef); ok && rightT.typeOID != 0 { + a.inferParam(pr.Number, rightT) + leftT = rightT + } + if pr, ok := e.Rexpr.(*ast.ParamRef); ok && leftT.typeOID != 0 { + a.inferParam(pr.Number, leftT) + rightT = leftT + } + + overload, err := a.resolveOperator(opName, leftT.typeOID, rightT.typeOID) + if err != nil { + return exprType{}, err + } + return exprType{ + typeOID: overload.ResultTypeOID, + nullable: leftT.nullable || rightT.nullable, + }, nil +} + +func opNameFromList(l *ast.List) string { + if l == nil { + return "" + } + parts := make([]string, 0, len(l.Items)) + for _, item := range l.Items { + if s, ok := item.(*ast.String); ok { + parts = append(parts, s.Str) + } + } + return strings.Join(parts, ".") +} + +// resolveOperator finds an operator overload, attempting implicit casts +// on either side if no exact match exists. +func (a *analyzer) resolveOperator(name string, leftOID, rightOID int64) (core.OperatorOverload, error) { + candidates, err := a.cat.FindOperators(name, leftOID, rightOID) + if err != nil { + return core.OperatorOverload{}, err + } + if len(candidates) > 0 { + return candidates[0], nil + } + + // Try implicit-cast both sides: enumerate all overloads of this name + // and pick the first whose operand types are reachable via implicit + // casts from our actual operand types. + all, err := a.cat.FindOperators(name, 0, 0) + if err != nil { + return core.OperatorOverload{}, err + } + for _, ov := range all { + if leftOID != 0 && ov.LeftTypeOID != 0 && leftOID != ov.LeftTypeOID { + ok, _ := a.cat.CastAllowed(leftOID, ov.LeftTypeOID, "i") + if !ok { + continue + } + } + if rightOID != 0 && ov.RightTypeOID != 0 && rightOID != ov.RightTypeOID { + ok, _ := a.cat.CastAllowed(rightOID, ov.RightTypeOID, "i") + if !ok { + continue + } + } + // Reject candidates with mismatched arity + if (leftOID == 0) != (ov.LeftTypeOID == 0) { + continue + } + if (rightOID == 0) != (ov.RightTypeOID == 0) { + continue + } + return ov, nil + } + return core.OperatorOverload{}, fmt.Errorf("no operator %q for (%d, %d)", name, leftOID, rightOID) +} + +func (a *analyzer) typeBoolExpr(b *ast.BoolExpr) (exprType, error) { + for _, item := range listItems(b.Args) { + if _, err := a.typeExpr(item); err != nil { + return exprType{}, err + } + } + return a.boolType(false) +} + +func (a *analyzer) typeFuncCall(f *ast.FuncCall) (exprType, error) { + // Function name comes through as a List of *String parts (qualified name) + // in either f.Funcname or f.Func. + name := funcCallName(f) + if name == "" { + return exprType{}, fmt.Errorf("func call: missing name") + } + + // COUNT(*) always returns bigint and is non-null. + if f.AggStar && (name == "count" || name == "count.*") { + oid, err := a.cat.TypeOID("int8") + if err != nil { + return exprType{}, err + } + return exprType{typeOID: oid, nullable: false}, nil + } + + // Type the args (we don't yet resolve overloads against argument types + // — we just take the first overload by name and trust the catalog). + for _, arg := range listItems(f.Args) { + if _, err := a.typeExpr(arg); err != nil { + return exprType{}, err + } + } + + overloads, err := a.cat.FindProcs(name, nil) + if err != nil { + return exprType{}, err + } + if len(overloads) == 0 { + return exprType{}, fmt.Errorf("unknown function %q", name) + } + p := overloads[0] + return exprType{typeOID: p.ReturnTypeOID, nullable: p.ReturnNullable}, nil +} + +func funcCallName(f *ast.FuncCall) string { + if f.Funcname != nil { + return opNameFromList(f.Funcname) + } + if f.Func != nil { + return strings.ToLower(f.Func.Name) + } + return "" +} + +func (a *analyzer) typeTypeCast(c *ast.TypeCast) (exprType, error) { + if c.TypeName == nil { + return exprType{}, fmt.Errorf("cast: missing target type") + } + if _, err := a.typeExpr(c.Arg); err != nil { + return exprType{}, err + } + oid, err := a.cat.TypeOID(strings.ToLower(c.TypeName.Name)) + if err != nil { + return exprType{}, fmt.Errorf("cast target %q: %w", c.TypeName.Name, err) + } + return exprType{typeOID: oid}, nil +} diff --git a/internal/core/analyzer/projection.go b/internal/core/analyzer/projection.go new file mode 100644 index 0000000000..067858111e --- /dev/null +++ b/internal/core/analyzer/projection.go @@ -0,0 +1,117 @@ +package analyzer + +import ( + "github.com/sqlc-dev/sqlc/internal/core" + "github.com/sqlc-dev/sqlc/internal/sql/ast" +) + +// projectTarget evaluates one entry in the SELECT list and records an +// output Column. Star expansions emit one column per source column. +func (a *analyzer) projectTarget(rt *ast.ResTarget) error { + if cr, ok := rt.Val.(*ast.ColumnRef); ok { + // "* " or "t.*" expansion + if isStarRef(cr) { + a.emitStar(cr) + return nil + } + } + + t, err := a.typeExpr(rt.Val) + if err != nil { + return err + } + col := core.Column{ + Name: targetName(rt), + TypeOID: t.typeOID, + NotNull: !t.nullable, + SourceClassOID: t.sourceClassOID, + SourceAttributeOID: t.sourceAttributeOID, + } + if t.typeOID != 0 { + if name, err := a.cat.TypeName(t.typeOID); err == nil { + col.DataType = name + } + } + a.decorateSource(&col, t.sourceAttributeOID, t.sourceTableAlias) + a.columns = append(a.columns, col) + return nil +} + +// decorateSource fills in the resolved Source struct and per-column +// flag fields for a Column whose value came from a real table column. +// Computed expressions (zero attOID) are left untouched. +func (a *analyzer) decorateSource(col *core.Column, attOID int64, tableAlias string) { + if attOID == 0 { + return + } + ad, err := a.cat.LookupAttribute(attOID) + if err != nil { + return + } + col.Source = &core.ColumnSource{ + Schema: ad.Schema, + Table: ad.Table, + TableAlias: tableAlias, + Column: ad.Column, + } + col.DeclType = ad.DeclType + col.TypeLength = ad.TypeLength + col.TypeScale = ad.TypeScale + col.IsPrimaryKey = ad.IsPrimaryKey + col.IsUnique = ad.IsUnique + col.IsAutoIncrement = ad.AutoIncrement +} + +// targetName picks the user-visible name for a SELECT-list entry: the +// alias if one was given, the source column name for direct refs, or +// the conventional "?column?" placeholder for computed expressions. +func targetName(rt *ast.ResTarget) string { + if rt.Name != nil && *rt.Name != "" { + return *rt.Name + } + if cr, ok := rt.Val.(*ast.ColumnRef); ok { + parts := flattenFields(cr.Fields) + if len(parts) > 0 { + return parts[len(parts)-1] + } + } + if fc, ok := rt.Val.(*ast.FuncCall); ok { + if name := funcCallName(fc); name != "" { + return name + } + } + return "?column?" +} + +func isStarRef(c *ast.ColumnRef) bool { + parts := flattenFields(c.Fields) + return len(parts) > 0 && parts[len(parts)-1] == "*" +} + +// emitStar expands * (or t.*) into one Column per source attribute. +func (a *analyzer) emitStar(cr *ast.ColumnRef) { + parts := flattenFields(cr.Fields) + relName := "" + if len(parts) > 1 { + relName = parts[0] + } + for _, rel := range a.scope.rels { + if relName != "" && rel.alias != relName { + continue + } + for _, c := range rel.cols { + col := core.Column{ + Name: c.name, + TypeOID: c.typeOID, + NotNull: c.notNull, + SourceClassOID: rel.classOID, + SourceAttributeOID: c.attOID, + } + if name, err := a.cat.TypeName(c.typeOID); err == nil { + col.DataType = name + } + a.decorateSource(&col, c.attOID, rel.alias) + a.columns = append(a.columns, col) + } + } +} diff --git a/internal/core/analyzer/scope.go b/internal/core/analyzer/scope.go new file mode 100644 index 0000000000..3d688228a4 --- /dev/null +++ b/internal/core/analyzer/scope.go @@ -0,0 +1,162 @@ +package analyzer + +import ( + "fmt" + + "github.com/sqlc-dev/sqlc/internal/core" + "github.com/sqlc-dev/sqlc/internal/sql/ast" +) + +// scope is the set of relations available to expression resolution at a +// given point in a query. For now it holds at most one parent (FROM +// list) — subquery / CTE / lateral parents are a follow-up. +type scope struct { + rels []scopeRel +} + +// scopeRel binds a FROM-list relation (table or aliased table) to the +// columns it contributes. +type scopeRel struct { + alias string // user-visible name for resolution; defaults to relation name + classOID int64 + cols []scopeCol +} + +// scopeCol is a single column visible through a scopeRel. +type scopeCol struct { + name string + attOID int64 + typeOID int64 + notNull bool +} + +// buildScope walks a FROM clause (currently: RangeVar plus JoinExpr +// trees, which we flatten into the rels slice) and produces a flat +// scope. Subqueries and table-valued functions are still TODO. +func (a *analyzer) buildScope(from *ast.List) (*scope, error) { + sc := &scope{} + for _, item := range listItems(from) { + if err := a.appendFromItem(sc, item); err != nil { + return nil, err + } + } + return sc, nil +} + +func (a *analyzer) appendFromItem(sc *scope, item ast.Node) error { + switch v := item.(type) { + case *ast.RangeVar: + rel, err := a.bindRangeVar(v) + if err != nil { + return err + } + sc.rels = append(sc.rels, rel) + return nil + case *ast.JoinExpr: + // Flatten both sides into the same scope. The join condition + // (Quals / USING) is type-checked against the assembled scope + // in analyzeSelect after buildScope returns; we don't enforce + // it here. Outer-join nullability is also a follow-up. + if err := a.appendFromItem(sc, v.Larg); err != nil { + return err + } + return a.appendFromItem(sc, v.Rarg) + default: + return fmt.Errorf("scope: unsupported FROM item %T", item) + } +} + +func (a *analyzer) bindRangeVar(rv *ast.RangeVar) (scopeRel, error) { + if rv.Relname == nil { + return scopeRel{}, fmt.Errorf("range var: missing relation name") + } + relName := *rv.Relname + schema := "" + if rv.Schemaname != nil { + schema = *rv.Schemaname + } + if schema == "" { + schema = "public" + } + nsOID, err := a.cat.NamespaceOID(schema) + if err != nil { + return scopeRel{}, fmt.Errorf("schema %q: %w", schema, err) + } + classOID, err := a.cat.ClassOID(nsOID, relName) + if err != nil { + return scopeRel{}, fmt.Errorf("relation %q.%q: %w", schema, relName, err) + } + rel := scopeRel{ + alias: relName, + classOID: classOID, + } + if rv.Alias != nil && rv.Alias.Aliasname != nil && *rv.Alias.Aliasname != "" { + rel.alias = *rv.Alias.Aliasname + } + + cols, err := a.classColumns(classOID) + if err != nil { + return scopeRel{}, err + } + rel.cols = cols + return rel, nil +} + +// classColumns reads the column layout of a relation directly from +// sql_attribute. We can't go through TableColumns because that resolves +// by name and doesn't return attribute OIDs. +func (a *analyzer) classColumns(classOID int64) ([]scopeCol, error) { + rows, err := a.cat.DB().Query( + `SELECT a.oid, a.name, a.type_oid, a.not_null + FROM sql_attribute a WHERE a.class_oid = ? ORDER BY a.num`, + classOID, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var out []scopeCol + for rows.Next() { + var c scopeCol + var nn int + if err := rows.Scan(&c.attOID, &c.name, &c.typeOID, &nn); err != nil { + return nil, err + } + c.notNull = nn != 0 + out = append(out, c) + } + return out, rows.Err() +} + +// resolveColumn locates a (relation, column) pair in the scope. +// relation may be empty for an unqualified reference; in that case we +// search every relation and error on ambiguity. +func (s *scope) resolveColumn(relation, column string) (rel scopeRel, col scopeCol, ok bool, err error) { + var matches []struct { + rel scopeRel + col scopeCol + } + for _, r := range s.rels { + if relation != "" && r.alias != relation { + continue + } + for _, c := range r.cols { + if c.name == column { + matches = append(matches, struct { + rel scopeRel + col scopeCol + }{r, c}) + } + } + } + if len(matches) == 0 { + return scopeRel{}, scopeCol{}, false, nil + } + if len(matches) > 1 { + return scopeRel{}, scopeCol{}, false, fmt.Errorf("ambiguous column reference %q", column) + } + return matches[0].rel, matches[0].col, true, nil +} + +// silence unused-import warning if core not referenced here directly +var _ = core.Column{} diff --git a/internal/core/attribute.go b/internal/core/attribute.go new file mode 100644 index 0000000000..95e6cc7be4 --- /dev/null +++ b/internal/core/attribute.go @@ -0,0 +1,220 @@ +package core + +import ( + "database/sql" + "fmt" +) + +// AttributeSpec carries the data needed to register a column on a +// relation. It is the full-fidelity counterpart to CreateAttribute, +// which is kept as a thin wrapper for callers that don't need to set +// any of the "extra" metadata. +type AttributeSpec struct { + ClassOID int64 + Name string + TypeOID int64 + Num int // 1-based ordinal position + NotNull bool + HasDefault bool + DeclType string // original declared type, before normalization + TypeLength int // varchar(N), numeric(p,s).p, etc. + TypeScale int // numeric(p,s).s + AutoIncrement bool + IsPrimaryKey bool + IsUnique bool +} + +// CreateAttributeSpec inserts a column with the full set of attribute +// metadata. +func (c *Catalog) CreateAttributeSpec(s AttributeSpec) error { + _, err := c.db.Exec(` + INSERT INTO sql_attribute ( + class_oid, name, type_oid, not_null, has_default, num, + decl_type, type_length, type_scale, + auto_increment, is_primary_key, is_unique + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + s.ClassOID, s.Name, s.TypeOID, + boolToInt(s.NotNull), boolToInt(s.HasDefault), s.Num, + s.DeclType, s.TypeLength, s.TypeScale, + boolToInt(s.AutoIncrement), + boolToInt(s.IsPrimaryKey), + boolToInt(s.IsUnique), + ) + if err != nil { + return fmt.Errorf("create attribute %q on class %d: %w", s.Name, s.ClassOID, err) + } + return nil +} + +// CreateAttribute inserts a column for the given relation. Wraps +// CreateAttributeSpec for callers that only set the basics. +func (c *Catalog) CreateAttribute(classOID int64, name string, typeOID int64, notNull bool, hasDefault bool, num int) error { + return c.CreateAttributeSpec(AttributeSpec{ + ClassOID: classOID, + Name: name, + TypeOID: typeOID, + Num: num, + NotNull: notNull, + HasDefault: hasDefault, + }) +} + +// SetAttributePrimaryKey flips the is_primary_key (and not_null) flag +// on every named column of a relation. Used when a table-level PRIMARY +// KEY constraint is processed after the columns have been registered. +func (c *Catalog) SetAttributePrimaryKey(classOID int64, columns []string) error { + for _, name := range columns { + _, err := c.db.Exec( + `UPDATE sql_attribute SET is_primary_key = 1, not_null = 1 + WHERE class_oid = ? AND name = ?`, + classOID, name, + ) + if err != nil { + return fmt.Errorf("mark pk %s on class %d: %w", name, classOID, err) + } + } + return nil +} + +// SetAttributeUnique flips is_unique on every named column. Multi-column +// UNIQUE constraints don't make individual columns unique, so callers +// should pass length-1 column lists only. +func (c *Catalog) SetAttributeUnique(classOID int64, columns []string) error { + for _, name := range columns { + _, err := c.db.Exec( + `UPDATE sql_attribute SET is_unique = 1 WHERE class_oid = ? AND name = ?`, + classOID, name, + ) + if err != nil { + return fmt.Errorf("mark unique %s on class %d: %w", name, classOID, err) + } + } + return nil +} + +// ColumnInfo describes a resolved column. +type ColumnInfo struct { + Name string + TypeName string + TypeOID int64 + NotNull bool + DeclType string + TypeLength int + TypeScale int + AutoIncrement bool + IsPrimaryKey bool + IsUnique bool + AttributeOID int64 + ClassOID int64 +} + +// ResolveColumn looks up a column by table name and column name. +func (c *Catalog) ResolveColumn(table, column string) (*ColumnInfo, error) { + var info ColumnInfo + var ai, ipk, iu int + err := c.db.QueryRow(` + SELECT a.oid, a.class_oid, a.name, t.name, a.type_oid, a.not_null, + a.decl_type, a.type_length, a.type_scale, + a.auto_increment, a.is_primary_key, a.is_unique + FROM sql_attribute a + JOIN sql_class c ON c.oid = a.class_oid + JOIN sql_type t ON t.oid = a.type_oid + WHERE c.name = ? AND a.name = ? + `, table, column).Scan( + &info.AttributeOID, &info.ClassOID, &info.Name, &info.TypeName, &info.TypeOID, &info.NotNull, + &info.DeclType, &info.TypeLength, &info.TypeScale, + &ai, &ipk, &iu, + ) + if err != nil { + return nil, fmt.Errorf("resolve %s.%s: %w", table, column, err) + } + info.AutoIncrement = ai != 0 + info.IsPrimaryKey = ipk != 0 + info.IsUnique = iu != 0 + return &info, nil +} + +// TableColumns returns all columns for a given table name, ordered by position. +func (c *Catalog) TableColumns(table string) ([]ColumnInfo, error) { + rows, err := c.db.Query(` + SELECT a.oid, a.class_oid, a.name, t.name, a.type_oid, a.not_null, + a.decl_type, a.type_length, a.type_scale, + a.auto_increment, a.is_primary_key, a.is_unique + FROM sql_attribute a + JOIN sql_class c ON c.oid = a.class_oid + JOIN sql_type t ON t.oid = a.type_oid + WHERE c.name = ? + ORDER BY a.num + `, table) + if err != nil { + return nil, fmt.Errorf("table columns %q: %w", table, err) + } + defer rows.Close() + + var cols []ColumnInfo + for rows.Next() { + var ci ColumnInfo + var ai, ipk, iu int + if err := rows.Scan( + &ci.AttributeOID, &ci.ClassOID, &ci.Name, &ci.TypeName, &ci.TypeOID, &ci.NotNull, + &ci.DeclType, &ci.TypeLength, &ci.TypeScale, + &ai, &ipk, &iu, + ); err != nil { + return nil, err + } + ci.AutoIncrement = ai != 0 + ci.IsPrimaryKey = ipk != 0 + ci.IsUnique = iu != 0 + cols = append(cols, ci) + } + return cols, rows.Err() +} + +// AttributeDetails describes a column resolved by its OID. Populated +// from sql_attribute joined back to sql_class / sql_namespace, so the +// caller doesn't have to re-resolve names from raw OIDs. +type AttributeDetails struct { + Schema string + Table string + Column string + Num int + DeclType string + TypeLength int + TypeScale int + AutoIncrement bool + IsPrimaryKey bool + IsUnique bool + NotNull bool +} + +// LookupAttribute returns the full source-side metadata for a column, +// keyed by attribute OID. Used by analyzers to populate Column.Source +// (and the per-column flag fields) once they've resolved an +// expression to a sql_attribute row. +func (c *Catalog) LookupAttribute(attOID int64) (AttributeDetails, error) { + var ad AttributeDetails + var schema sql.NullString + var ai, ipk, iu, nn int + err := c.db.QueryRow(` + SELECT ns.name, cls.name, a.name, a.num, + a.decl_type, a.type_length, a.type_scale, + a.auto_increment, a.is_primary_key, a.is_unique, a.not_null + FROM sql_attribute a + JOIN sql_class cls ON cls.oid = a.class_oid + JOIN sql_namespace ns ON ns.oid = cls.namespace_oid + WHERE a.oid = ? + `, attOID).Scan( + &schema, &ad.Table, &ad.Column, &ad.Num, + &ad.DeclType, &ad.TypeLength, &ad.TypeScale, + &ai, &ipk, &iu, &nn, + ) + if err != nil { + return ad, fmt.Errorf("lookup attribute %d: %w", attOID, err) + } + ad.Schema = schema.String + ad.AutoIncrement = ai != 0 + ad.IsPrimaryKey = ipk != 0 + ad.IsUnique = iu != 0 + ad.NotNull = nn != 0 + return ad, nil +} diff --git a/internal/core/cast.go b/internal/core/cast.go new file mode 100644 index 0000000000..96f02f839c --- /dev/null +++ b/internal/core/cast.go @@ -0,0 +1,77 @@ +package core + +import "fmt" + +// CastSpec describes a type-coercion rule. +// +// Context: +// - 'i' implicit — applied automatically by the analyzer +// - 'a' assignment — applied for INSERT/UPDATE assignments +// - 'e' explicit — only when the user writes CAST or :: +// +// ProcOID == 0 means binary-coercible (no function call required). +type CastSpec struct { + SourceTypeOID int64 + TargetTypeOID int64 + ProcOID int64 + Context string // 'i' | 'a' | 'e'; default 'e' + DialectOID int64 +} + +// CreateCast inserts a cast rule. (source, target) is the primary key, so +// re-inserting the same pair will fail; use ReplaceCast to overwrite. +func (c *Catalog) CreateCast(cs CastSpec) error { + if cs.Context == "" { + cs.Context = "e" + } + _, err := c.db.Exec( + `INSERT INTO sql_cast (source_type_oid, target_type_oid, proc_oid, context, dialect_oid) + VALUES (?, ?, ?, ?, ?)`, + cs.SourceTypeOID, cs.TargetTypeOID, + nullableOID(cs.ProcOID), cs.Context, nullableOID(cs.DialectOID), + ) + if err != nil { + return fmt.Errorf("create cast %d->%d: %w", cs.SourceTypeOID, cs.TargetTypeOID, err) + } + return nil +} + +// FindCast returns the cast rule from src to tgt, if one exists, plus +// whether it was found. +func (c *Catalog) FindCast(src, tgt int64) (CastSpec, bool, error) { + var cs CastSpec + var procOID, dialectOID int64 + err := c.db.QueryRow( + `SELECT source_type_oid, target_type_oid, + COALESCE(proc_oid, 0), context, COALESCE(dialect_oid, 0) + FROM sql_cast WHERE source_type_oid = ? AND target_type_oid = ?`, + src, tgt, + ).Scan(&cs.SourceTypeOID, &cs.TargetTypeOID, &procOID, &cs.Context, &dialectOID) + if err != nil { + return cs, false, nil + } + cs.ProcOID = procOID + cs.DialectOID = dialectOID + return cs, true, nil +} + +// CastAllowed reports whether src can be coerced to tgt in the given context. +// Same-type pairs always succeed. +func (c *Catalog) CastAllowed(src, tgt int64, ctx string) (bool, error) { + if src == tgt { + return true, nil + } + cs, ok, err := c.FindCast(src, tgt) + if err != nil || !ok { + return false, err + } + switch ctx { + case "i": + return cs.Context == "i", nil + case "a": + return cs.Context == "i" || cs.Context == "a", nil + case "e": + return true, nil + } + return false, fmt.Errorf("unknown cast context %q", ctx) +} diff --git a/internal/core/cast_test.go b/internal/core/cast_test.go new file mode 100644 index 0000000000..08c68b91c1 --- /dev/null +++ b/internal/core/cast_test.go @@ -0,0 +1,53 @@ +package core + +import "testing" + +func TestCastAllowed(t *testing.T) { + cat, err := New() + if err != nil { + t.Fatal(err) + } + defer cat.Close() + + intOID, _ := cat.CreateType("integer", 4) + bigintOID, _ := cat.CreateType("bigint", 8) + textOID, _ := cat.CreateType("text", 0) + + // integer -> bigint is implicit + if err := cat.CreateCast(CastSpec{ + SourceTypeOID: intOID, TargetTypeOID: bigintOID, Context: "i", + }); err != nil { + t.Fatal(err) + } + // integer -> text is explicit only + if err := cat.CreateCast(CastSpec{ + SourceTypeOID: intOID, TargetTypeOID: textOID, Context: "e", + }); err != nil { + t.Fatal(err) + } + + cases := []struct { + src, tgt int64 + ctx string + want bool + }{ + {intOID, intOID, "i", true}, // identity always OK + {intOID, bigintOID, "i", true}, // declared implicit + {intOID, bigintOID, "a", true}, // implicit subsumes assignment + {intOID, bigintOID, "e", true}, // implicit subsumes explicit + {intOID, textOID, "i", false}, // explicit-only blocked from implicit + {intOID, textOID, "a", false}, // and from assignment + {intOID, textOID, "e", true}, // but works for explicit + {textOID, intOID, "e", false}, // no rule defined + } + for _, c := range cases { + got, err := cat.CastAllowed(c.src, c.tgt, c.ctx) + if err != nil { + t.Errorf("CastAllowed(%d,%d,%q): %v", c.src, c.tgt, c.ctx, err) + continue + } + if got != c.want { + t.Errorf("CastAllowed(%d,%d,%q): got %v, want %v", c.src, c.tgt, c.ctx, got, c.want) + } + } +} diff --git a/internal/core/catalog.go b/internal/core/catalog.go new file mode 100644 index 0000000000..ed3a015a72 --- /dev/null +++ b/internal/core/catalog.go @@ -0,0 +1,79 @@ +// Package core provides a SQL catalog backed by an in-memory SQLite database. +// +// The catalog stores schema metadata (namespaces, types, tables, columns, +// constraints) in tables prefixed with sql_. Engine packages populate the +// catalog from DDL and then query it to resolve types during statement +// preparation. +package core + +import ( + "database/sql" + _ "embed" + "fmt" + + _ "modernc.org/sqlite" +) + +//go:embed schema.sql +var ddl string + +// Catalog is an in-memory SQLite database that stores schema metadata in +// sql_* tables modeled after PostgreSQL's system catalogs. +type Catalog struct { + db *sql.DB +} + +// Option configures a Catalog at creation time. The most common use is +// to seed dialect-specific built-ins; see WithSeed. +type Option func(*Catalog) error + +// WithSeed runs a seed function (typically dialect-provided) against the +// freshly bootstrapped catalog. Failures abort Catalog creation. +func WithSeed(fn func(*Catalog) error) Option { + return func(c *Catalog) error { return fn(c) } +} + +// New creates a new in-memory catalog and initializes the sql_* tables. +// Each Option runs after schema initialization and the default-namespace +// bootstrap; an Option that fails aborts catalog creation. +func New(opts ...Option) (*Catalog, error) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + return nil, fmt.Errorf("core: open catalog: %w", err) + } + if _, err := db.Exec(ddl); err != nil { + db.Close() + return nil, fmt.Errorf("core: init schema: %w", err) + } + c := &Catalog{db: db} + if err := c.bootstrap(); err != nil { + db.Close() + return nil, fmt.Errorf("core: bootstrap: %w", err) + } + for i, opt := range opts { + if err := opt(c); err != nil { + db.Close() + return nil, fmt.Errorf("core: option %d: %w", i, err) + } + } + return c, nil +} + +// Close closes the underlying database connection. +func (c *Catalog) Close() error { + return c.db.Close() +} + +// DB returns the underlying *sql.DB for advanced queries. +func (c *Catalog) DB() *sql.DB { + return c.db +} + +// bootstrap seeds the catalog with the default "public" namespace. +// Built-in types and functions are added by per-dialect seed files, +// not here, so that nothing in the catalog is hardcoded to a single +// dialect's view of the world. +func (c *Catalog) bootstrap() error { + _, err := c.CreateNamespace("public") + return err +} diff --git a/internal/core/class.go b/internal/core/class.go new file mode 100644 index 0000000000..5aad1b21f0 --- /dev/null +++ b/internal/core/class.go @@ -0,0 +1,42 @@ +package core + +import "fmt" + +// CreateClass inserts a relation (table, view, etc.) and returns its OID. +// Kind should be 'r' (table), 'v' (view), or 'i' (index). +func (c *Catalog) CreateClass(namespaceOID int64, name string, kind string) (int64, error) { + res, err := c.db.Exec( + `INSERT INTO sql_class (namespace_oid, name, kind) VALUES (?, ?, ?)`, + namespaceOID, name, kind, + ) + if err != nil { + return 0, fmt.Errorf("create class %q: %w", name, err) + } + return res.LastInsertId() +} + +// ClassOID returns the OID for the given relation in the given namespace. +func (c *Catalog) ClassOID(namespaceOID int64, name string) (int64, error) { + var oid int64 + err := c.db.QueryRow( + `SELECT oid FROM sql_class WHERE namespace_oid = ? AND name = ?`, + namespaceOID, name, + ).Scan(&oid) + if err != nil { + return 0, fmt.Errorf("class %q: %w", name, err) + } + return oid, nil +} + +// ClassOIDByName looks up a relation by name across all namespaces. +// If multiple matches exist, it returns the first found. +func (c *Catalog) ClassOIDByName(name string) (int64, error) { + var oid int64 + err := c.db.QueryRow( + `SELECT oid FROM sql_class WHERE name = ? LIMIT 1`, name, + ).Scan(&oid) + if err != nil { + return 0, fmt.Errorf("class %q: %w", name, err) + } + return oid, nil +} diff --git a/internal/core/constraint.go b/internal/core/constraint.go new file mode 100644 index 0000000000..7f87479623 --- /dev/null +++ b/internal/core/constraint.go @@ -0,0 +1,17 @@ +package core + +import "fmt" + +// CreateConstraint inserts a constraint for the given relation. +// Kind should be 'p' (primary key), 'f' (foreign key), 'u' (unique), or 'c' (check). +// columns is a comma-separated list of attribute ordinal positions. +func (c *Catalog) CreateConstraint(classOID int64, name string, kind string, columns string) error { + _, err := c.db.Exec( + `INSERT INTO sql_constraint (class_oid, name, kind, columns) VALUES (?, ?, ?, ?)`, + classOID, name, kind, columns, + ) + if err != nil { + return fmt.Errorf("create constraint %q on class %d: %w", name, classOID, err) + } + return nil +} diff --git a/internal/core/dialect.go b/internal/core/dialect.go new file mode 100644 index 0000000000..283f63f57f --- /dev/null +++ b/internal/core/dialect.go @@ -0,0 +1,49 @@ +package core + +import "fmt" + +// CreateDialect inserts a SQL dialect and returns its OID. +func (c *Catalog) CreateDialect(name string) (int64, error) { + res, err := c.db.Exec(`INSERT INTO sql_dialect (name) VALUES (?)`, name) + if err != nil { + return 0, fmt.Errorf("create dialect %q: %w", name, err) + } + return res.LastInsertId() +} + +// DialectOID returns the OID for a registered dialect. +func (c *Catalog) DialectOID(name string) (int64, error) { + var oid int64 + err := c.db.QueryRow(`SELECT oid FROM sql_dialect WHERE name = ?`, name).Scan(&oid) + if err != nil { + return 0, fmt.Errorf("dialect %q: %w", name, err) + } + return oid, nil +} + +// SetDialectFlag stores a per-dialect configuration value. +// If the key already exists, the value is replaced. +func (c *Catalog) SetDialectFlag(dialectOID int64, key, value string) error { + _, err := c.db.Exec( + `INSERT INTO sql_dialect_flag (dialect_oid, key, value) VALUES (?, ?, ?) + ON CONFLICT(dialect_oid, key) DO UPDATE SET value = excluded.value`, + dialectOID, key, value, + ) + if err != nil { + return fmt.Errorf("set dialect flag %s.%s: %w", key, value, err) + } + return nil +} + +// DialectFlag returns the value of a dialect flag, or "" if not set. +func (c *Catalog) DialectFlag(dialectOID int64, key string) (string, error) { + var value string + err := c.db.QueryRow( + `SELECT value FROM sql_dialect_flag WHERE dialect_oid = ? AND key = ?`, + dialectOID, key, + ).Scan(&value) + if err != nil { + return "", nil // missing flag = empty string, not an error + } + return value, nil +} diff --git a/internal/core/dialect_test.go b/internal/core/dialect_test.go new file mode 100644 index 0000000000..e3818c3aab --- /dev/null +++ b/internal/core/dialect_test.go @@ -0,0 +1,44 @@ +package core + +import "testing" + +func TestDialectAndFlags(t *testing.T) { + cat, err := New() + if err != nil { + t.Fatal(err) + } + defer cat.Close() + + pgOID, err := cat.CreateDialect("postgresql") + if err != nil { + t.Fatal(err) + } + + got, err := cat.DialectOID("postgresql") + if err != nil || got != pgOID { + t.Fatalf("DialectOID: got %d (err=%v), want %d", got, err, pgOID) + } + + if err := cat.SetDialectFlag(pgOID, "identifier_case", "fold_lower"); err != nil { + t.Fatal(err) + } + v, err := cat.DialectFlag(pgOID, "identifier_case") + if err != nil || v != "fold_lower" { + t.Errorf("DialectFlag: got %q (err=%v), want fold_lower", v, err) + } + + // upsert + if err := cat.SetDialectFlag(pgOID, "identifier_case", "fold_upper"); err != nil { + t.Fatal(err) + } + v, _ = cat.DialectFlag(pgOID, "identifier_case") + if v != "fold_upper" { + t.Errorf("upsert: got %q, want fold_upper", v) + } + + // missing flag returns "" + v, err = cat.DialectFlag(pgOID, "nonexistent") + if err != nil || v != "" { + t.Errorf("missing flag: got %q (err=%v), want empty", v, err) + } +} diff --git a/internal/core/namespace.go b/internal/core/namespace.go new file mode 100644 index 0000000000..498f89d795 --- /dev/null +++ b/internal/core/namespace.go @@ -0,0 +1,22 @@ +package core + +import "fmt" + +// CreateNamespace inserts a namespace and returns its OID. +func (c *Catalog) CreateNamespace(name string) (int64, error) { + res, err := c.db.Exec(`INSERT INTO sql_namespace (name) VALUES (?)`, name) + if err != nil { + return 0, fmt.Errorf("create namespace %q: %w", name, err) + } + return res.LastInsertId() +} + +// NamespaceOID returns the OID for the given namespace name. +func (c *Catalog) NamespaceOID(name string) (int64, error) { + var oid int64 + err := c.db.QueryRow(`SELECT oid FROM sql_namespace WHERE name = ?`, name).Scan(&oid) + if err != nil { + return 0, fmt.Errorf("namespace %q: %w", name, err) + } + return oid, nil +} diff --git a/internal/core/operator.go b/internal/core/operator.go new file mode 100644 index 0000000000..2e77739901 --- /dev/null +++ b/internal/core/operator.go @@ -0,0 +1,76 @@ +package core + +import "fmt" + +// OperatorSpec describes an operator overload for insertion into the catalog. +// A NULL left_type_oid encodes a prefix unary operator; NULL right_type_oid +// encodes a postfix unary. +type OperatorSpec struct { + Name string + NamespaceOID int64 // 0 = NULL + DialectOID int64 // 0 = NULL (shared) + LeftTypeOID int64 // 0 = NULL + RightTypeOID int64 // 0 = NULL + ResultTypeOID int64 + ProcOID int64 // 0 = NULL +} + +// CreateOperator inserts an operator overload and returns its OID. +func (c *Catalog) CreateOperator(o OperatorSpec) (int64, error) { + res, err := c.db.Exec( + `INSERT INTO sql_operator + (namespace_oid, dialect_oid, name, + left_type_oid, right_type_oid, result_type_oid, proc_oid) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + nullableOID(o.NamespaceOID), nullableOID(o.DialectOID), o.Name, + nullableOID(o.LeftTypeOID), nullableOID(o.RightTypeOID), + o.ResultTypeOID, nullableOID(o.ProcOID), + ) + if err != nil { + return 0, fmt.Errorf("create operator %q: %w", o.Name, err) + } + return res.LastInsertId() +} + +// OperatorOverload is a resolved candidate from FindOperators. +type OperatorOverload struct { + OID int64 + Name string + LeftTypeOID int64 // 0 = NULL (prefix unary) + RightTypeOID int64 // 0 = NULL (postfix unary) + ResultTypeOID int64 + ProcOID int64 // 0 = NULL (binary-compatible) +} + +// FindOperators returns all operator overloads matching the given name and +// (left,right) operand types. A 0 type means "any" and skips the filter on +// that side; useful for listing all overloads of a name. +func (c *Catalog) FindOperators(name string, leftTypeOID, rightTypeOID int64) ([]OperatorOverload, error) { + q := `SELECT oid, name, + COALESCE(left_type_oid, 0), COALESCE(right_type_oid, 0), + result_type_oid, COALESCE(proc_oid, 0) + FROM sql_operator WHERE name = ?` + args := []any{name} + if leftTypeOID != 0 { + q += ` AND left_type_oid = ?` + args = append(args, leftTypeOID) + } + if rightTypeOID != 0 { + q += ` AND right_type_oid = ?` + args = append(args, rightTypeOID) + } + rows, err := c.db.Query(q, args...) + if err != nil { + return nil, fmt.Errorf("find operators %q: %w", name, err) + } + defer rows.Close() + var out []OperatorOverload + for rows.Next() { + var o OperatorOverload + if err := rows.Scan(&o.OID, &o.Name, &o.LeftTypeOID, &o.RightTypeOID, &o.ResultTypeOID, &o.ProcOID); err != nil { + return nil, err + } + out = append(out, o) + } + return out, rows.Err() +} diff --git a/internal/core/operator_test.go b/internal/core/operator_test.go new file mode 100644 index 0000000000..c1631ce35f --- /dev/null +++ b/internal/core/operator_test.go @@ -0,0 +1,48 @@ +package core + +import "testing" + +func TestOperatorCreateAndFind(t *testing.T) { + cat, err := New() + if err != nil { + t.Fatal(err) + } + defer cat.Close() + + intOID, _ := cat.CreateType("integer", 4) + boolOID, _ := cat.CreateType("boolean", 1) + + // > (int, int) -> bool + gtOID, err := cat.CreateOperator(OperatorSpec{ + Name: ">", + LeftTypeOID: intOID, + RightTypeOID: intOID, + ResultTypeOID: boolOID, + }) + if err != nil { + t.Fatal(err) + } + if gtOID == 0 { + t.Fatal("expected non-zero operator oid") + } + + got, err := cat.FindOperators(">", intOID, intOID) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("FindOperators: want 1, got %d", len(got)) + } + if got[0].ResultTypeOID != boolOID { + t.Errorf("result type: got %d, want %d", got[0].ResultTypeOID, boolOID) + } + + // listing all overloads of > + all, err := cat.FindOperators(">", 0, 0) + if err != nil { + t.Fatal(err) + } + if len(all) != 1 { + t.Errorf("listing >: want 1, got %d", len(all)) + } +} diff --git a/internal/core/proc.go b/internal/core/proc.go new file mode 100644 index 0000000000..181b5d755f --- /dev/null +++ b/internal/core/proc.go @@ -0,0 +1,154 @@ +package core + +import ( + "database/sql" + "fmt" + "strings" +) + +// ProcSpec describes a function/aggregate/window/procedure for insertion +// into the catalog. +type ProcSpec struct { + Name string + NamespaceOID int64 // 0 = NULL + DialectOID int64 // 0 = NULL (shared) + Kind string // 'f'unc | 'a'gg | 'w'in | 'p'roc; default 'f' + ReturnTypeOID int64 + ReturnSet bool + ReturnNullable bool + Strict bool + VariadicKind string // 'n'one | 'a'rray | 'v'ariadic-any; default 'n' + Args []ProcArg +} + +// ProcArg is a single argument in a proc signature. +type ProcArg struct { + Name string + TypeOID int64 + Mode string // 'i'n | 'o'ut | 'b'oth | 't'able | 'v'ariadic; default 'i' + HasDefault bool +} + +// CreateProc inserts a proc and its arguments, returning the proc OID. +func (c *Catalog) CreateProc(p ProcSpec) (int64, error) { + if p.Kind == "" { + p.Kind = "f" + } + if p.VariadicKind == "" { + p.VariadicKind = "n" + } + res, err := c.db.Exec( + `INSERT INTO sql_proc + (namespace_oid, dialect_oid, name, kind, + return_type_oid, return_set, return_nullable, strict, variadic_kind) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + nullableOID(p.NamespaceOID), nullableOID(p.DialectOID), + strings.ToLower(p.Name), p.Kind, + p.ReturnTypeOID, boolToInt(p.ReturnSet), boolToInt(p.ReturnNullable), + boolToInt(p.Strict), p.VariadicKind, + ) + if err != nil { + return 0, fmt.Errorf("create proc %q: %w", p.Name, err) + } + procOID, err := res.LastInsertId() + if err != nil { + return 0, err + } + for i, a := range p.Args { + mode := a.Mode + if mode == "" { + mode = "i" + } + _, err := c.db.Exec( + `INSERT INTO sql_proc_arg (proc_oid, ord, name, type_oid, mode, has_default) + VALUES (?, ?, ?, ?, ?, ?)`, + procOID, i+1, a.Name, a.TypeOID, mode, boolToInt(a.HasDefault), + ) + if err != nil { + return 0, fmt.Errorf("create proc %q arg %d: %w", p.Name, i+1, err) + } + } + return procOID, nil +} + +// ProcOverload describes one resolved candidate from FindProcs. +type ProcOverload struct { + OID int64 + Name string + Kind string + ReturnTypeOID int64 + ReturnNullable bool + ArgTypes []int64 +} + +// FindProcs returns all procs matching the given (case-insensitive) name in +// any of the supplied namespaces. Pass an empty namespace list to search +// every namespace. Argument lists are populated in declaration order. +func (c *Catalog) FindProcs(name string, namespaceOIDs []int64) ([]ProcOverload, error) { + q := `SELECT oid, name, kind, return_type_oid, return_nullable + FROM sql_proc WHERE name = ?` + args := []any{strings.ToLower(name)} + if len(namespaceOIDs) > 0 { + q += " AND namespace_oid IN (" + placeholders(len(namespaceOIDs)) + ")" + for _, ns := range namespaceOIDs { + args = append(args, ns) + } + } + rows, err := c.db.Query(q, args...) + if err != nil { + return nil, fmt.Errorf("find procs %q: %w", name, err) + } + defer rows.Close() + var out []ProcOverload + for rows.Next() { + var o ProcOverload + var rn int + if err := rows.Scan(&o.OID, &o.Name, &o.Kind, &o.ReturnTypeOID, &rn); err != nil { + return nil, err + } + o.ReturnNullable = rn != 0 + out = append(out, o) + } + if err := rows.Err(); err != nil { + return nil, err + } + for i := range out { + argTypes, err := c.procArgTypes(out[i].OID) + if err != nil { + return nil, err + } + out[i].ArgTypes = argTypes + } + return out, nil +} + +func (c *Catalog) procArgTypes(procOID int64) ([]int64, error) { + rows, err := c.db.Query( + `SELECT type_oid FROM sql_proc_arg + WHERE proc_oid = ? AND mode IN ('i','b','v') + ORDER BY ord`, procOID, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var out []int64 + for rows.Next() { + var t int64 + if err := rows.Scan(&t); err != nil { + return nil, err + } + out = append(out, t) + } + return out, rows.Err() +} + +func placeholders(n int) string { + if n <= 0 { + return "" + } + return strings.Repeat("?,", n-1) + "?" +} + +// silence unused-import warning if we drop the sql.Null* usages +var _ sql.NullString diff --git a/internal/core/proc_test.go b/internal/core/proc_test.go new file mode 100644 index 0000000000..54ff899bcd --- /dev/null +++ b/internal/core/proc_test.go @@ -0,0 +1,69 @@ +package core + +import "testing" + +func TestProcCreateAndFind(t *testing.T) { + cat, err := New() + if err != nil { + t.Fatal(err) + } + defer cat.Close() + + intOID, err := cat.CreateType("integer", 4) + if err != nil { + t.Fatal(err) + } + textOID, err := cat.CreateType("text", 0) + if err != nil { + t.Fatal(err) + } + + // length(text) -> integer + procOID, err := cat.CreateProc(ProcSpec{ + Name: "length", + Kind: "f", + ReturnTypeOID: intOID, + ReturnNullable: true, + Args: []ProcArg{{Name: "s", TypeOID: textOID}}, + }) + if err != nil { + t.Fatalf("create length: %v", err) + } + if procOID == 0 { + t.Fatal("expected non-zero proc oid") + } + + // concat(text, text) -> text + if _, err := cat.CreateProc(ProcSpec{ + Name: "concat", + ReturnTypeOID: textOID, + Args: []ProcArg{ + {Name: "a", TypeOID: textOID}, + {Name: "b", TypeOID: textOID}, + }, + }); err != nil { + t.Fatal(err) + } + + got, err := cat.FindProcs("length", nil) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("FindProcs length: want 1, got %d", len(got)) + } + if got[0].ReturnTypeOID != intOID { + t.Errorf("length return type: got %d, want %d", got[0].ReturnTypeOID, intOID) + } + if len(got[0].ArgTypes) != 1 || got[0].ArgTypes[0] != textOID { + t.Errorf("length args: got %v, want [%d]", got[0].ArgTypes, textOID) + } + + got, err = cat.FindProcs("concat", nil) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || len(got[0].ArgTypes) != 2 { + t.Fatalf("FindProcs concat: got %+v", got) + } +} diff --git a/internal/core/schema.sql b/internal/core/schema.sql new file mode 100644 index 0000000000..e9d9741d4b --- /dev/null +++ b/internal/core/schema.sql @@ -0,0 +1,156 @@ +-- sql_namespace: schemas / namespaces +CREATE TABLE sql_namespace ( + oid INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE +); + +-- sql_dialect: registered SQL dialects (postgresql, sqlite, mysql, ...). +CREATE TABLE sql_dialect ( + oid INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE +); + +-- sql_dialect_flag: per-dialect configuration knobs (case-folding, +-- identifier quoting, alias scoping rules, etc.). Values are opaque +-- strings; the analyzer interprets them per key. +CREATE TABLE sql_dialect_flag ( + dialect_oid INTEGER NOT NULL REFERENCES sql_dialect(oid), + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (dialect_oid, key) +); + +-- sql_type: data types. Modeled on pg_type. +-- typtype: 'b'ase | 'c'omposite | 'd'omain | 'e'num | 'p'seudo | 'r'ange +-- category: 'N'umeric | 'S'tring | 'B'oolean | 'D'atetime | 'A'rray | +-- 'C'omposite | 'E'num | 'U'serdef | 'X'unknown +-- preferred: tie-breaker for implicit cast resolution within a category +-- element_oid: for arrays, points at the element type +-- dialect_oid: NULL = standard / shared across dialects +CREATE TABLE sql_type ( + oid INTEGER PRIMARY KEY AUTOINCREMENT, + namespace_oid INTEGER NOT NULL REFERENCES sql_namespace(oid), + dialect_oid INTEGER REFERENCES sql_dialect(oid), + name TEXT NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + typtype TEXT NOT NULL DEFAULT 'b', + category TEXT, + preferred INTEGER NOT NULL DEFAULT 0, + element_oid INTEGER REFERENCES sql_type(oid), + UNIQUE (namespace_oid, name) +); +CREATE INDEX idx_sql_type_name ON sql_type(name); + +-- sql_class: relations (tables, views, indexes). +-- kind: 'r' = table, 'v' = view, 'i' = index, 'c' = composite type, 'f' = foreign +CREATE TABLE sql_class ( + oid INTEGER PRIMARY KEY AUTOINCREMENT, + namespace_oid INTEGER NOT NULL REFERENCES sql_namespace(oid), + name TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'r', + UNIQUE(namespace_oid, name) +); + +-- sql_attribute: columns of a relation. +-- decl_type: original declared type string before normalization +-- (e.g. VARCHAR(10), BIGINT UNSIGNED, INTEGER PRIMARY KEY). +-- Useful for SQLite where multiple syntaxes collapse to +-- one of five affinities, and as a debugging aid. +-- type_length: length / precision (varchar(N), numeric(p,s).p, +-- char(N), bit(N)). 0 = unspecified. +-- type_scale: scale for numeric/decimal. 0 = unspecified. +-- auto_increment: rowid alias (sqlite INTEGER PRIMARY KEY), AUTOINCREMENT, +-- pg serial/bigserial/identity, mysql AUTO_INCREMENT. +-- is_primary_key: this column participates in the relation's primary key. +-- Set both for inline-column PK and for table-level PK. +-- is_unique: column has a UNIQUE constraint or a single-column UNIQUE +-- table constraint. +CREATE TABLE sql_attribute ( + oid INTEGER PRIMARY KEY AUTOINCREMENT, + class_oid INTEGER NOT NULL REFERENCES sql_class(oid), + name TEXT NOT NULL, + type_oid INTEGER NOT NULL REFERENCES sql_type(oid), + not_null INTEGER NOT NULL DEFAULT 0, + has_default INTEGER NOT NULL DEFAULT 0, + num INTEGER NOT NULL, -- ordinal position (1-based) + decl_type TEXT NOT NULL DEFAULT '', + type_length INTEGER NOT NULL DEFAULT 0, + type_scale INTEGER NOT NULL DEFAULT 0, + auto_increment INTEGER NOT NULL DEFAULT 0, + is_primary_key INTEGER NOT NULL DEFAULT 0, + is_unique INTEGER NOT NULL DEFAULT 0, + UNIQUE(class_oid, name), + UNIQUE(class_oid, num) +); + +-- sql_constraint: constraints on a relation. +-- kind: 'p' = primary key, 'f' = foreign key, 'u' = unique, 'c' = check +CREATE TABLE sql_constraint ( + oid INTEGER PRIMARY KEY AUTOINCREMENT, + class_oid INTEGER NOT NULL REFERENCES sql_class(oid), + name TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL, + columns TEXT NOT NULL DEFAULT '' -- comma-separated attribute nums +); + +-- sql_proc: functions, aggregates, window functions, procedures. +-- Modeled on pg_proc. +-- kind: 'f' = function, 'a' = aggregate, 'w' = window, 'p' = procedure +-- variadic_kind: 'n' = none, 'a' = array (VARIADIC any[]), 'v' = variadic-any +-- return_set: 1 if SETOF / table-returning +CREATE TABLE sql_proc ( + oid INTEGER PRIMARY KEY AUTOINCREMENT, + namespace_oid INTEGER REFERENCES sql_namespace(oid), + dialect_oid INTEGER REFERENCES sql_dialect(oid), + name TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'f', + return_type_oid INTEGER NOT NULL REFERENCES sql_type(oid), + return_set INTEGER NOT NULL DEFAULT 0, + return_nullable INTEGER NOT NULL DEFAULT 1, + strict INTEGER NOT NULL DEFAULT 0, + variadic_kind TEXT NOT NULL DEFAULT 'n' +); + +-- sql_proc_arg: ordered argument list for a proc. +-- mode: 'i' = in, 'o' = out, 'b' = both, 't' = table, 'v' = variadic +CREATE TABLE sql_proc_arg ( + proc_oid INTEGER NOT NULL REFERENCES sql_proc(oid), + ord INTEGER NOT NULL, + name TEXT NOT NULL DEFAULT '', + type_oid INTEGER NOT NULL REFERENCES sql_type(oid), + mode TEXT NOT NULL DEFAULT 'i', + has_default INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (proc_oid, ord) +); + +-- sql_operator: operator overloads. +-- left_type_oid is NULL for prefix unary; right_type_oid is NULL for postfix. +CREATE TABLE sql_operator ( + oid INTEGER PRIMARY KEY AUTOINCREMENT, + namespace_oid INTEGER REFERENCES sql_namespace(oid), + dialect_oid INTEGER REFERENCES sql_dialect(oid), + name TEXT NOT NULL, + left_type_oid INTEGER REFERENCES sql_type(oid), + right_type_oid INTEGER REFERENCES sql_type(oid), + result_type_oid INTEGER NOT NULL REFERENCES sql_type(oid), + proc_oid INTEGER REFERENCES sql_proc(oid), + commutator_oid INTEGER REFERENCES sql_operator(oid), + negator_oid INTEGER REFERENCES sql_operator(oid) +); + +-- sql_cast: type coercion rules. +-- context: 'i' = implicit, 'a' = assignment-only, 'e' = explicit-only +-- proc_oid NULL = binary-coercible (no function needed) +CREATE TABLE sql_cast ( + source_type_oid INTEGER NOT NULL REFERENCES sql_type(oid), + target_type_oid INTEGER NOT NULL REFERENCES sql_type(oid), + proc_oid INTEGER REFERENCES sql_proc(oid), + context TEXT NOT NULL DEFAULT 'e', + dialect_oid INTEGER REFERENCES sql_dialect(oid), + PRIMARY KEY (source_type_oid, target_type_oid) +); + +-- Resolution-speed indexes. +CREATE INDEX idx_sql_proc_name ON sql_proc(name, namespace_oid); +CREATE INDEX idx_sql_operator_name ON sql_operator(name, left_type_oid, right_type_oid); +CREATE INDEX idx_sql_attribute_name ON sql_attribute(name); diff --git a/internal/core/types.go b/internal/core/types.go new file mode 100644 index 0000000000..6acc2de1c0 --- /dev/null +++ b/internal/core/types.go @@ -0,0 +1,137 @@ +package core + +import ( + "database/sql" + "fmt" + "strings" +) + +// TypeSpec describes a SQL type for insertion into the catalog. +// All fields are optional; Name is required. +type TypeSpec struct { + Name string + Size int + Typtype string // 'b'ase | 'c'omposite | 'd'omain | 'e'num | 'p'seudo | 'r'ange + Category string // 'N'umeric | 'S'tring | 'B'oolean | 'D'atetime | 'A'rray | ... + Preferred bool + NamespaceOID int64 // 0 = NULL + DialectOID int64 // 0 = NULL (shared) + ElementOID int64 // for arrays; 0 = NULL +} + +// CreateType inserts a base type (typtype='b') and returns its OID. +// For full control, use CreateTypeSpec. +func (c *Catalog) CreateType(name string, size int) (int64, error) { + return c.CreateTypeSpec(TypeSpec{Name: name, Size: size, Typtype: "b"}) +} + +// CreateTypeSpec inserts a type with full metadata. NamespaceOID defaults +// to the "public" namespace when zero so the simpler CreateType API and +// engines that don't track namespaces (sqlite) continue to work. +func (c *Catalog) CreateTypeSpec(t TypeSpec) (int64, error) { + if t.Typtype == "" { + t.Typtype = "b" + } + if t.NamespaceOID == 0 { + oid, err := c.NamespaceOID("public") + if err != nil { + return 0, fmt.Errorf("create type %q: default namespace: %w", t.Name, err) + } + t.NamespaceOID = oid + } + res, err := c.db.Exec( + `INSERT INTO sql_type + (name, size, typtype, category, preferred, namespace_oid, dialect_oid, element_oid) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + strings.ToLower(t.Name), t.Size, t.Typtype, + nullableString(t.Category), boolToInt(t.Preferred), + t.NamespaceOID, nullableOID(t.DialectOID), nullableOID(t.ElementOID), + ) + if err != nil { + return 0, fmt.Errorf("create type %q: %w", t.Name, err) + } + return res.LastInsertId() +} + +// TypeOID returns the OID for the given (unqualified) type name. When +// the same name lives in multiple namespaces (e.g. a system-internal +// duplicate in information_schema and pg_catalog), the lookup prefers +// pg_catalog, then public, then any other namespace by name. +func (c *Catalog) TypeOID(name string) (int64, error) { + var oid int64 + err := c.db.QueryRow( + `SELECT t.oid FROM sql_type t + JOIN sql_namespace ns ON ns.oid = t.namespace_oid + WHERE t.name = ? + ORDER BY CASE ns.name + WHEN 'pg_catalog' THEN 0 + WHEN 'public' THEN 1 + ELSE 2 END, + ns.name + LIMIT 1`, + strings.ToLower(name), + ).Scan(&oid) + if err != nil { + return 0, fmt.Errorf("type %q: %w", name, err) + } + return oid, nil +} + +// TypeName returns the name for the given type OID. +func (c *Catalog) TypeName(oid int64) (string, error) { + var name string + err := c.db.QueryRow(`SELECT name FROM sql_type WHERE oid = ?`, oid).Scan(&name) + if err != nil { + return "", fmt.Errorf("type oid %d: %w", oid, err) + } + return name, nil +} + +// TypeInfo bundles the metadata returned by LookupType. +type TypeInfo struct { + OID int64 + Name string + Category string + Typtype string + Preferred bool +} + +// LookupType returns metadata for the given type OID. +func (c *Catalog) LookupType(oid int64) (TypeInfo, error) { + var ( + ti TypeInfo + category sql.NullString + pref int + ) + err := c.db.QueryRow( + `SELECT oid, name, COALESCE(category, ''), typtype, preferred FROM sql_type WHERE oid = ?`, + oid, + ).Scan(&ti.OID, &ti.Name, &category, &ti.Typtype, &pref) + if err != nil { + return ti, fmt.Errorf("lookup type oid %d: %w", oid, err) + } + ti.Category = category.String + ti.Preferred = pref != 0 + return ti, nil +} + +func nullableOID(oid int64) any { + if oid == 0 { + return nil + } + return oid +} + +func nullableString(s string) any { + if s == "" { + return nil + } + return s +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} From 071254eb28e8b5beccaef8ca17c7318550a1c082 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Wed, 22 Jul 2026 16:05:40 +0000 Subject: [PATCH 2/3] clickhouse: analyze queries through the core catalog + analyzer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire ClickHouse onto the merged core: a dialect seed registering the built-in ClickHouse types, and a DDL handler that populates the core catalog from CREATE TABLE using sqlc's existing ClickHouse parser. A smoke test proves the full vertical path — ClickHouse SQL -> sqlc's ClickHouse parser -> internal/sql/ast -> core catalog + analyzer -> PrepareResult — resolving column names, types, nullability, source bindings, and star expansion, with none of the legacy compiler analyze step involved. Also fix the ClickHouse converter to render nested type parameters (the inner type of Nullable(T)/Array(T), Decimal precision, etc.) into TypeName.Name instead of dropping them as TODO nodes, so wrapped types resolve to their effective scalar type. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XTGxNHW6v1S1YyC9FDSgrK --- internal/engine/clickhouse/analyze_test.go | 100 +++++++++ internal/engine/clickhouse/convert.go | 52 ++++- internal/engine/clickhouse/schema.go | 227 +++++++++++++++++++++ internal/engine/clickhouse/seed.go | 79 +++++++ 4 files changed, 450 insertions(+), 8 deletions(-) create mode 100644 internal/engine/clickhouse/analyze_test.go create mode 100644 internal/engine/clickhouse/schema.go create mode 100644 internal/engine/clickhouse/seed.go diff --git a/internal/engine/clickhouse/analyze_test.go b/internal/engine/clickhouse/analyze_test.go new file mode 100644 index 0000000000..8d0369cf59 --- /dev/null +++ b/internal/engine/clickhouse/analyze_test.go @@ -0,0 +1,100 @@ +package clickhouse_test + +import ( + "strings" + "testing" + + "github.com/sqlc-dev/sqlc/internal/core" + "github.com/sqlc-dev/sqlc/internal/core/analyzer" + "github.com/sqlc-dev/sqlc/internal/engine/clickhouse" +) + +// analyzeOne seeds a ClickHouse-dialect catalog, applies ddl, then runs +// the dialect-neutral core analyzer over the trailing query. It proves +// the full ClickHouse vertical path: ClickHouse SQL -> sqlc's ClickHouse +// parser -> internal/sql/ast -> core catalog + analyzer -> PrepareResult, +// with no legacy compiler analyze step involved. +func analyzeOne(t *testing.T, ddl, query string) core.PrepareResult { + t.Helper() + cat, err := core.New(clickhouse.Dialect()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + + if err := clickhouse.LoadSchema(cat, ddl); err != nil { + t.Fatalf("load schema: %v", err) + } + + stmts, err := clickhouse.NewParser().Parse(strings.NewReader(query)) + if err != nil { + t.Fatalf("parse query: %v", err) + } + if len(stmts) != 1 { + t.Fatalf("expected 1 stmt, got %d", len(stmts)) + } + res, err := analyzer.Prepare(cat, stmts[0].Raw) + if err != nil { + t.Fatalf("analyze: %v", err) + } + return res +} + +func colByName(res core.PrepareResult, name string) (core.Column, bool) { + for _, c := range res.Columns { + if c.Name == name { + return c, true + } + } + return core.Column{}, false +} + +const eventsDDL = ` +CREATE TABLE events ( + id UInt64, + name String, + tag Nullable(String), + amount Decimal(18, 4) +) ENGINE = MergeTree ORDER BY id +` + +func TestClickHouseSelectColumns(t *testing.T) { + res := analyzeOne(t, eventsDDL, `SELECT id, name, tag FROM events`) + + if len(res.Columns) != 3 { + t.Fatalf("got %d cols, want 3: %+v", len(res.Columns), res.Columns) + } + + id, ok := colByName(res, "id") + if !ok || id.DataType != "uint64" || !id.NotNull { + t.Errorf("id: %+v (ok=%v)", id, ok) + } + name, ok := colByName(res, "name") + if !ok || name.DataType != "string" || !name.NotNull { + t.Errorf("name: %+v (ok=%v)", name, ok) + } + // Nullable(String) must resolve to the inner scalar and be nullable. + tag, ok := colByName(res, "tag") + if !ok || tag.DataType != "string" || tag.NotNull { + t.Errorf("tag: want string/nullable, got %+v (ok=%v)", tag, ok) + } + + for _, c := range res.Columns { + if c.SourceClassOID == 0 || c.SourceAttributeOID == 0 { + t.Errorf("col %s missing source binding: %+v", c.Name, c) + } + } +} + +func TestClickHouseSelectStar(t *testing.T) { + res := analyzeOne(t, eventsDDL, `SELECT * FROM events`) + if len(res.Columns) != 4 { + t.Fatalf("got %d cols, want 4: %+v", len(res.Columns), res.Columns) + } + want := []string{"id", "name", "tag", "amount"} + for i, w := range want { + if res.Columns[i].Name != w { + t.Errorf("col %d: got %q, want %q", i, res.Columns[i].Name, w) + } + } +} diff --git a/internal/engine/clickhouse/convert.go b/internal/engine/clickhouse/convert.go index ba2817e2bb..f1fad62be7 100644 --- a/internal/engine/clickhouse/convert.go +++ b/internal/engine/clickhouse/convert.go @@ -1,6 +1,7 @@ package clickhouse import ( + "fmt" "strconv" "strings" @@ -825,15 +826,14 @@ func (c *cc) convertColumnDeclaration(n *chast.ColumnDeclaration) *ast.ColumnDef } if n.Type != nil { + // Render the full type text (including nested type parameters such + // as the inner type of Nullable(T) / Array(T) and the precision of + // Decimal(P, S)) into Name so downstream consumers can recover the + // complete declaration. Nested type parameters are themselves + // *chast.DataType nodes, which the generic expression converter + // cannot represent, so they are rendered here instead. colDef.TypeName = &ast.TypeName{ - Name: n.Type.Name, - } - // Handle type parameters (e.g., Decimal(10, 2)) - if len(n.Type.Parameters) > 0 { - colDef.TypeName.Typmods = &ast.List{} - for _, param := range n.Type.Parameters { - colDef.TypeName.Typmods.Items = append(colDef.TypeName.Typmods.Items, c.convertExpr(param)) - } + Name: renderDataType(n.Type), } } @@ -855,6 +855,42 @@ func (c *cc) convertColumnDeclaration(n *chast.ColumnDeclaration) *ast.ColumnDef return colDef } +// renderDataType renders a ClickHouse type node back to its canonical +// textual form, e.g. Nullable(String), Array(UInt64), Decimal(18, 4), or +// Map(String, Array(Nullable(UInt32))). Nested type parameters recurse. +func renderDataType(dt *chast.DataType) string { + if dt == nil { + return "" + } + if len(dt.Parameters) == 0 { + return dt.Name + } + parts := make([]string, 0, len(dt.Parameters)) + for _, p := range dt.Parameters { + parts = append(parts, renderTypeParam(p)) + } + return dt.Name + "(" + strings.Join(parts, ", ") + ")" +} + +// renderTypeParam renders a single type parameter: a nested type, a +// numeric/string literal (Decimal precision, FixedString length, Enum +// value), or an identifier. +func renderTypeParam(e chast.Expression) string { + switch v := e.(type) { + case *chast.DataType: + return renderDataType(v) + case *chast.Literal: + if v.Source != "" { + return v.Source + } + return fmt.Sprintf("%v", v.Value) + case *chast.Identifier: + return strings.Join(v.Parts, ".") + default: + return "" + } +} + func (c *cc) convertUpdateQuery(n *chast.UpdateQuery) *ast.UpdateStmt { rv := &ast.RangeVar{ Relname: &n.Table, diff --git a/internal/engine/clickhouse/schema.go b/internal/engine/clickhouse/schema.go new file mode 100644 index 0000000000..283d71e70a --- /dev/null +++ b/internal/engine/clickhouse/schema.go @@ -0,0 +1,227 @@ +package clickhouse + +import ( + "fmt" + "strings" + + "github.com/sqlc-dev/sqlc/internal/core" + "github.com/sqlc-dev/sqlc/internal/sql/ast" +) + +// LoadSchema parses ClickHouse DDL and applies every schema-altering +// statement to the core catalog. It is the ClickHouse entry point for +// populating a catalog from migration files. +func LoadSchema(cat *core.Catalog, ddl string) error { + stmts, err := NewParser().Parse(strings.NewReader(ddl)) + if err != nil { + return fmt.Errorf("clickhouse: parse schema: %w", err) + } + for _, s := range stmts { + if err := Apply(cat, s.Raw); err != nil { + return err + } + } + return nil +} + +// Apply walks one parsed ClickHouse statement and applies it to the +// catalog when it is schema-altering (CREATE TABLE, DROP TABLE). +// Non-DDL and unsupported DDL forms are ignored so a mixed DDL/DML +// stream can be handed to the catalog without pre-filtering. +func Apply(cat *core.Catalog, n ast.Node) error { + switch v := n.(type) { + case nil: + return nil + case *ast.RawStmt: + return Apply(cat, v.Stmt) + case *ast.List: + for _, it := range v.Items { + if err := Apply(cat, it); err != nil { + return err + } + } + return nil + case *ast.CreateTableStmt: + return applyCreateTable(cat, v) + case *ast.DropTableStmt: + return applyDropTable(cat, v) + } + return nil +} + +func applyCreateTable(cat *core.Catalog, stmt *ast.CreateTableStmt) error { + if stmt.Name == nil { + return fmt.Errorf("clickhouse: create table with nil name") + } + nsOID, err := resolveOrCreateNamespace(cat, stmt.Name.Schema) + if err != nil { + return err + } + if _, err := cat.ClassOID(nsOID, stmt.Name.Name); err == nil { + if stmt.IfNotExists { + return nil + } + return fmt.Errorf("clickhouse: relation %q already exists", stmt.Name.Name) + } + classOID, err := cat.CreateClass(nsOID, stmt.Name.Name, "r") + if err != nil { + return err + } + for i, col := range stmt.Cols { + if col == nil || col.TypeName == nil { + return fmt.Errorf("clickhouse: column %d on %q: missing type", i+1, stmt.Name.Name) + } + // TODO: the core catalog does not yet model array-ness on an + // attribute; for now an Array(T) column is stored as its element + // type T. Preserving the array wrapper is a follow-up. + typeName, _, _ := unwrapType(col.TypeName) + typeOID, err := resolveOrCreateType(cat, typeName) + if err != nil { + return fmt.Errorf("clickhouse: column %s.%s: %w", stmt.Name.Name, col.Colname, err) + } + if err := cat.CreateAttributeSpec(core.AttributeSpec{ + ClassOID: classOID, + Name: col.Colname, + TypeOID: typeOID, + Num: i + 1, + NotNull: col.IsNotNull || col.PrimaryKey, + IsPrimaryKey: col.PrimaryKey, + DeclType: col.TypeName.Name, + }); err != nil { + return fmt.Errorf("clickhouse: attr %s.%s: %w", stmt.Name.Name, col.Colname, err) + } + } + return nil +} + +func applyDropTable(cat *core.Catalog, stmt *ast.DropTableStmt) error { + for _, tn := range stmt.Tables { + if tn == nil { + continue + } + nsOID, err := cat.NamespaceOID(nsName(tn.Schema)) + if err != nil { + if stmt.IfExists { + continue + } + return err + } + classOID, err := cat.ClassOID(nsOID, tn.Name) + if err != nil { + if stmt.IfExists { + continue + } + return fmt.Errorf("clickhouse: drop table %q: %w", tn.Name, err) + } + db := cat.DB() + if _, err := db.Exec(`DELETE FROM sql_attribute WHERE class_oid = ?`, classOID); err != nil { + return err + } + if _, err := db.Exec(`DELETE FROM sql_class WHERE oid = ?`, classOID); err != nil { + return err + } + } + return nil +} + +// nsName maps an empty schema to the catalog's default namespace. The +// analyzer resolves unqualified table references against "public", so +// ClickHouse's "default" database is mapped onto it for now. Wiring a +// per-dialect default namespace is a follow-up. +func nsName(schema string) string { + if schema == "" { + return "public" + } + return schema +} + +func resolveOrCreateNamespace(cat *core.Catalog, schema string) (int64, error) { + name := nsName(schema) + if oid, err := cat.NamespaceOID(name); err == nil { + return oid, nil + } + return cat.CreateNamespace(name) +} + +// resolveOrCreateType looks a type up by name, registering it as an +// unknown user type if the seed did not provide it (e.g. an Enum with +// inline variants, or a type name we have not catalogued yet). +func resolveOrCreateType(cat *core.Catalog, name string) (int64, error) { + canonical := strings.ToLower(strings.TrimSpace(name)) + if canonical == "" { + canonical = "nothing" + } + if oid, err := cat.TypeOID(canonical); err == nil { + return oid, nil + } + return cat.CreateType(canonical, 0) +} + +// unwrapType reduces a ClickHouse column type to the effective scalar +// type used for storage, reporting whether it is an array and whether it +// is nullable. The ClickHouse converter renders the full type text into +// TypeName.Name (e.g. "Nullable(Array(UInt64))"), which is parsed here. +// It unwraps any nesting of Nullable / LowCardinality / Array; the +// remaining scalar (String, UInt64, Decimal(18, 4), ...) is returned as +// its bare type name, lower-cased. +func unwrapType(tn *ast.TypeName) (name string, isArray, nullable bool) { + return unwrapTypeString(tn.Name) +} + +func unwrapTypeString(s string) (name string, isArray, nullable bool) { + base, args := splitType(s) + switch strings.ToLower(base) { + case "nullable": + if len(args) == 1 { + inner, arr, _ := unwrapTypeString(args[0]) + return inner, arr, true + } + return strings.ToLower(base), false, true + case "lowcardinality": + if len(args) == 1 { + return unwrapTypeString(args[0]) + } + return strings.ToLower(base), false, false + case "array": + if len(args) == 1 { + inner, _, nul := unwrapTypeString(args[0]) + return inner, true, nul + } + return strings.ToLower(base), true, false + default: + // Scalar type. Drop any (length/precision) modifiers for the stored + // type name; capturing them into TypeLength/TypeScale is a follow-up. + return strings.ToLower(base), false, false + } +} + +// splitType breaks "Name(arg1, arg2, ...)" into its base name and +// top-level arguments, respecting nested parentheses. A type with no +// parentheses returns (name, nil). +func splitType(s string) (base string, args []string) { + s = strings.TrimSpace(s) + open := strings.IndexByte(s, '(') + if open < 0 || !strings.HasSuffix(s, ")") { + return s, nil + } + base = strings.TrimSpace(s[:open]) + inner := s[open+1 : len(s)-1] + depth, start := 0, 0 + for i := 0; i < len(inner); i++ { + switch inner[i] { + case '(': + depth++ + case ')': + depth-- + case ',': + if depth == 0 { + args = append(args, strings.TrimSpace(inner[start:i])) + start = i + 1 + } + } + } + if last := strings.TrimSpace(inner[start:]); last != "" { + args = append(args, last) + } + return base, args +} diff --git a/internal/engine/clickhouse/seed.go b/internal/engine/clickhouse/seed.go new file mode 100644 index 0000000000..af7d149875 --- /dev/null +++ b/internal/engine/clickhouse/seed.go @@ -0,0 +1,79 @@ +package clickhouse + +import ( + "fmt" + + "github.com/sqlc-dev/sqlc/internal/core" +) + +// Dialect returns a core.Option that registers the "clickhouse" dialect +// and seeds the catalog with the built-in ClickHouse data types. It is +// the ClickHouse counterpart to the per-dialect Dialect() helpers in +// xqlc, and is how a ClickHouse-backed core.Catalog is constructed: +// +// cat, err := core.New(clickhouse.Dialect()) +func Dialect() core.Option { + return core.WithSeed(Seed) +} + +// chType pairs a ClickHouse type name with its pg_type-style category so +// the catalog can reason about it during (future) operator and cast +// resolution. Names are stored lower-cased by the catalog. +type chType struct { + name string + category string // 'N'umeric | 'S'tring | 'B'oolean | 'D'atetime | 'A'rray | 'U'serdef +} + +// clickhouseTypes is the built-in type surface. It is intentionally a +// curated hand list for now; once testgen/clickhouse exists these will +// be regenerated against a real ClickHouse instance, matching the +// seedgen approach used by the other engines. +var clickhouseTypes = []chType{ + // Unsigned integers + {"UInt8", "N"}, {"UInt16", "N"}, {"UInt32", "N"}, {"UInt64", "N"}, + {"UInt128", "N"}, {"UInt256", "N"}, + // Signed integers + {"Int8", "N"}, {"Int16", "N"}, {"Int32", "N"}, {"Int64", "N"}, + {"Int128", "N"}, {"Int256", "N"}, + // Floating point / decimal + {"Float32", "N"}, {"Float64", "N"}, {"BFloat16", "N"}, + {"Decimal", "N"}, {"Decimal32", "N"}, {"Decimal64", "N"}, + {"Decimal128", "N"}, {"Decimal256", "N"}, + // Boolean + {"Bool", "B"}, + // Strings / binary + {"String", "S"}, {"FixedString", "S"}, {"UUID", "S"}, + // Date & time + {"Date", "D"}, {"Date32", "D"}, {"DateTime", "D"}, {"DateTime64", "D"}, + // Network / misc scalar + {"IPv4", "S"}, {"IPv6", "S"}, {"JSON", "U"}, + // Enums are declared with their variants; the base names still resolve. + {"Enum8", "U"}, {"Enum16", "U"}, + // Wrapper / composite type names. Columns declared with these are + // normally unwrapped to their inner scalar by the DDL handler, but we + // register the names so an un-unwrappable declaration still resolves. + {"Nullable", "U"}, {"LowCardinality", "U"}, {"Array", "A"}, + {"Map", "U"}, {"Tuple", "U"}, {"Nested", "U"}, {"Nothing", "U"}, +} + +// Seed registers the ClickHouse dialect row and its built-in types on a +// freshly bootstrapped catalog. +func Seed(cat *core.Catalog) error { + dialectOID, err := cat.CreateDialect("clickhouse") + if err != nil { + return err + } + // ClickHouse identifiers are case-sensitive, so unlike MySQL/SQLite we + // do not set a case-folding flag. + for _, t := range clickhouseTypes { + if _, err := cat.CreateTypeSpec(core.TypeSpec{ + Name: t.name, + Typtype: "b", + Category: t.category, + DialectOID: dialectOID, + }); err != nil { + return fmt.Errorf("seed clickhouse type %q: %w", t.name, err) + } + } + return nil +} From fda4d41588fe124208ae19492321e13b46d8bd8c Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Wed, 22 Jul 2026 16:31:31 +0000 Subject: [PATCH 3/3] clickhouse: wire end-to-end sqlc generate onto the core Add the EngineClickHouse engine and a dedicated compile path so `sqlc generate` produces Go for ClickHouse entirely through the xqlc core, bypassing the legacy compiler analyze step and the in-memory sql/catalog: - config: add the "clickhouse" engine constant. - compiler: NewCompiler builds a core.Catalog seeded with the ClickHouse dialect; parseCatalog applies schema DDL to it; a new parseQueryCore resolves each query's columns and parameters via core/analyzer and assembles *compiler.Query, reusing only the shared query-metadata parsing. The legacy analyzeQuery/inferQuery/outputColumns path and the analyzer.Analyzer seam are never entered. - codegen: project the core catalog into plugin.Catalog for model/enum generation, and add a ClickHouse -> Go type map (Nullable(T) -> *T, the integer ladder, Float32/64, String, DateTime -> time.Time, ...). - clickhouse parser: compute statement byte-spans with a running offset and a semicolon scan (doubleclick reports statement starts but not ends), so leading "-- name:" annotations fall inside each statement. An endtoend case (clickhouse_select) exercises the full pipeline and its golden Go output is committed. Updating parse_basic/clickhouse's golden reflects the corrected statement spans and now-detected query name/cmd. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XTGxNHW6v1S1YyC9FDSgrK --- internal/cmd/shim.go | 83 +++++++++++- internal/codegen/golang/clickhouse_type.go | 68 ++++++++++ internal/codegen/golang/go_type.go | 2 + internal/compiler/compile.go | 18 ++- internal/compiler/engine.go | 21 +++ internal/compiler/parse.go | 6 + internal/compiler/parse_clickhouse.go | 126 ++++++++++++++++++ internal/compiler/result.go | 7 + internal/config/config.go | 1 + internal/core/analyzer/scope.go | 4 +- internal/core/attribute.go | 70 +++++----- internal/core/cast_test.go | 16 +-- .../clickhouse_select/clickhouse/db/db.go | 31 +++++ .../clickhouse_select/clickhouse/db/models.go | 5 + .../clickhouse/db/query.sql.go | 52 ++++++++ .../clickhouse_select/clickhouse/query.sql | 2 + .../clickhouse_select/clickhouse/schema.sql | 7 + .../clickhouse_select/clickhouse/sqlc.json | 16 +++ .../parse_basic/clickhouse/stdout.txt | 6 +- internal/engine/clickhouse/convert.go | 4 +- internal/engine/clickhouse/parse.go | 94 +++++++++++-- 21 files changed, 577 insertions(+), 62 deletions(-) create mode 100644 internal/codegen/golang/clickhouse_type.go create mode 100644 internal/compiler/parse_clickhouse.go create mode 100644 internal/endtoend/testdata/clickhouse_select/clickhouse/db/db.go create mode 100644 internal/endtoend/testdata/clickhouse_select/clickhouse/db/models.go create mode 100644 internal/endtoend/testdata/clickhouse_select/clickhouse/db/query.sql.go create mode 100644 internal/endtoend/testdata/clickhouse_select/clickhouse/query.sql create mode 100644 internal/endtoend/testdata/clickhouse_select/clickhouse/schema.sql create mode 100644 internal/endtoend/testdata/clickhouse_select/clickhouse/sqlc.json diff --git a/internal/cmd/shim.go b/internal/cmd/shim.go index 654500429a..690364eee9 100644 --- a/internal/cmd/shim.go +++ b/internal/cmd/shim.go @@ -4,6 +4,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/compiler" "github.com/sqlc-dev/sqlc/internal/config" "github.com/sqlc-dev/sqlc/internal/config/convert" + "github.com/sqlc-dev/sqlc/internal/core" "github.com/sqlc-dev/sqlc/internal/info" "github.com/sqlc-dev/sqlc/internal/plugin" "github.com/sqlc-dev/sqlc/internal/sql/catalog" @@ -224,10 +225,90 @@ func pluginQueryParam(p compiler.Parameter) *plugin.Parameter { } func codeGenRequest(r *compiler.Result, settings config.CombinedSettings) *plugin.GenerateRequest { + // Engines on the xqlc core (ClickHouse) project the codegen catalog + // from the core catalog rather than the in-memory sql/catalog, which is + // nil on that path. + var cat *plugin.Catalog + if r.CoreCatalog != nil { + cat = pluginCatalogFromCore(r.CoreCatalog) + } else { + cat = pluginCatalog(r.Catalog) + } return &plugin.GenerateRequest{ Settings: pluginSettings(r, settings), - Catalog: pluginCatalog(r.Catalog), + Catalog: cat, Queries: pluginQueries(r), SqlcVersion: info.Version, } } + +// pluginCatalogFromCore projects a core.Catalog (the xqlc SQLite-backed +// catalog) into the plugin.Catalog that codegen consumes to emit models +// and enums. It reads the namespace / class / attribute / type tables +// directly. Projection is best-effort: an unexpected query error against +// the in-memory catalog yields a partial catalog rather than aborting +// generation. +func pluginCatalogFromCore(cc *core.Catalog) *plugin.Catalog { + db := cc.DB() + var schemas []*plugin.Schema + + type row struct { + oid int64 + name string + } + readRows := func(query string, args ...any) []row { + rows, err := db.Query(query, args...) + if err != nil { + return nil + } + defer rows.Close() + var out []row + for rows.Next() { + var r row + if err := rows.Scan(&r.oid, &r.name); err != nil { + return out + } + out = append(out, r) + } + return out + } + + for _, ns := range readRows(`SELECT oid, name FROM sql_namespace ORDER BY oid`) { + var tables []*plugin.Table + for _, cl := range readRows( + `SELECT oid, name FROM sql_class WHERE namespace_oid = ? AND kind = 'r' ORDER BY oid`, ns.oid, + ) { + rel := &plugin.Identifier{Schema: ns.name, Name: cl.name} + var columns []*plugin.Column + crows, err := db.Query( + `SELECT a.name, t.name, a.not_null + FROM sql_attribute a JOIN sql_type t ON t.oid = a.type_oid + WHERE a.class_oid = ? ORDER BY a.num`, cl.oid, + ) + if err != nil { + continue + } + for crows.Next() { + var name, typeName string + var notNull int + if err := crows.Scan(&name, &typeName, ¬Null); err != nil { + break + } + columns = append(columns, &plugin.Column{ + Name: name, + Type: &plugin.Identifier{Name: typeName}, + NotNull: notNull != 0, + Table: rel, + }) + } + crows.Close() + tables = append(tables, &plugin.Table{Rel: rel, Columns: columns}) + } + schemas = append(schemas, &plugin.Schema{Name: ns.name, Tables: tables}) + } + + return &plugin.Catalog{ + DefaultSchema: "public", + Schemas: schemas, + } +} diff --git a/internal/codegen/golang/clickhouse_type.go b/internal/codegen/golang/clickhouse_type.go new file mode 100644 index 0000000000..93613eec06 --- /dev/null +++ b/internal/codegen/golang/clickhouse_type.go @@ -0,0 +1,68 @@ +package golang + +import ( + "strings" + + "github.com/sqlc-dev/sqlc/internal/codegen/golang/opts" + "github.com/sqlc-dev/sqlc/internal/codegen/sdk" + "github.com/sqlc-dev/sqlc/internal/plugin" +) + +// clickhouseType maps a ClickHouse column type to a Go type. Type names +// arrive lower-cased from the core catalog (e.g. "uint64", "string", +// "datetime"). Nullable columns (NotNull == false) map to a pointer, which +// is how the clickhouse-go driver represents Nullable(T). +func clickhouseType(req *plugin.GenerateRequest, options *opts.Options, col *plugin.Column) string { + dt := strings.ToLower(sdk.DataType(col.Type)) + notNull := col.NotNull + + switch dt { + case "uint8": + return nullable(notNull, "uint8") + case "uint16": + return nullable(notNull, "uint16") + case "uint32": + return nullable(notNull, "uint32") + case "uint64": + return nullable(notNull, "uint64") + case "int8": + return nullable(notNull, "int8") + case "int16": + return nullable(notNull, "int16") + case "int32": + return nullable(notNull, "int32") + case "int64": + return nullable(notNull, "int64") + case "uint128", "uint256", "int128", "int256": + // Big integers are represented as *big.Int by clickhouse-go. + return "*big.Int" + case "float32", "bfloat16": + return nullable(notNull, "float32") + case "float64": + return nullable(notNull, "float64") + case "bool": + return nullable(notNull, "bool") + case "string", "fixedstring": + return nullable(notNull, "string") + case "date", "date32", "datetime", "datetime64": + return nullable(notNull, "time.Time") + + // The following resolve to string for now; richer mappings + // (decimal.Decimal, uuid.UUID, netip.Addr, json.RawMessage) require + // wiring their imports into the Go importer and are a follow-up. + case "decimal", "decimal32", "decimal64", "decimal128", "decimal256", + "uuid", "ipv4", "ipv6", "json", "enum8", "enum16": + return nullable(notNull, "string") + + default: + return "interface{}" + } +} + +// nullable wraps a base Go type in a pointer when the column is nullable. +func nullable(notNull bool, base string) string { + if notNull { + return base + } + return "*" + base +} diff --git a/internal/codegen/golang/go_type.go b/internal/codegen/golang/go_type.go index 21914736d0..b203dd525d 100644 --- a/internal/codegen/golang/go_type.go +++ b/internal/codegen/golang/go_type.go @@ -86,6 +86,8 @@ func goInnerType(req *plugin.GenerateRequest, options *opts.Options, col *plugin return postgresType(req, options, col) case "sqlite": return sqliteType(req, options, col) + case "clickhouse": + return clickhouseType(req, options, col) default: return "interface{}" } diff --git a/internal/compiler/compile.go b/internal/compiler/compile.go index b6bba42e16..b5665c28ef 100644 --- a/internal/compiler/compile.go +++ b/internal/compiler/compile.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" + "github.com/sqlc-dev/sqlc/internal/engine/clickhouse" "github.com/sqlc-dev/sqlc/internal/migrations" "github.com/sqlc-dev/sqlc/internal/multierr" "github.com/sqlc-dev/sqlc/internal/opts" @@ -55,6 +56,18 @@ func (c *Compiler) parseCatalog(schemas []string) error { continue } + // ClickHouse populates the core catalog instead of the in-memory + // sql/catalog. + if c.coreCatalog != nil { + for i := range stmts { + if err := clickhouse.Apply(c.coreCatalog, stmts[i].Raw); err != nil { + merr.Add(filename, contents, stmts[i].Pos(), err) + continue + } + } + continue + } + for i := range stmts { if err := c.catalog.Update(stmts[i], c); err != nil { merr.Add(filename, contents, stmts[i].Pos(), err) @@ -135,7 +148,8 @@ func (c *Compiler) parseQueries(o opts.Parser) (*Result, error) { } return &Result{ - Catalog: c.catalog, - Queries: q, + Catalog: c.catalog, + CoreCatalog: c.coreCatalog, + Queries: q, }, nil } diff --git a/internal/compiler/engine.go b/internal/compiler/engine.go index 64fdf3d5c7..43767b0faf 100644 --- a/internal/compiler/engine.go +++ b/internal/compiler/engine.go @@ -6,7 +6,9 @@ import ( "github.com/sqlc-dev/sqlc/internal/analyzer" "github.com/sqlc-dev/sqlc/internal/config" + "github.com/sqlc-dev/sqlc/internal/core" "github.com/sqlc-dev/sqlc/internal/dbmanager" + "github.com/sqlc-dev/sqlc/internal/engine/clickhouse" "github.com/sqlc-dev/sqlc/internal/engine/dolphin" "github.com/sqlc-dev/sqlc/internal/engine/postgresql" pganalyze "github.com/sqlc-dev/sqlc/internal/engine/postgresql/analyzer" @@ -27,6 +29,11 @@ type Compiler struct { client dbmanager.Client selector selector + // coreCatalog is the xqlc-derived catalog used by engines whose + // analysis runs on the core analyzer (currently ClickHouse) instead of + // the legacy compiler analyze step. It is nil for other engines. + coreCatalog *core.Catalog + schema []string // databaseOnlyMode indicates that the compiler should use database-only analysis @@ -111,6 +118,17 @@ func NewCompiler(conf config.SQL, combo config.CombinedSettings, parserOpts opts ) } } + case config.EngineClickHouse: + // ClickHouse runs on the xqlc analysis core: its schema and queries + // are resolved against a core.Catalog by the core analyzer, not the + // legacy compiler analyze step or the in-memory sql/catalog. + c.parser = clickhouse.NewParser() + c.selector = newDefaultSelector() + cat, err := core.New(clickhouse.Dialect()) + if err != nil { + return nil, fmt.Errorf("clickhouse: init catalog: %w", err) + } + c.coreCatalog = cat default: return nil, fmt.Errorf("unknown engine: %s", conf.Engine) } @@ -145,4 +163,7 @@ func (c *Compiler) Close(ctx context.Context) { if c.client != nil { c.client.Close(ctx) } + if c.coreCatalog != nil { + c.coreCatalog.Close() + } } diff --git a/internal/compiler/parse.go b/internal/compiler/parse.go index 2f9afb72c1..61b79ec1f7 100644 --- a/internal/compiler/parse.go +++ b/internal/compiler/parse.go @@ -19,6 +19,12 @@ import ( var debugDumpAST = sqlcdebug.New("dumpast") func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query, error) { + // ClickHouse resolves types through the core analyzer, entirely + // bypassing the legacy analyze step below. + if c.coreCatalog != nil { + return c.parseQueryCore(stmt, src) + } + ctx := context.Background() if debugDumpAST.Value() == "1" { diff --git a/internal/compiler/parse_clickhouse.go b/internal/compiler/parse_clickhouse.go new file mode 100644 index 0000000000..f10466b764 --- /dev/null +++ b/internal/compiler/parse_clickhouse.go @@ -0,0 +1,126 @@ +package compiler + +import ( + "errors" + "strings" + + "github.com/sqlc-dev/sqlc/internal/core" + coreanalyzer "github.com/sqlc-dev/sqlc/internal/core/analyzer" + "github.com/sqlc-dev/sqlc/internal/metadata" + "github.com/sqlc-dev/sqlc/internal/source" + "github.com/sqlc-dev/sqlc/internal/sql/ast" + "github.com/sqlc-dev/sqlc/internal/sql/validate" +) + +// parseQueryCore is the analysis path for engines backed by the xqlc core +// catalog and analyzer (currently ClickHouse). It reuses the shared, +// engine-agnostic query-metadata parsing but resolves columns and +// parameters through core/analyzer, never touching the legacy compiler +// analyze step (inferQuery/outputColumns/parameters) or the +// analyzer.Analyzer seam. +func (c *Compiler) parseQueryCore(stmt ast.Node, src string) (*Query, error) { + raw, ok := stmt.(*ast.RawStmt) + if !ok { + return nil, errors.New("node is not a statement") + } + rawSQL, err := source.Pluck(src, raw.StmtLocation, raw.StmtLen) + if err != nil { + return nil, err + } + if strings.TrimSpace(rawSQL) == "" { + return nil, errors.New("missing semicolon at end of file") + } + + name, cmd, err := metadata.ParseQueryNameAndType(rawSQL, metadata.CommentSyntax(c.parser.CommentSyntax())) + if err != nil { + return nil, err + } + if name == "" { + return nil, nil + } + if err := validate.Cmd(raw.Stmt, name, cmd); err != nil { + return nil, err + } + + md := metadata.Metadata{Name: name, Cmd: cmd} + cleanedComments, err := source.CleanedComments(rawSQL, c.parser.CommentSyntax()) + if err != nil { + return nil, err + } + md.Params, md.Flags, md.RuleSkiplist, err = metadata.ParseCommentFlags(cleanedComments) + if err != nil { + return nil, err + } + + var cols []*Column + var params []Parameter + // Only result-shaped statements produce columns/parameters. Non-SELECT + // statements (:exec, DDL) currently yield an empty shape; growing the + // core analyzer to cover INSERT/UPDATE/DELETE is a follow-up. + if _, ok := raw.Stmt.(*ast.SelectStmt); ok { + res, err := coreanalyzer.Prepare(c.coreCatalog, raw) + if err != nil { + return nil, err + } + for _, col := range res.Columns { + cols = append(cols, coreColumn(col)) + } + for _, p := range res.Parameters { + params = append(params, Parameter{Number: p.Number, Column: coreParamColumn(p)}) + } + } + + trimmed, comments, err := source.StripComments(rawSQL) + if err != nil { + return nil, err + } + md.Comments = comments + + var insertTable *ast.TableName + if ins, ok := raw.Stmt.(*ast.InsertStmt); ok { + insertTable, _ = ParseTableName(ins.Relation) + } + + return &Query{ + RawStmt: raw, + Metadata: md, + Params: params, + Columns: cols, + SQL: trimmed, + InsertIntoTable: insertTable, + }, nil +} + +// coreColumn maps a core.Column (analyzer output) onto the compiler's +// Column. The type name is carried in DataType; Type is left nil so the +// codegen shim uses DataType directly. +func coreColumn(c core.Column) *Column { + col := &Column{ + Name: c.Name, + DataType: c.DataType, + NotNull: c.NotNull, + } + if c.Source != nil && c.Source.Table != "" { + col.Table = &ast.TableName{Schema: c.Source.Schema, Name: c.Source.Table} + col.TableAlias = c.Source.TableAlias + col.OriginalName = c.Source.Column + } + if c.TypeLength > 0 { + l := c.TypeLength + col.Length = &l + } + return col +} + +func coreParamColumn(p core.Parameter) *Column { + col := &Column{ + Name: p.Name, + DataType: p.DataType, + NotNull: p.NotNull, + } + if p.Source != nil && p.Source.Table != "" { + col.Table = &ast.TableName{Schema: p.Source.Schema, Name: p.Source.Table} + col.OriginalName = p.Source.Column + } + return col +} diff --git a/internal/compiler/result.go b/internal/compiler/result.go index 3647da630f..e77933d555 100644 --- a/internal/compiler/result.go +++ b/internal/compiler/result.go @@ -1,10 +1,17 @@ package compiler import ( + "github.com/sqlc-dev/sqlc/internal/core" "github.com/sqlc-dev/sqlc/internal/sql/catalog" ) type Result struct { Catalog *catalog.Catalog Queries []*Query + + // CoreCatalog, when set, is the xqlc-derived catalog that drives both + // analysis and codegen for engines on the core (currently ClickHouse). + // When it is non-nil, codegen projects the catalog from it instead of + // from Catalog. + CoreCatalog *core.Catalog } diff --git a/internal/config/config.go b/internal/config/config.go index ff7faedcaa..6f0a9bf0eb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -54,6 +54,7 @@ const ( EngineMySQL Engine = "mysql" EnginePostgreSQL Engine = "postgresql" EngineSQLite Engine = "sqlite" + EngineClickHouse Engine = "clickhouse" ) type Config struct { diff --git a/internal/core/analyzer/scope.go b/internal/core/analyzer/scope.go index 3d688228a4..f7d78a1a23 100644 --- a/internal/core/analyzer/scope.go +++ b/internal/core/analyzer/scope.go @@ -24,8 +24,8 @@ type scopeRel struct { // scopeCol is a single column visible through a scopeRel. type scopeCol struct { - name string - attOID int64 + name string + attOID int64 typeOID int64 notNull bool } diff --git a/internal/core/attribute.go b/internal/core/attribute.go index 95e6cc7be4..540f87adb7 100644 --- a/internal/core/attribute.go +++ b/internal/core/attribute.go @@ -10,18 +10,18 @@ import ( // which is kept as a thin wrapper for callers that don't need to set // any of the "extra" metadata. type AttributeSpec struct { - ClassOID int64 - Name string - TypeOID int64 - Num int // 1-based ordinal position - NotNull bool - HasDefault bool - DeclType string // original declared type, before normalization - TypeLength int // varchar(N), numeric(p,s).p, etc. - TypeScale int // numeric(p,s).s - AutoIncrement bool - IsPrimaryKey bool - IsUnique bool + ClassOID int64 + Name string + TypeOID int64 + Num int // 1-based ordinal position + NotNull bool + HasDefault bool + DeclType string // original declared type, before normalization + TypeLength int // varchar(N), numeric(p,s).p, etc. + TypeScale int // numeric(p,s).s + AutoIncrement bool + IsPrimaryKey bool + IsUnique bool } // CreateAttributeSpec inserts a column with the full set of attribute @@ -94,18 +94,18 @@ func (c *Catalog) SetAttributeUnique(classOID int64, columns []string) error { // ColumnInfo describes a resolved column. type ColumnInfo struct { - Name string - TypeName string - TypeOID int64 - NotNull bool - DeclType string - TypeLength int - TypeScale int - AutoIncrement bool - IsPrimaryKey bool - IsUnique bool - AttributeOID int64 - ClassOID int64 + Name string + TypeName string + TypeOID int64 + NotNull bool + DeclType string + TypeLength int + TypeScale int + AutoIncrement bool + IsPrimaryKey bool + IsUnique bool + AttributeOID int64 + ClassOID int64 } // ResolveColumn looks up a column by table name and column name. @@ -174,17 +174,17 @@ func (c *Catalog) TableColumns(table string) ([]ColumnInfo, error) { // from sql_attribute joined back to sql_class / sql_namespace, so the // caller doesn't have to re-resolve names from raw OIDs. type AttributeDetails struct { - Schema string - Table string - Column string - Num int - DeclType string - TypeLength int - TypeScale int - AutoIncrement bool - IsPrimaryKey bool - IsUnique bool - NotNull bool + Schema string + Table string + Column string + Num int + DeclType string + TypeLength int + TypeScale int + AutoIncrement bool + IsPrimaryKey bool + IsUnique bool + NotNull bool } // LookupAttribute returns the full source-side metadata for a column, diff --git a/internal/core/cast_test.go b/internal/core/cast_test.go index 08c68b91c1..442ef4ebb6 100644 --- a/internal/core/cast_test.go +++ b/internal/core/cast_test.go @@ -31,14 +31,14 @@ func TestCastAllowed(t *testing.T) { ctx string want bool }{ - {intOID, intOID, "i", true}, // identity always OK - {intOID, bigintOID, "i", true}, // declared implicit - {intOID, bigintOID, "a", true}, // implicit subsumes assignment - {intOID, bigintOID, "e", true}, // implicit subsumes explicit - {intOID, textOID, "i", false}, // explicit-only blocked from implicit - {intOID, textOID, "a", false}, // and from assignment - {intOID, textOID, "e", true}, // but works for explicit - {textOID, intOID, "e", false}, // no rule defined + {intOID, intOID, "i", true}, // identity always OK + {intOID, bigintOID, "i", true}, // declared implicit + {intOID, bigintOID, "a", true}, // implicit subsumes assignment + {intOID, bigintOID, "e", true}, // implicit subsumes explicit + {intOID, textOID, "i", false}, // explicit-only blocked from implicit + {intOID, textOID, "a", false}, // and from assignment + {intOID, textOID, "e", true}, // but works for explicit + {textOID, intOID, "e", false}, // no rule defined } for _, c := range cases { got, err := cat.CastAllowed(c.src, c.tgt, c.ctx) diff --git a/internal/endtoend/testdata/clickhouse_select/clickhouse/db/db.go b/internal/endtoend/testdata/clickhouse_select/clickhouse/db/db.go new file mode 100644 index 0000000000..f43598b1eb --- /dev/null +++ b/internal/endtoend/testdata/clickhouse_select/clickhouse/db/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package db + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/clickhouse_select/clickhouse/db/models.go b/internal/endtoend/testdata/clickhouse_select/clickhouse/db/models.go new file mode 100644 index 0000000000..21f493ecbc --- /dev/null +++ b/internal/endtoend/testdata/clickhouse_select/clickhouse/db/models.go @@ -0,0 +1,5 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package db diff --git a/internal/endtoend/testdata/clickhouse_select/clickhouse/db/query.sql.go b/internal/endtoend/testdata/clickhouse_select/clickhouse/db/query.sql.go new file mode 100644 index 0000000000..19233a0b45 --- /dev/null +++ b/internal/endtoend/testdata/clickhouse_select/clickhouse/db/query.sql.go @@ -0,0 +1,52 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: query.sql + +package db + +import ( + "context" + "time" +) + +const listEvents = `-- name: ListEvents :many +SELECT id, name, tag, amount, created FROM events; +` + +type ListEventsRow struct { + ID uint64 + Name string + Tag *string + Amount float64 + Created time.Time +} + +func (q *Queries) ListEvents(ctx context.Context) ([]ListEventsRow, error) { + rows, err := q.db.QueryContext(ctx, listEvents) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListEventsRow + for rows.Next() { + var i ListEventsRow + if err := rows.Scan( + &i.ID, + &i.Name, + &i.Tag, + &i.Amount, + &i.Created, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/endtoend/testdata/clickhouse_select/clickhouse/query.sql b/internal/endtoend/testdata/clickhouse_select/clickhouse/query.sql new file mode 100644 index 0000000000..4c756ec0f8 --- /dev/null +++ b/internal/endtoend/testdata/clickhouse_select/clickhouse/query.sql @@ -0,0 +1,2 @@ +-- name: ListEvents :many +SELECT id, name, tag, amount, created FROM events; diff --git a/internal/endtoend/testdata/clickhouse_select/clickhouse/schema.sql b/internal/endtoend/testdata/clickhouse_select/clickhouse/schema.sql new file mode 100644 index 0000000000..29960ee63d --- /dev/null +++ b/internal/endtoend/testdata/clickhouse_select/clickhouse/schema.sql @@ -0,0 +1,7 @@ +CREATE TABLE events ( + id UInt64, + name String, + tag Nullable(String), + amount Float64, + created DateTime +) ENGINE = MergeTree ORDER BY id; diff --git a/internal/endtoend/testdata/clickhouse_select/clickhouse/sqlc.json b/internal/endtoend/testdata/clickhouse_select/clickhouse/sqlc.json new file mode 100644 index 0000000000..995102c9d6 --- /dev/null +++ b/internal/endtoend/testdata/clickhouse_select/clickhouse/sqlc.json @@ -0,0 +1,16 @@ +{ + "version": "2", + "sql": [ + { + "engine": "clickhouse", + "queries": "query.sql", + "schema": "schema.sql", + "gen": { + "go": { + "package": "db", + "out": "db" + } + } + } + ] +} diff --git a/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt b/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt index e2c49df3fa..28a5ce7e1f 100644 --- a/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt +++ b/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt @@ -1,5 +1,7 @@ [ { + "name": "GetValue", + "cmd": ":one", "ast": { "Stmt": { "DistinctClause": null, @@ -35,8 +37,8 @@ "Larg": null, "Rarg": null }, - "StmtLocation": 24, - "StmtLen": 0 + "StmtLocation": 0, + "StmtLen": 32 } } ] diff --git a/internal/engine/clickhouse/convert.go b/internal/engine/clickhouse/convert.go index f1fad62be7..aef7dd6419 100644 --- a/internal/engine/clickhouse/convert.go +++ b/internal/engine/clickhouse/convert.go @@ -486,8 +486,8 @@ func (c *cc) convertFunctionCall(n *chast.FunctionCall) *ast.FuncCall { Funcname: &ast.List{ Items: []ast.Node{&ast.String{Str: n.Name}}, }, - Location: n.Pos().Offset, - AggDistinct: n.Distinct, + Location: n.Pos().Offset, + AggDistinct: n.Distinct, } // Convert arguments diff --git a/internal/engine/clickhouse/parse.go b/internal/engine/clickhouse/parse.go index 282089f31d..83f9488dfe 100644 --- a/internal/engine/clickhouse/parse.go +++ b/internal/engine/clickhouse/parse.go @@ -29,36 +29,110 @@ func (p *Parser) Parse(r io.Reader) ([]ast.Statement, error) { return nil, err } + // doubleclick exposes each statement's start (Pos, 1-based) but not its + // end (End == Pos). Reconstruct byte spans the way the other engines do: + // StmtLocation is a running offset that starts at the end of the + // previous statement, so leading comments (including the sqlc "-- name:" + // annotation) fall inside the statement's span, and StmtLen runs through + // the terminating semicolon. var stmts []ast.Statement + loc := 0 for _, stmt := range stmtNodes { + start := stmt.Pos().Offset - 1 // Pos is 1-based; convert to a byte index + if start < loc { + start = loc + } + end := statementEnd(blob, start) + converter := &cc{} out := converter.convert(stmt) if _, ok := out.(*ast.TODO); ok { + loc = end continue } - // Get position information from the statement - pos := stmt.Pos() - end := stmt.End() - stmtLen := end.Offset - pos.Offset - stmts = append(stmts, ast.Statement{ Raw: &ast.RawStmt{ Stmt: out, - StmtLocation: pos.Offset, - StmtLen: stmtLen, + StmtLocation: loc, + StmtLen: end - loc, }, }) + loc = end } return stmts, nil } +// statementEnd returns the byte index just past the terminating semicolon +// of the statement beginning at start, or len(blob) if there is none. It +// skips over string / quoted-identifier literals and comments so that a +// semicolon inside them does not end the statement early. +func statementEnd(blob []byte, start int) int { + for i := start; i < len(blob); i++ { + switch blob[i] { + case '\'', '"', '`': + i = skipQuoted(blob, i) + case '-': + if i+1 < len(blob) && blob[i+1] == '-' { + i = skipLineComment(blob, i) + } + case '#': + i = skipLineComment(blob, i) + case '/': + if i+1 < len(blob) && blob[i+1] == '*' { + i = skipBlockComment(blob, i) + } + case ';': + return i + 1 + } + } + return len(blob) +} + +// skipQuoted returns the index of the closing quote of the literal that +// opens at i (blob[i] is the opening quote). Backslash escapes and doubled +// quotes are honored. +func skipQuoted(blob []byte, i int) int { + q := blob[i] + for j := i + 1; j < len(blob); j++ { + switch blob[j] { + case '\\': + j++ // skip the escaped byte + case q: + if j+1 < len(blob) && blob[j+1] == q { + j++ // doubled quote is an escaped quote + continue + } + return j + } + } + return len(blob) - 1 +} + +func skipLineComment(blob []byte, i int) int { + for j := i; j < len(blob); j++ { + if blob[j] == '\n' { + return j + } + } + return len(blob) - 1 +} + +func skipBlockComment(blob []byte, i int) int { + for j := i + 2; j < len(blob); j++ { + if blob[j] == '*' && j+1 < len(blob) && blob[j+1] == '/' { + return j + 1 + } + } + return len(blob) - 1 +} + // https://clickhouse.com/docs/en/sql-reference/syntax#comments func (p *Parser) CommentSyntax() source.CommentSyntax { return source.CommentSyntax{ - Dash: true, // -- comment - SlashStar: true, // /* comment */ - Hash: true, // # comment (ClickHouse supports this) + Dash: true, // -- comment + SlashStar: true, // /* comment */ + Hash: true, // # comment (ClickHouse supports this) } }