diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index 210c379e8373..c2ab4e331ef5 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -27,6 +27,7 @@ import com.google.cloud.bigquery.BigQuery.TableListOption; import com.google.cloud.bigquery.BigQuery.TableOption; import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.ColumnReference; import com.google.cloud.bigquery.Dataset; import com.google.cloud.bigquery.DatasetId; import com.google.cloud.bigquery.Field; @@ -34,6 +35,7 @@ import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.ForeignKey; import com.google.cloud.bigquery.PrimaryKey; import com.google.cloud.bigquery.Routine; import com.google.cloud.bigquery.RoutineArgument; @@ -43,7 +45,6 @@ import com.google.cloud.bigquery.StandardSQLField; import com.google.cloud.bigquery.StandardSQLTableType; import com.google.cloud.bigquery.StandardSQLTypeName; -import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableConstraints; import com.google.cloud.bigquery.TableDefinition; @@ -98,10 +99,7 @@ class BigQueryDatabaseMetaData implements DatabaseMetaData { private static final String SCHEMA_TERM = "Dataset"; private static final String CATALOG_TERM = "Project"; private static final String PROCEDURE_TERM = "Procedure"; - private static final String GET_PRIMARY_KEYS_SQL = "DatabaseMetaData_GetPrimaryKeys.sql"; - private static final String GET_IMPORTED_KEYS_SQL = "DatabaseMetaData_GetImportedKeys.sql"; private static final String GET_EXPORTED_KEYS_SQL = "DatabaseMetaData_GetExportedKeys.sql"; - private static final String GET_CROSS_REFERENCE_SQL = "DatabaseMetaData_GetCrossReference.sql"; private static final int DEFAULT_PAGE_SIZE = 500; private static final int DEFAULT_QUEUE_CAPACITY = 5000; // Declared package-private for testing. @@ -2451,10 +2449,7 @@ public ResultSet getPrimaryKeys(String catalog, String schema, String table) thr resultSchemaFields, ignoreAccessErrors, (bqTable, results, fields) -> { - TableConstraints constraints = null; - if (bqTable.getDefinition() instanceof StandardTableDefinition) { - constraints = ((StandardTableDefinition) bqTable.getDefinition()).getTableConstraints(); - } + TableConstraints constraints = bqTable.getTableConstraints(); processPrimaryKey(constraints, bqTable.getTableId(), results, fields); }); @@ -2529,16 +2524,47 @@ private Comparator defineGetPrimaryKeysComparator(FieldList resu @Override public ResultSet getImportedKeys(String catalog, String schema, String table) throws SQLException { - String sql = readSqlFromFile(GET_IMPORTED_KEYS_SQL); - Statement stmt = this.connection.createStatement(); - try { - stmt.closeOnCompletion(); - String formattedSql = replaceSqlParameters(sql, catalog, schema, table); - return stmt.executeQuery(formattedSql); - } catch (SQLException e) { - closeStatementIgnoreException(stmt); - throw new BigQueryJdbcException("Error executing getImportedKeys", e); + if ((catalog != null && catalog.isEmpty()) + || (schema != null && schema.isEmpty()) + || table == null + || table.isEmpty()) { + LOG.warning( + "Returning empty ResultSet as required parameters are null/empty, or catalog/schema parameters are empty."); + return new BigQueryJsonResultSet(); } + + final Schema resultSchema = defineForeignKeyResultSetSchema(); + final FieldList resultSchemaFields = resultSchema.getFields(); + + final List collectedResults = Collections.synchronizedList(new ArrayList<>()); + List targetDatasets = getTargetDatasets(catalog, schema); + + boolean ignoreAccessErrors = (catalog == null); + processTargetTablesConcurrently( + targetDatasets, + table, + collectedResults, + resultSchemaFields, + ignoreAccessErrors, + (bqTable, results, fields) -> { + TableConstraints constraints = bqTable.getTableConstraints(); + if (constraints == null || constraints.getForeignKeys() == null) { + return; + } + for (ForeignKey fk : constraints.getForeignKeys()) { + TableId pkTableId = fk.getReferencedTable(); + processForeignKey(fk, pkTableId, bqTable.getTableId(), results, fields); + } + }); + + Comparator comparator = definePkTableSortComparator(resultSchemaFields); + sortResults(collectedResults, comparator, "getImportedKeys", LOG); + + final BlockingQueue queue = + new LinkedBlockingQueue<>(DEFAULT_QUEUE_CAPACITY); + populateQueue(collectedResults, queue, resultSchemaFields); + signalEndOfData(queue, resultSchemaFields); + return BigQueryJsonResultSet.of(resultSchema, -1, queue, null); } @Override @@ -2565,24 +2591,56 @@ public ResultSet getCrossReference( String foreignSchema, String foreignTable) throws SQLException { - String sql = readSqlFromFile(GET_CROSS_REFERENCE_SQL); - Statement stmt = this.connection.createStatement(); - try { - stmt.closeOnCompletion(); - String formattedSql = - replaceSqlParameters( - sql, - parentCatalog, - parentSchema, - parentTable, - foreignCatalog, - foreignSchema, - foreignTable); - return stmt.executeQuery(formattedSql); - } catch (SQLException e) { - closeStatementIgnoreException(stmt); - throw new BigQueryJdbcException("Error executing getCrossReference", e); + if ((parentCatalog != null && parentCatalog.isEmpty()) + || (parentSchema != null && parentSchema.isEmpty()) + || parentTable == null + || parentTable.isEmpty() + || (foreignCatalog != null && foreignCatalog.isEmpty()) + || (foreignSchema != null && foreignSchema.isEmpty()) + || foreignTable == null + || foreignTable.isEmpty()) { + LOG.warning( + "Returning empty ResultSet as required parameters are null/empty, or catalog/schema parameters are empty."); + return new BigQueryJsonResultSet(); } + + final Schema resultSchema = defineForeignKeyResultSetSchema(); + final FieldList resultSchemaFields = resultSchema.getFields(); + + final List collectedResults = Collections.synchronizedList(new ArrayList<>()); + List targetDatasets = getTargetDatasets(foreignCatalog, foreignSchema); + + boolean ignoreAccessErrors = (foreignCatalog == null); + processTargetTablesConcurrently( + targetDatasets, + foreignTable, + collectedResults, + resultSchemaFields, + ignoreAccessErrors, + (bqTable, results, fields) -> { + TableConstraints constraints = bqTable.getTableConstraints(); + if (constraints == null || constraints.getForeignKeys() == null) { + return; + } + for (ForeignKey fk : constraints.getForeignKeys()) { + TableId pkTableId = fk.getReferencedTable(); + if (!equalsOrNullMatchesAll(parentCatalog, pkTableId.getProject()) + || !equalsOrNullMatchesAll(parentSchema, pkTableId.getDataset()) + || !parentTable.equals(pkTableId.getTable())) { + continue; + } + processForeignKey(fk, pkTableId, bqTable.getTableId(), results, fields); + } + }); + + Comparator comparator = defineFkTableSortComparator(resultSchemaFields); + sortResults(collectedResults, comparator, "getCrossReference", LOG); + + final BlockingQueue queue = + new LinkedBlockingQueue<>(DEFAULT_QUEUE_CAPACITY); + populateQueue(collectedResults, queue, resultSchemaFields); + signalEndOfData(queue, resultSchemaFields); + return BigQueryJsonResultSet.of(resultSchema, -1, queue, null); } @Override @@ -5004,6 +5062,10 @@ private List getAccessibleCatalogNames() throws SQLException { return sortedCatalogs; } + private boolean equalsOrNullMatchesAll(String expected, String actual) { + return expected == null || expected.equals(actual); + } + static String readSqlFromFile(String filename) { InputStream in; in = BigQueryDatabaseMetaData.class.getResourceAsStream(filename); @@ -5152,4 +5214,98 @@ private void processTargetTablesConcurrently( taskFutures.forEach(future -> future.cancel(true)); } } + + private Schema defineForeignKeyResultSetSchema() { + return Schema.of( + Field.of("PKTABLE_CAT", StandardSQLTypeName.STRING), + Field.of("PKTABLE_SCHEM", StandardSQLTypeName.STRING), + Field.of("PKTABLE_NAME", StandardSQLTypeName.STRING), + Field.of("PKCOLUMN_NAME", StandardSQLTypeName.STRING), + Field.of("FKTABLE_CAT", StandardSQLTypeName.STRING), + Field.of("FKTABLE_SCHEM", StandardSQLTypeName.STRING), + Field.of("FKTABLE_NAME", StandardSQLTypeName.STRING), + Field.of("FKCOLUMN_NAME", StandardSQLTypeName.STRING), + Field.of("KEY_SEQ", StandardSQLTypeName.INT64), + Field.of("UPDATE_RULE", StandardSQLTypeName.INT64), + Field.of("DELETE_RULE", StandardSQLTypeName.INT64), + Field.of("FK_NAME", StandardSQLTypeName.STRING), + Field.of("PK_NAME", StandardSQLTypeName.STRING), + Field.of("DEFERRABILITY", StandardSQLTypeName.INT64)); + } + + private void processForeignKey( + ForeignKey fk, + TableId pkTableId, + TableId fkTableId, + List collectedResults, + FieldList resultSchemaFields) { + if (pkTableId == null) { + LOG.warning( + String.format( + "Skipping foreign key '%s' on table '%s' because its referenced table ID is null.", + fk.getName() != null ? fk.getName() : "unnamed", fkTableId.getTable())); + return; + } + List colRefs = fk.getColumnReferences(); + if (colRefs != null) { + for (int i = 0; i < colRefs.size(); i++) { + ColumnReference colRef = colRefs.get(i); + List row = new ArrayList<>(); + row.add(createStringFieldValue(pkTableId.getProject())); // PKTABLE_CAT + row.add(createStringFieldValue(pkTableId.getDataset())); // PKTABLE_SCHEM + row.add(createStringFieldValue(pkTableId.getTable())); // PKTABLE_NAME + row.add(createStringFieldValue(colRef.getReferencedColumn())); // PKCOLUMN_NAME + row.add(createStringFieldValue(fkTableId.getProject())); // FKTABLE_CAT + row.add(createStringFieldValue(fkTableId.getDataset())); // FKTABLE_SCHEM + row.add(createStringFieldValue(fkTableId.getTable())); // FKTABLE_NAME + row.add(createStringFieldValue(colRef.getReferencingColumn())); // FKCOLUMN_NAME + row.add(createLongFieldValue((long) (i + 1))); // KEY_SEQ + row.add(createNullFieldValue()); // UPDATE_RULE + row.add(createNullFieldValue()); // DELETE_RULE + row.add(createStringFieldValue(fk.getName())); // FK_NAME + row.add(createNullFieldValue()); // PK_NAME + row.add(createNullFieldValue()); // DEFERRABILITY + + collectedResults.add(FieldValueList.of(row, resultSchemaFields)); + } + } + } + + private Comparator definePkTableSortComparator(FieldList resultSchemaFields) { + final int PKTABLE_CAT_IDX = resultSchemaFields.getIndex("PKTABLE_CAT"); + final int PKTABLE_SCHEM_IDX = resultSchemaFields.getIndex("PKTABLE_SCHEM"); + final int PKTABLE_NAME_IDX = resultSchemaFields.getIndex("PKTABLE_NAME"); + final int KEY_SEQ_IDX = resultSchemaFields.getIndex("KEY_SEQ"); + return Comparator.comparing( + (FieldValueList fvl) -> getStringValueOrNull(fvl, PKTABLE_CAT_IDX), + Comparator.nullsFirst(String::compareTo)) + .thenComparing( + (FieldValueList fvl) -> getStringValueOrNull(fvl, PKTABLE_SCHEM_IDX), + Comparator.nullsFirst(String::compareTo)) + .thenComparing( + (FieldValueList fvl) -> getStringValueOrNull(fvl, PKTABLE_NAME_IDX), + Comparator.nullsFirst(String::compareTo)) + .thenComparing( + (FieldValueList fvl) -> getLongValueOrNull(fvl, KEY_SEQ_IDX), + Comparator.nullsFirst(Long::compareTo)); + } + + private Comparator defineFkTableSortComparator(FieldList resultSchemaFields) { + final int FKTABLE_CAT_IDX = resultSchemaFields.getIndex("FKTABLE_CAT"); + final int FKTABLE_SCHEM_IDX = resultSchemaFields.getIndex("FKTABLE_SCHEM"); + final int FKTABLE_NAME_IDX = resultSchemaFields.getIndex("FKTABLE_NAME"); + final int KEY_SEQ_IDX = resultSchemaFields.getIndex("KEY_SEQ"); + return Comparator.comparing( + (FieldValueList fvl) -> getStringValueOrNull(fvl, FKTABLE_CAT_IDX), + Comparator.nullsFirst(String::compareTo)) + .thenComparing( + (FieldValueList fvl) -> getStringValueOrNull(fvl, FKTABLE_SCHEM_IDX), + Comparator.nullsFirst(String::compareTo)) + .thenComparing( + (FieldValueList fvl) -> getStringValueOrNull(fvl, FKTABLE_NAME_IDX), + Comparator.nullsFirst(String::compareTo)) + .thenComparing( + (FieldValueList fvl) -> getLongValueOrNull(fvl, KEY_SEQ_IDX), + Comparator.nullsFirst(Long::compareTo)); + } } diff --git a/java-bigquery-jdbc/src/main/resources/com/google/cloud/bigquery/jdbc/DatabaseMetaData_GetCrossReference.sql b/java-bigquery-jdbc/src/main/resources/com/google/cloud/bigquery/jdbc/DatabaseMetaData_GetCrossReference.sql deleted file mode 100644 index da8386270457..000000000000 --- a/java-bigquery-jdbc/src/main/resources/com/google/cloud/bigquery/jdbc/DatabaseMetaData_GetCrossReference.sql +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -SELECT PKTABLE_CAT, - PKTABLE_SCHEM, - PKTABLE_NAME, - PRIMARY.column_name AS PKCOLUMN_NAME, - FOREIGN.constraint_catalog AS FKTABLE_CAT, - FOREIGN.constraint_schema AS FKTABLE_SCHEM, - FOREIGN.table_name AS FKTABLE_NAME, - FOREIGN.column_name AS FKCOLUMN_NAME, - FOREIGN.ordinal_position AS KEY_SEQ, - NULL AS UPDATE_RULE, - NULL AS DELETE_RULE, - FOREIGN.constraint_name AS FK_NAME, - PRIMARY.constraint_name AS PK_NAME, - NULL AS DEFERRABILITY -FROM (SELECT DISTINCT CCU.table_catalog AS PKTABLE_CAT, - CCU.table_schema AS PKTABLE_SCHEM, - CCU.table_name AS PKTABLE_NAME, - TC.constraint_catalog, - TC.constraint_schema, - TC.constraint_name, - TC.table_catalog, - TC.table_schema, - TC.table_name, - TC.constraint_type, - KCU.column_name, - KCU.ordinal_position, - KCU.position_in_unique_constraint - FROM `%1$s.%2$s.INFORMATION_SCHEMA.TABLE_CONSTRAINTS` TC - INNER JOIN - `%1$s.%2$s.INFORMATION_SCHEMA.KEY_COLUMN_USAGE` KCU - USING - (constraint_catalog, - constraint_schema, - constraint_name, - table_catalog, - table_schema, - table_name) - INNER JOIN - `%1$s.%2$s.INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE` CCU - USING - (constraint_catalog, - constraint_schema, - constraint_name) - WHERE constraint_type = 'FOREIGN KEY' - AND TC.table_name = '%6$s') FOREIGN - INNER JOIN (SELECT * - FROM `%1$s.%2$s.INFORMATION_SCHEMA.KEY_COLUMN_USAGE` - WHERE position_in_unique_constraint IS NULL - AND RTRIM(table_name) = '%3$s') PRIMARY -ON - FOREIGN.PKTABLE_CAT = PRIMARY.table_catalog - AND FOREIGN.PKTABLE_SCHEM = PRIMARY.table_schema - AND FOREIGN.PKTABLE_NAME = PRIMARY.table_name - AND FOREIGN.position_in_unique_constraint = - PRIMARY.ordinal_position -ORDER BY FKTABLE_CAT, FKTABLE_SCHEM, FKTABLE_NAME, KEY_SEQ \ No newline at end of file diff --git a/java-bigquery-jdbc/src/main/resources/com/google/cloud/bigquery/jdbc/DatabaseMetaData_GetImportedKeys.sql b/java-bigquery-jdbc/src/main/resources/com/google/cloud/bigquery/jdbc/DatabaseMetaData_GetImportedKeys.sql deleted file mode 100644 index 3f4142eb051f..000000000000 --- a/java-bigquery-jdbc/src/main/resources/com/google/cloud/bigquery/jdbc/DatabaseMetaData_GetImportedKeys.sql +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -SELECT PKTABLE_CAT, - PKTABLE_SCHEM, - PKTABLE_NAME, - PRIMARY.column_name AS PKCOLUMN_NAME, - FOREIGN.constraint_catalog AS FKTABLE_CAT, - FOREIGN.constraint_schema AS FKTABLE_SCHEM, - FOREIGN.table_name AS FKTABLE_NAME, - FOREIGN.column_name AS FKCOLUMN_NAME, - FOREIGN.ordinal_position AS KEY_SEQ, - NULL AS UPDATE_RULE, - NULL AS DELETE_RULE, - FOREIGN.constraint_name AS FK_NAME, - PRIMARY.constraint_name AS PK_NAME, - NULL AS DEFERRABILITY -FROM (SELECT DISTINCT CCU.table_catalog AS PKTABLE_CAT, - CCU.table_schema AS PKTABLE_SCHEM, - CCU.table_name AS PKTABLE_NAME, - TC.constraint_catalog, - TC.constraint_schema, - TC.constraint_name, - TC.table_catalog, - TC.table_schema, - TC.table_name, - TC.constraint_type, - KCU.column_name, - KCU.ordinal_position, - KCU.position_in_unique_constraint - FROM `%1$s.%2$s.INFORMATION_SCHEMA.TABLE_CONSTRAINTS` TC - INNER JOIN - `%1$s.%2$s.INFORMATION_SCHEMA.KEY_COLUMN_USAGE` KCU - USING - (constraint_catalog, - constraint_schema, - constraint_name, - table_catalog, - table_schema, - table_name) - INNER JOIN - `%1$s.%2$s.INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE` CCU - USING - (constraint_catalog, - constraint_schema, - constraint_name) - WHERE constraint_type = 'FOREIGN KEY' - AND TC.table_name = '%3$s') FOREIGN - INNER JOIN (SELECT * - FROM `%1$s.%2$s.INFORMATION_SCHEMA.KEY_COLUMN_USAGE` - WHERE position_in_unique_constraint IS NULL) PRIMARY -ON - FOREIGN.PKTABLE_CAT = PRIMARY.table_catalog - AND FOREIGN.PKTABLE_SCHEM = PRIMARY.table_schema - AND FOREIGN.PKTABLE_NAME = PRIMARY.table_name - AND FOREIGN.position_in_unique_constraint = - PRIMARY.ordinal_position -ORDER BY PKTABLE_CAT, PKTABLE_SCHEM, PKTABLE_NAME, KEY_SEQ \ No newline at end of file