Skip to content

Commit 8fcdde1

Browse files
author
qiye
authored
[fix](Index)Make column unique ids in index dynamically computed (apache#48988)
Issue Number: close apache#48989 Related PR: apache#46648 apache#42882 Problem Summary: We introduced `Index.columnUniqueIds` in 3.0.3, and make sure `Index.columnUniqueIds` is not empty in 3.0.4. But when we upgrade Doris from 3.0.2 and before to 3.0.4, the `Index.columnUniqueIds` will be `null`. Versions 302 and earlier do not have this variable, and gson deserialization will assign it to null. We need to initialize the column unique ids associated with the index correctly. Key Changes Removed stored `columnUniqueIds` from `Index` class: Previously, column unique IDs were stored as a field in the `Index` class Now they're dynamically computed when needed rather than stored redundantly Added dynamic computation method: New method `getColumnUniqueIds(List<Column> schema)` computes IDs at runtime Column IDs are looked up by matching index column names with schema columns Updated method signatures: Changed `toThrift()` to `toThrift(List<Integer> indexColumnUniqueIds)` Changed `toPb()` to `toPb(Map<Integer, Column> columnMap, List<Integer> indexColumnUniqueIds)` Updated all callers to pass column IDs as parameters
1 parent 5e7dc1e commit 8fcdde1

17 files changed

Lines changed: 203 additions & 53 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2726,7 +2726,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
27262726
olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(),
27272727
olapTable.getInvertedIndexFileStorageFormat(),
27282728
disableInvertedIndexV1ForVariant);
2729-
indexDef.getColumnUniqueIds().add(column.getUniqueId());
27302729
} else {
27312730
throw new DdlException("index column does not exist in table. invalid column: " + col);
27322731
}
@@ -2737,7 +2736,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
27372736
// so here update column name in CreateIndexClause after checkColumn for indexDef,
27382737
// there will use the column name in olapTable instead of the column name in CreateIndexClause.
27392738
alterIndex.setColumns(indexDef.getColumns());
2740-
alterIndex.setColumnUniqueIds(indexDef.getColumnUniqueIds());
27412739
newIndexes.add(alterIndex);
27422740
return false;
27432741
}

‎fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
8686
indexDef.analyze();
8787
this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(),
8888
indexDef.getColumns(), indexDef.getIndexType(),
89-
indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds());
89+
indexDef.getProperties(), indexDef.getComment());
9090
}
9191

9292
@Override

‎fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
8282
indexDef.analyze();
8383
this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(),
8484
indexDef.getColumns(), indexDef.getIndexType(),
85-
indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds());
85+
indexDef.getProperties(), indexDef.getComment());
8686
}
8787

8888
@Override

‎fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -637,8 +637,7 @@ public void analyze(Analyzer analyzer) throws UserException {
637637
}
638638
}
639639
indexes.add(new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(),
640-
indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment(),
641-
indexDef.getColumnUniqueIds()));
640+
indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment()));
642641
distinct.add(indexDef.getIndexName());
643642
distinctCol.add(Pair.of(indexDef.getIndexType(),
644643
indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList())));

‎fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ public class IndexDef {
4343
private Map<String, String> properties;
4444
private boolean isBuildDeferred = false;
4545
private PartitionNames partitionNames;
46-
private List<Integer> columnUniqueIds = Lists.newArrayList();
4746
public static final int MIN_NGRAM_SIZE = 1;
4847
public static final int MAX_NGRAM_SIZE = 255;
4948
public static final int MIN_BF_SIZE = 64;
@@ -203,10 +202,6 @@ public List<String> getPartitionNames() {
203202
return partitionNames == null ? Lists.newArrayList() : partitionNames.getPartitionNames();
204203
}
205204

206-
public List<Integer> getColumnUniqueIds() {
207-
return columnUniqueIds;
208-
}
209-
210205
public enum IndexType {
211206
BITMAP,
212207
INVERTED,

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -65,19 +65,15 @@ public class Index implements Writable {
6565
private Map<String, String> properties;
6666
@SerializedName(value = "ct", alternate = {"comment"})
6767
private String comment;
68-
@SerializedName(value = "cui", alternate = {"columnUniqueIds"})
69-
private List<Integer> columnUniqueIds;
7068

7169
public Index(long indexId, String indexName, List<String> columns,
72-
IndexDef.IndexType indexType, Map<String, String> properties, String comment,
73-
List<Integer> columnUniqueIds) {
70+
IndexDef.IndexType indexType, Map<String, String> properties, String comment) {
7471
this.indexId = indexId;
7572
this.indexName = indexName;
7673
this.columns = columns == null ? Lists.newArrayList() : Lists.newArrayList(columns);
7774
this.indexType = indexType;
7875
this.properties = properties == null ? Maps.newHashMap() : Maps.newHashMap(properties);
7976
this.comment = comment;
80-
this.columnUniqueIds = columnUniqueIds == null ? Lists.newArrayList() : Lists.newArrayList(columnUniqueIds);
8177
if (indexType == IndexDef.IndexType.INVERTED) {
8278
if (this.properties != null && !this.properties.isEmpty()) {
8379
if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
@@ -101,7 +97,6 @@ public Index() {
10197
this.indexType = null;
10298
this.properties = null;
10399
this.comment = null;
104-
this.columnUniqueIds = null;
105100
}
106101

107102
public long getIndexId() {
@@ -191,14 +186,6 @@ public void setComment(String comment) {
191186
this.comment = comment;
192187
}
193188

194-
public List<Integer> getColumnUniqueIds() {
195-
return columnUniqueIds;
196-
}
197-
198-
public void setColumnUniqueIds(List<Integer> columnUniqueIds) {
199-
this.columnUniqueIds = columnUniqueIds;
200-
}
201-
202189
@Override
203190
public void write(DataOutput out) throws IOException {
204191
Text.writeString(out, GsonUtils.GSON.toJson(this));
@@ -216,7 +203,7 @@ public int hashCode() {
216203

217204
public Index clone() {
218205
return new Index(indexId, indexName, new ArrayList<>(columns),
219-
indexType, new HashMap<>(properties), comment, columnUniqueIds);
206+
indexType, new HashMap<>(properties), comment);
220207
}
221208

222209
@Override
@@ -251,7 +238,21 @@ public String toSql() {
251238
return sb.toString();
252239
}
253240

254-
public TOlapTableIndex toThrift() {
241+
public List<Integer> getColumnUniqueIds(List<Column> schema) {
242+
List<Integer> columnUniqueIds = new ArrayList<>();
243+
if (schema != null) {
244+
for (String columnName : columns) {
245+
for (Column column : schema) {
246+
if (columnName.equalsIgnoreCase(column.getName())) {
247+
columnUniqueIds.add(column.getUniqueId());
248+
}
249+
}
250+
}
251+
}
252+
return columnUniqueIds;
253+
}
254+
255+
public TOlapTableIndex toThrift(List<Integer> indexColumnUniqueIds) {
255256
TOlapTableIndex tIndex = new TOlapTableIndex();
256257
tIndex.setIndexId(indexId);
257258
tIndex.setIndexName(indexName);
@@ -260,15 +261,16 @@ public TOlapTableIndex toThrift() {
260261
if (properties != null) {
261262
tIndex.setProperties(properties);
262263
}
263-
tIndex.setColumnUniqueIds(columnUniqueIds);
264+
tIndex.setColumnUniqueIds(indexColumnUniqueIds);
264265
return tIndex;
265266
}
266267

267-
public OlapFile.TabletIndexPB toPb(Map<Integer, Column> columnMap) {
268+
public OlapFile.TabletIndexPB toPb(Map<Integer, Column> columnMap, List<Integer> indexColumnUniqueIds) {
268269
OlapFile.TabletIndexPB.Builder builder = OlapFile.TabletIndexPB.newBuilder();
269270
builder.setIndexId(indexId);
270271
builder.setIndexName(indexName);
271-
for (Integer columnUniqueId : columnUniqueIds) {
272+
273+
for (Integer columnUniqueId : indexColumnUniqueIds) {
272274
Column column = columnMap.get(columnUniqueId);
273275
if (column != null) {
274276
builder.addColUniqueId(column.getUniqueId());

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -391,13 +391,6 @@ public void initSchemaColumnUniqueId() {
391391
maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
392392
this.schema.forEach(column -> {
393393
column.setUniqueId(incAndGetMaxColUniqueId());
394-
this.indexes.forEach(index -> {
395-
index.getColumns().forEach(col -> {
396-
if (col.equalsIgnoreCase(column.getName())) {
397-
index.getColumnUniqueIds().add(column.getUniqueId());
398-
}
399-
});
400-
});
401394
if (LOG.isDebugEnabled()) {
402395
LOG.debug("indexId: {}, column:{}, uniqueId:{}",
403396
indexId, column, column.getUniqueId());

‎fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -327,9 +327,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId,
327327
columnMap.put(column.getUniqueId(), column);
328328
}
329329
if (indexes != null) {
330-
for (int i = 0; i < indexes.size(); i++) {
331-
Index index = indexes.get(i);
332-
schemaBuilder.addIndex(index.toPb(columnMap));
330+
for (Index index : indexes) {
331+
schemaBuilder.addIndex(index.toPb(columnMap, index.getColumnUniqueIds(schemaColumns)));
333332
}
334333
}
335334

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ public IndexType getIndexType() {
233233

234234
public Index translateToCatalogStyle() {
235235
return new Index(Env.getCurrentEnv().getNextId(), name, cols, indexType, properties,
236-
comment, null);
236+
comment);
237237
}
238238

239239
/**

‎fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1519,7 +1519,7 @@ protected void toThrift(TPlanNode msg) {
15191519
}
15201520

15211521
for (Index index : olapTable.getIndexes()) {
1522-
TOlapTableIndex tIndex = index.toThrift();
1522+
TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(olapTable.getBaseSchema()));
15231523
indexDesc.add(tIndex);
15241524
}
15251525

0 commit comments

Comments
 (0)