From 54d7b963c760f4398253228d13351f51b4588016 Mon Sep 17 00:00:00 2001 From: James Henderson Date: Wed, 6 Oct 2021 14:49:18 +0100 Subject: [PATCH 01/14] suppress "static" warnings on generated DUV source --- vector/src/main/codegen/templates/DenseUnionVector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/src/main/codegen/templates/DenseUnionVector.java b/vector/src/main/codegen/templates/DenseUnionVector.java index e9a9a2878b..e6f04a462a 100644 --- a/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/vector/src/main/codegen/templates/DenseUnionVector.java @@ -74,7 +74,7 @@ /* * This class is generated using freemarker and the ${.template_name} template. */ -@SuppressWarnings("unused") +@SuppressWarnings({"unused", "static"}) /** From 3f69adb39ef2724b01b5d6ff3cd94d775630d905 Mon Sep 17 00:00:00 2001 From: James Henderson Date: Wed, 6 Oct 2021 16:42:01 +0100 Subject: [PATCH 02/14] bug in DUV calling setNegative - expecting start + end not start + length --- vector/src/main/codegen/templates/DenseUnionVector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/src/main/codegen/templates/DenseUnionVector.java b/vector/src/main/codegen/templates/DenseUnionVector.java index e6f04a462a..59351cbdfe 100644 --- a/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/vector/src/main/codegen/templates/DenseUnionVector.java @@ -447,7 +447,7 @@ private void reallocTypeBuffer() { typeBuffer.getReferenceManager().release(1); typeBuffer = newBuf; typeBufferAllocationSizeInBytes = (int)newAllocationSize; - setNegative(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity); + setNegative(currentBufferCapacity, newBuf.capacity()); } private void reallocOffsetBuffer() { From b379fabb5701396f3b2de1b94d2dfe8bf7e3f86b Mon Sep 17 00:00:00 2001 From: James Henderson Date: Tue, 26 Jan 2021 18:18:59 +0000 Subject: [PATCH 03/14] Ensuring that DenseUnionVector.getField returns an up-to-date Field --- vector/src/main/codegen/templates/DenseUnionVector.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vector/src/main/codegen/templates/DenseUnionVector.java b/vector/src/main/codegen/templates/DenseUnionVector.java index 59351cbdfe..5e3be2890d 100644 --- a/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/vector/src/main/codegen/templates/DenseUnionVector.java @@ -522,7 +522,8 @@ public Field getField() { List childFields = new ArrayList<>(childCount); int[] typeIds = new int[childCount]; for (int i = 0; i < typeFields.length; i++) { - if (typeFields[i] != null) { + if (childVectors[i] != null) { + typeFields[i] = childVectors[i].getField(); int curIdx = childFields.size(); typeIds[curIdx] = i; childFields.add(typeFields[i]); From 8460d5ece26183f5bed3351a0e02376453247b55 Mon Sep 17 00:00:00 2001 From: James Henderson Date: Wed, 10 Feb 2021 17:12:55 +0000 Subject: [PATCH 04/14] Ensuring that we use the typeId mapping in DenseUnionVector$TransferImpl --- .../codegen/templates/DenseUnionVector.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/vector/src/main/codegen/templates/DenseUnionVector.java b/vector/src/main/codegen/templates/DenseUnionVector.java index 5e3be2890d..2bc7c92653 100644 --- a/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/vector/src/main/codegen/templates/DenseUnionVector.java @@ -624,12 +624,13 @@ public TransferImpl(DenseUnionVector to) { } private void createTransferPairs() { - for (int i = 0; i < nextTypeId; i++) { + for (byte i = 0; i < nextTypeId; i++) { + byte typeId = typeMapFields[i]; ValueVector srcVec = internalStruct.getVectorById(i); ValueVector dstVec = to.internalStruct.getVectorById(i); - to.typeFields[i] = typeFields[i]; + to.typeFields[typeId] = typeFields[typeId]; to.typeMapFields[i] = typeMapFields[i]; - to.childVectors[i] = dstVec; + to.childVectors[typeId] = dstVec; internalTransferPairs[i] = srcVec.makeTransferPair(dstVec); } } @@ -644,13 +645,15 @@ public void transfer() { refManager = offsetBuffer.getReferenceManager(); to.offsetBuffer = refManager.transferOwnership(offsetBuffer, to.allocator).getTransferredBuffer(); - for (int i = 0; i < nextTypeId; i++) { + for (byte i = 0; i < nextTypeId; i++) { + byte typeId = typeMapFields[i]; if (internalTransferPairs[i] != null) { internalTransferPairs[i].transfer(); - to.childVectors[i] = internalTransferPairs[i].getTo(); + to.childVectors[typeId] = internalTransferPairs[i].getTo(); } } to.valueCount = valueCount; + to.nextTypeId = nextTypeId; clear(); } @@ -670,8 +673,8 @@ public void splitAndTransfer(int startIndex, int length) { to.reallocOffsetBuffer(); } - int [] typeCounts = new int[nextTypeId]; - int [] typeStarts = new int[nextTypeId]; + int [] typeCounts = new int[Byte.MAX_VALUE + 1]; + int [] typeStarts = new int[Byte.MAX_VALUE + 1]; for (int i = 0; i < typeCounts.length; i++) { typeCounts[i] = 0; typeStarts[i] = -1; @@ -690,13 +693,15 @@ public void splitAndTransfer(int startIndex, int length) { // transfer vector values for (int i = 0; i < nextTypeId; i++) { - if (typeCounts[i] > 0 && typeStarts[i] != -1) { - internalTransferPairs[i].splitAndTransfer(typeStarts[i], typeCounts[i]); - to.childVectors[i] = internalTransferPairs[i].getTo(); + byte typeId = typeMapFields[i]; + if (typeCounts[typeId] > 0 && typeStarts[typeId] != -1) { + internalTransferPairs[i].splitAndTransfer(typeStarts[typeId], typeCounts[typeId]); + to.childVectors[typeId] = internalTransferPairs[i].getTo(); } } to.setValueCount(length); + to.nextTypeId = nextTypeId; } @Override From 06b9069f0b9293f168bdd13aae02f21c43d92f46 Mon Sep 17 00:00:00 2001 From: James Henderson Date: Wed, 13 Oct 2021 10:50:28 +0100 Subject: [PATCH 05/14] hack fix for out-of-bounds DUV error registering a new typeId on a vector with a typeIds mapping --- vector/src/main/codegen/templates/DenseUnionVector.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vector/src/main/codegen/templates/DenseUnionVector.java b/vector/src/main/codegen/templates/DenseUnionVector.java index 2bc7c92653..34d081666d 100644 --- a/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/vector/src/main/codegen/templates/DenseUnionVector.java @@ -236,9 +236,11 @@ public synchronized byte registerNewTypeId(Field field) { typeFields.length + " relative types. Please use union of union instead"); } byte typeId = nextTypeId; - if (this.fieldType != null) { - int[] typeIds = ((ArrowType.Union) this.fieldType.getType()).getTypeIds(); - if (typeIds != null) { + if (fieldType != null) { + int[] typeIds = ((ArrowType.Union) fieldType.getType()).getTypeIds(); + // HACK (JH) I'm not sure how this works for cases when you _do_ want a manual typeId mapping + // - I suspect some reused type ids :| + if (typeIds != null && nextTypeId < typeIds.length) { int thisTypeId = typeIds[nextTypeId]; if (thisTypeId > Byte.MAX_VALUE) { throw new IllegalStateException("Dense union vector types must be bytes. " + thisTypeId + " is too large"); From 57bf50b04e6cf1d46a0952d80c003cc5b149ebf0 Mon Sep 17 00:00:00 2001 From: James Henderson Date: Wed, 1 Jun 2022 15:51:46 +0100 Subject: [PATCH 06/14] DUV can't have be nullable - so we ensure the Field isn't marked nullable --- vector/src/main/codegen/templates/DenseUnionVector.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vector/src/main/codegen/templates/DenseUnionVector.java b/vector/src/main/codegen/templates/DenseUnionVector.java index 34d081666d..583f4d30c6 100644 --- a/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/vector/src/main/codegen/templates/DenseUnionVector.java @@ -534,11 +534,11 @@ public Field getField() { FieldType fieldType; if (this.fieldType == null) { - fieldType = FieldType.nullable(new ArrowType.Union(Dense, typeIds)); + fieldType = FieldType.notNullable(new ArrowType.Union(Dense, typeIds)); } else { final UnionMode mode = UnionMode.Dense; - fieldType = new FieldType(this.fieldType.isNullable(), new ArrowType.Union(mode, typeIds), - this.fieldType.getDictionary(), this.fieldType.getMetadata()); + fieldType = new FieldType(false, new ArrowType.Union(mode, typeIds), + this.fieldType.getDictionary(), this.fieldType.getMetadata()); } return new Field(name, fieldType, childFields); From db4b6772c178c2f4df784dbd257f822b51c89c2b Mon Sep 17 00:00:00 2001 From: James Henderson Date: Thu, 30 Jun 2022 17:42:29 +0100 Subject: [PATCH 07/14] DUV keeps a cache of typeCounts s.t. `setValueCount` isn't a linear scan of the typeBuffer --- .../codegen/templates/DenseUnionVector.java | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/vector/src/main/codegen/templates/DenseUnionVector.java b/vector/src/main/codegen/templates/DenseUnionVector.java index 583f4d30c6..7f6bfe8c74 100644 --- a/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/vector/src/main/codegen/templates/DenseUnionVector.java @@ -102,6 +102,12 @@ public class DenseUnionVector extends AbstractContainerVector implements FieldVe * The index is the type id, and the value is the type field. */ private Field[] typeFields = new Field[Byte.MAX_VALUE + 1]; + + /** + * The count of values for each type-id. + */ + private int[] typeCounts = new int[Byte.MAX_VALUE + 1]; + /** * The index is the index into the typeFields array, and the value is the logical field id. */ @@ -189,6 +195,12 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers offsetBufferAllocationSizeInBytes = offsetBuffer.capacity(); this.valueCount = fieldNode.getLength(); + + Arrays.fill(typeCounts, 0); + for (int i = 0; i < this.valueCount; i++) { + byte typeId = getTypeId(i); + if (typeId >= 0) typeCounts[typeId]++; + } } @Override @@ -503,6 +515,7 @@ public void close() { @Override public void clear() { valueCount = 0; + Arrays.fill(typeCounts, 0); typeBuffer.getReferenceManager().release(); typeBuffer = allocator.getEmpty(); offsetBuffer.getReferenceManager().release(); @@ -513,6 +526,7 @@ public void clear() { @Override public void reset() { valueCount = 0; + Arrays.fill(typeCounts, 0); setNegative(0, typeBuffer.capacity()); offsetBuffer.setZero(0, offsetBuffer.capacity()); internalStruct.reset(); @@ -654,6 +668,7 @@ public void transfer() { to.childVectors[typeId] = internalTransferPairs[i].getTo(); } } + to.typeCounts = typeCounts.clone(); to.valueCount = valueCount; to.nextTypeId = nextTypeId; clear(); @@ -701,9 +716,9 @@ public void splitAndTransfer(int startIndex, int length) { to.childVectors[typeId] = internalTransferPairs[i].getTo(); } } - - to.setValueCount(length); + to.typeCounts = typeCounts; to.nextTypeId = nextTypeId; + to.setValueCount(length); } @Override @@ -838,19 +853,9 @@ public void setValueCount(int valueCount) { reallocTypeBuffer(); reallocOffsetBuffer(); } - setChildVectorValueCounts(); - } - private void setChildVectorValueCounts() { - int [] counts = new int[Byte.MAX_VALUE + 1]; - for (int i = 0; i < this.valueCount; i++) { - byte typeId = getTypeId(i); - if (typeId != -1) { - counts[typeId] += 1; - } - } for (int i = 0; i < nextTypeId; i++) { - childVectors[typeMapFields[i]].setValueCount(counts[typeMapFields[i]]); + childVectors[typeMapFields[i]].setValueCount(typeCounts[typeMapFields[i]]); } } @@ -913,7 +918,11 @@ public void setTypeId(int index, byte typeId) { while (index >= getTypeBufferValueCapacity()) { reallocTypeBuffer(); } + int oldTypeId = getTypeId(index); + if (oldTypeId >= 0) typeCounts[oldTypeId]--; + typeBuffer.setByte(index * TYPE_WIDTH , typeId); + if (typeId >= 0) typeCounts[typeId]++; } private int getTypeBufferValueCapacity() { From c2b7f18be65306e2382f4d9baacafe98d6225257 Mon Sep 17 00:00:00 2001 From: James Henderson Date: Mon, 18 Jul 2022 15:48:39 +0100 Subject: [PATCH 08/14] suppress deprecation warning in `JsonFileWriter` because we compile with `-Werror` --- .../main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index 68700fe6af..18c815e8aa 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java +++ b/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -122,6 +122,7 @@ public JsonFileWriter(File outputFile) throws IOException { } /** Constructs a new writer that will output to outputFile with the given options. */ + @SuppressWarnings("deprecation") public JsonFileWriter(File outputFile, JSONWriteConfig config) throws IOException { MappingJsonFactory jsonFactory = new MappingJsonFactory(); this.generator = jsonFactory.createGenerator(outputFile, JsonEncoding.UTF8); From 1ccca57bcc7e8c735d458eacf88f3250aaf8bead Mon Sep 17 00:00:00 2001 From: James Henderson Date: Fri, 5 Aug 2022 12:20:25 +0100 Subject: [PATCH 09/14] include DENSEUNION in JsonFileWriter offset buffer writer switch --- .../main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index 18c815e8aa..91478c7de3 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java +++ b/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -425,6 +425,7 @@ private void writeValueToGenerator( case VARBINARY: case LIST: case MAP: + case DENSEUNION: generator.writeNumber(buffer.getInt((long) index * BaseVariableWidthVector.OFFSET_WIDTH)); break; case LISTVIEW: From fcead0ffcfc0a66a6519eec6ecc8c6b726f43355 Mon Sep 17 00:00:00 2001 From: James Henderson Date: Tue, 30 Aug 2022 15:50:08 +0100 Subject: [PATCH 10/14] fix JsonFileWriter for extension types - recurse with the underlying vector --- .../java/org/apache/arrow/vector/ipc/JsonFileWriter.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index 91478c7de3..1080b4988b 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java +++ b/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -45,6 +45,7 @@ import org.apache.arrow.vector.Decimal256Vector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.DurationVector; +import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; @@ -203,6 +204,11 @@ private void writeBatch(VectorSchemaRoot recordBatch) throws IOException { } private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOException { + if (vector instanceof ExtensionTypeVector) { + writeFromVectorIntoJson(field, ((ExtensionTypeVector) vector).getUnderlyingVector()); + return; + } + TypeLayout typeLayout = TypeLayout.getTypeLayout(field.getType()); List vectorTypes = typeLayout.getBufferTypes(); List vectorBuffers = vector.getFieldBuffers(); From 93ab072ae2c26d8471c7083ce7a3d1425ef33868 Mon Sep 17 00:00:00 2001 From: James Henderson Date: Thu, 27 Oct 2022 12:29:37 +0100 Subject: [PATCH 11/14] [DUV] only using the typeIds field when the Field itself specifies them, related to https://github.com/xtdb/core2/issues/245 --- vector/src/main/codegen/templates/DenseUnionVector.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vector/src/main/codegen/templates/DenseUnionVector.java b/vector/src/main/codegen/templates/DenseUnionVector.java index 7f6bfe8c74..6873bae487 100644 --- a/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/vector/src/main/codegen/templates/DenseUnionVector.java @@ -548,10 +548,11 @@ public Field getField() { FieldType fieldType; if (this.fieldType == null) { - fieldType = FieldType.notNullable(new ArrowType.Union(Dense, typeIds)); + fieldType = FieldType.notNullable(new ArrowType.Union(Dense, null)); } else { final UnionMode mode = UnionMode.Dense; - fieldType = new FieldType(false, new ArrowType.Union(mode, typeIds), + boolean indirectTypeIds = ((Union) this.fieldType.getType()).getTypeIds() != null; + fieldType = new FieldType(false, new ArrowType.Union(mode, indirectTypeIds ? typeIds : null), this.fieldType.getDictionary(), this.fieldType.getMetadata()); } From 18688a06a69ab1d8645ec3e43ad3d3aff1456ea5 Mon Sep 17 00:00:00 2001 From: James Henderson Date: Mon, 25 Mar 2024 14:59:51 +0000 Subject: [PATCH 12/14] add DUV.putVector so that we can promote DUV children --- .../src/main/codegen/templates/DenseUnionVector.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vector/src/main/codegen/templates/DenseUnionVector.java b/vector/src/main/codegen/templates/DenseUnionVector.java index 6873bae487..2b74539e57 100644 --- a/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/vector/src/main/codegen/templates/DenseUnionVector.java @@ -605,6 +605,18 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { copyFrom(inIndex, outIndex, from); } + FieldVector putVector(byte typeId, FieldVector child) { + internalStruct.putVector(child.getName(), child); + childVectors[typeId] = child; + typeFields[typeId] = child.getField(); + + if (callBack != null) { + callBack.doWork(); + } + + return child; + } + public FieldVector addVector(byte typeId, FieldVector v) { final String name = v.getName().isEmpty() ? fieldName(typeId, v.getMinorType()) : v.getName(); Preconditions.checkState(internalStruct.getChild(name) == null, String.format("%s vector already exists", name)); From 32fb320b45ffcea7c3cc34bcb35e3d7cf2dccbb5 Mon Sep 17 00:00:00 2001 From: James Henderson Date: Thu, 28 Mar 2024 11:52:31 +0000 Subject: [PATCH 13/14] add setTypeId to sparse UnionVector --- vector/src/main/codegen/templates/UnionVector.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vector/src/main/codegen/templates/UnionVector.java b/vector/src/main/codegen/templates/UnionVector.java index 67efdf60f7..8e403d2db7 100644 --- a/vector/src/main/codegen/templates/UnionVector.java +++ b/vector/src/main/codegen/templates/UnionVector.java @@ -848,10 +848,14 @@ public void setSafe(int index, Nullable${name}Holder holder) { public void setType(int index, MinorType type) { + setTypeId(index, (byte) type.ordinal()); + } + + public void setTypeId(int index, byte typeId) { while (index >= getTypeBufferValueCapacity()) { reallocTypeBuffer(); } - typeBuffer.setByte(index * TYPE_WIDTH , (byte) type.ordinal()); + typeBuffer.setByte(index * TYPE_WIDTH , typeId); } private int getTypeBufferValueCapacity() { From b2f93358f11bb774c43b48d4612371b419af8aac Mon Sep 17 00:00:00 2001 From: James Henderson Date: Thu, 12 Dec 2024 15:34:26 +0000 Subject: [PATCH 14/14] =?UTF-8?q?cut=20out=20O(n=C2=B2)=20operation=20in?= =?UTF-8?q?=20DUV.splitAndTransfer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../codegen/templates/DenseUnionVector.java | 47 ++++++++++++------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/vector/src/main/codegen/templates/DenseUnionVector.java b/vector/src/main/codegen/templates/DenseUnionVector.java index 2b74539e57..8dbf34ea87 100644 --- a/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/vector/src/main/codegen/templates/DenseUnionVector.java @@ -699,24 +699,36 @@ public void splitAndTransfer(int startIndex, int length) { to.typeBuffer = refManager.transferOwnership(slicedBuffer, to.allocator).getTransferredBuffer(); // transfer offset buffer - while (to.offsetBuffer.capacity() < (long) length * OFFSET_WIDTH) { - to.reallocOffsetBuffer(); - } - - int [] typeCounts = new int[Byte.MAX_VALUE + 1]; + int [] typeCounts; int [] typeStarts = new int[Byte.MAX_VALUE + 1]; - for (int i = 0; i < typeCounts.length; i++) { - typeCounts[i] = 0; - typeStarts[i] = -1; - } - for (int i = startIndex; i < startIndex + length; i++) { - byte typeId = typeBuffer.getByte(i); - if (typeId >= 0) { - to.offsetBuffer.setInt((long) (i - startIndex) * OFFSET_WIDTH, typeCounts[typeId]); - typeCounts[typeId] += 1; - if (typeStarts[typeId] == -1) { - typeStarts[typeId] = offsetBuffer.getInt((long) i * OFFSET_WIDTH); + if (startIndex == 0 && length == valueCount) { + // transferring the whole vector, skip the offset mapping + int[] srcTypeCounts = DenseUnionVector.this.typeCounts; + typeCounts = Arrays.copyOf(srcTypeCounts, srcTypeCounts.length); + + ArrowBuf slicedOffsetBuffer = offsetBuffer.slice(0, (long) length * OFFSET_WIDTH); + ReferenceManager offsetRefManager = slicedOffsetBuffer.getReferenceManager(); + to.offsetBuffer = offsetRefManager.transferOwnership(slicedOffsetBuffer, to.allocator).getTransferredBuffer(); + } else { + typeCounts = new int[Byte.MAX_VALUE + 1]; + + for (int i = 0; i < typeCounts.length; i++) { + typeStarts[i] = -1; + } + + while (to.offsetBuffer.capacity() < (long) length * OFFSET_WIDTH) { + to.reallocOffsetBuffer(); + } + + for (int i = startIndex; i < startIndex + length; i++) { + byte typeId = typeBuffer.getByte(i); + if (typeId >= 0) { + to.offsetBuffer.setInt((long) (i - startIndex) * OFFSET_WIDTH, typeCounts[typeId]); + typeCounts[typeId] += 1; + if (typeStarts[typeId] == -1) { + typeStarts[typeId] = offsetBuffer.getInt((long) i * OFFSET_WIDTH); + } } } } @@ -726,8 +738,9 @@ public void splitAndTransfer(int startIndex, int length) { byte typeId = typeMapFields[i]; if (typeCounts[typeId] > 0 && typeStarts[typeId] != -1) { internalTransferPairs[i].splitAndTransfer(typeStarts[typeId], typeCounts[typeId]); - to.childVectors[typeId] = internalTransferPairs[i].getTo(); } + + to.childVectors[typeId] = internalTransferPairs[i].getTo(); } to.typeCounts = typeCounts; to.nextTypeId = nextTypeId;