(
+ "pipe_memory_allocate_for_ts_file_sequence_reader_in_bytes", 2 * 1024 * 1024L) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Long.parseLong(valueString);
+ }
+ };
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java
index 8ace81c5..07c60a20 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java
@@ -49,6 +49,11 @@ public long getCreationTime() {
return creationTime;
}
+ @Override
+ public int getRegionId() {
+ return getInstanceIndex();
+ }
+
public int getParallelism() {
return parallelism;
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/annotation/TableModel.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/annotation/TableModel.java
new file mode 100644
index 00000000..f73a1af0
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/annotation/TableModel.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Indicates that a plugin can be used in table model environments.
+ *
+ * When implementing a custom {@link org.apache.iotdb.pipe.api.PipePlugin} that needs to operate
+ * under table model settings, declare this annotation on the plugin class. Through the {@code
+ * CREATE PIPEPLUGIN} statement, a plugin annotated with {@link TableModel} is valid for both tree
+ * model connections and table model connections.
+ *
+ * @since 2.0.0
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface TableModel {}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/annotation/TreeModel.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/annotation/TreeModel.java
new file mode 100644
index 00000000..7e163fe7
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/annotation/TreeModel.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Indicates that a plugin can be used in tree model environments.
+ *
+ *
When implementing a custom {@link org.apache.iotdb.pipe.api.PipePlugin} that needs to operate
+ * under tree model settings, declare this annotation on the plugin class. Through the {@code CREATE
+ * PIPEPLUGIN} statement, a plugin annotated with {@link TreeModel} is valid for both tree model
+ * connections and tree model connections.
+ *
+ * @since 2.0.0
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface TreeModel {}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressor.java
new file mode 100644
index 00000000..dc3f1e79
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressor.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import java.io.IOException;
+
+public abstract class PipeCompressor {
+
+ public enum PipeCompressionType {
+ SNAPPY((byte) 0),
+ GZIP((byte) 1),
+ LZ4((byte) 2),
+ ZSTD((byte) 3),
+ LZMA2((byte) 4);
+
+ final byte index;
+
+ PipeCompressionType(byte index) {
+ this.index = index;
+ }
+
+ public byte getIndex() {
+ return index;
+ }
+ }
+
+ private final PipeCompressionType compressionType;
+
+ protected PipeCompressor(PipeCompressionType compressionType) {
+ this.compressionType = compressionType;
+ }
+
+ public abstract byte[] compress(byte[] data) throws IOException;
+
+ /**
+ * Decompress the byte array to a byte array. NOTE: the length of the decompressed byte array is
+ * not provided in this method, and some decompressors (LZ4) may construct large byte arrays,
+ * leading to potential OOM.
+ *
+ * @param byteArray the byte array to be decompressed
+ * @return the decompressed byte array
+ * @throws IOException
+ */
+ public abstract byte[] decompress(byte[] byteArray) throws IOException;
+
+ /**
+ * Decompress the byte array to a byte array with a known length.
+ *
+ * @param byteArray the byte array to be decompressed
+ * @param decompressedLength the length of the decompressed byte array
+ * @return the decompressed byte array
+ * @throws IOException
+ */
+ public abstract byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException;
+
+ public byte serialize() {
+ return compressionType.getIndex();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorConfig.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorConfig.java
new file mode 100644
index 00000000..fc54f3af
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorConfig.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+public class PipeCompressorConfig {
+
+ private final String name;
+ private final int zstdCompressionLevel;
+
+ public PipeCompressorConfig(String name, int zstdCompressionLevel) {
+ this.name = name;
+ this.zstdCompressionLevel = zstdCompressionLevel;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getZstdCompressionLevel() {
+ return zstdCompressionLevel;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorFactory.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorFactory.java
new file mode 100644
index 00000000..df941f48
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorFactory.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_GZIP;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_LZ4;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_LZMA2;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_SNAPPY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE;
+
+public class PipeCompressorFactory {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeCompressorFactory.class);
+
+ private static final Map COMPRESSOR_NAME_TO_INSTANCE =
+ new ConcurrentHashMap<>();
+
+ static {
+ COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_SNAPPY, new PipeSnappyCompressor());
+ COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_GZIP, new PipeGZIPCompressor());
+ COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_LZ4, new PipeLZ4Compressor());
+ COMPRESSOR_NAME_TO_INSTANCE.put(
+ CONNECTOR_COMPRESSOR_ZSTD,
+ new PipeZSTDCompressor(CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE));
+ COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_LZMA2, new PipeLZMA2Compressor());
+ }
+
+ public static PipeCompressor getCompressor(PipeCompressorConfig config) {
+ if (config == null) {
+ throw new IllegalArgumentException("PipeCompressorConfig is null");
+ }
+ if (config.getName() == null) {
+ throw new IllegalArgumentException("PipeCompressorConfig.getName() is null");
+ }
+
+ final String compressorName = config.getName();
+
+ // For ZSTD compressor, we need to consider the compression level
+ if (compressorName.equals(CONNECTOR_COMPRESSOR_ZSTD)) {
+ final int zstdCompressionLevel = config.getZstdCompressionLevel();
+ return COMPRESSOR_NAME_TO_INSTANCE.computeIfAbsent(
+ CONNECTOR_COMPRESSOR_ZSTD + "_" + zstdCompressionLevel,
+ key -> {
+ LOGGER.info("Create new PipeZSTDCompressor with level: {}", zstdCompressionLevel);
+ return new PipeZSTDCompressor(zstdCompressionLevel);
+ });
+ }
+
+ // For other compressors, we can directly get the instance by name
+ final PipeCompressor compressor = COMPRESSOR_NAME_TO_INSTANCE.get(compressorName);
+ if (compressor != null) {
+ return compressor;
+ }
+
+ throw new UnsupportedOperationException("PipeCompressor not found for name: " + compressorName);
+ }
+
+ private static Map COMPRESSOR_INDEX_TO_INSTANCE = new HashMap<>();
+
+ static {
+ COMPRESSOR_INDEX_TO_INSTANCE.put(
+ PipeCompressor.PipeCompressionType.SNAPPY.getIndex(),
+ COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_SNAPPY));
+ COMPRESSOR_INDEX_TO_INSTANCE.put(
+ PipeCompressor.PipeCompressionType.GZIP.getIndex(),
+ COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_GZIP));
+ COMPRESSOR_INDEX_TO_INSTANCE.put(
+ PipeCompressor.PipeCompressionType.LZ4.getIndex(),
+ COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_LZ4));
+ COMPRESSOR_INDEX_TO_INSTANCE.put(
+ PipeCompressor.PipeCompressionType.ZSTD.getIndex(),
+ COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_ZSTD));
+ COMPRESSOR_INDEX_TO_INSTANCE.put(
+ PipeCompressor.PipeCompressionType.LZMA2.getIndex(),
+ COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_LZMA2));
+ COMPRESSOR_INDEX_TO_INSTANCE = Collections.unmodifiableMap(COMPRESSOR_INDEX_TO_INSTANCE);
+ }
+
+ public static PipeCompressor getCompressor(byte index) {
+ final PipeCompressor compressor = COMPRESSOR_INDEX_TO_INSTANCE.get(index);
+ if (compressor == null) {
+ throw new UnsupportedOperationException("PipeCompressor not found for index: " + index);
+ }
+ return compressor;
+ }
+
+ private PipeCompressorFactory() {
+ // Empty constructor
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeGZIPCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeGZIPCompressor.java
new file mode 100644
index 00000000..155af20e
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeGZIPCompressor.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import java.io.IOException;
+import org.apache.tsfile.compress.ICompressor;
+import org.apache.tsfile.compress.IUnCompressor;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+
+public class PipeGZIPCompressor extends PipeCompressor {
+
+ private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.GZIP);
+ private static final IUnCompressor DECOMPRESSOR =
+ IUnCompressor.getUnCompressor(CompressionType.GZIP);
+
+ public PipeGZIPCompressor() {
+ super(PipeCompressionType.GZIP);
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ return COMPRESSOR.compress(data);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray) throws IOException {
+ return DECOMPRESSOR.uncompress(byteArray);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException {
+ byte[] uncompressed = new byte[decompressedLength];
+ DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0);
+ return uncompressed;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZ4Compressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZ4Compressor.java
new file mode 100644
index 00000000..2c0a5297
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZ4Compressor.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import java.io.IOException;
+import org.apache.tsfile.compress.ICompressor;
+import org.apache.tsfile.compress.IUnCompressor;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+
+public class PipeLZ4Compressor extends PipeCompressor {
+
+ private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.LZ4);
+ private static final IUnCompressor DECOMPRESSOR =
+ IUnCompressor.getUnCompressor(CompressionType.LZ4);
+
+ public PipeLZ4Compressor() {
+ super(PipeCompressionType.LZ4);
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ return COMPRESSOR.compress(data);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray) throws IOException {
+ return DECOMPRESSOR.uncompress(byteArray);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException {
+ byte[] uncompressed = new byte[decompressedLength];
+ DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0);
+ return uncompressed;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZMA2Compressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZMA2Compressor.java
new file mode 100644
index 00000000..e964c951
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZMA2Compressor.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import java.io.IOException;
+import org.apache.tsfile.compress.ICompressor;
+import org.apache.tsfile.compress.IUnCompressor;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+
+public class PipeLZMA2Compressor extends PipeCompressor {
+
+ private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.LZMA2);
+ private static final IUnCompressor DECOMPRESSOR =
+ IUnCompressor.getUnCompressor(CompressionType.LZMA2);
+
+ public PipeLZMA2Compressor() {
+ super(PipeCompressionType.LZMA2);
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ return COMPRESSOR.compress(data);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray) throws IOException {
+ return DECOMPRESSOR.uncompress(byteArray);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException {
+ byte[] uncompressed = new byte[decompressedLength];
+ DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0);
+ return uncompressed;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeSnappyCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeSnappyCompressor.java
new file mode 100644
index 00000000..7e84db51
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeSnappyCompressor.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import java.io.IOException;
+import org.apache.tsfile.compress.ICompressor;
+import org.apache.tsfile.compress.IUnCompressor;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+
+public class PipeSnappyCompressor extends PipeCompressor {
+
+ private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.SNAPPY);
+ private static final IUnCompressor DECOMPRESSOR =
+ IUnCompressor.getUnCompressor(CompressionType.SNAPPY);
+
+ public PipeSnappyCompressor() {
+ super(PipeCompressionType.SNAPPY);
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ return COMPRESSOR.compress(data);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray) throws IOException {
+ return DECOMPRESSOR.uncompress(byteArray);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException {
+ byte[] uncompressed = new byte[decompressedLength];
+ DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0);
+ return uncompressed;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeZSTDCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeZSTDCompressor.java
new file mode 100644
index 00000000..25687705
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeZSTDCompressor.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import com.github.luben.zstd.Zstd;
+import java.io.IOException;
+
+public class PipeZSTDCompressor extends PipeCompressor {
+
+ private final int compressionLevel;
+
+ public PipeZSTDCompressor(int compressionLevel) {
+ super(PipeCompressionType.ZSTD);
+ this.compressionLevel = compressionLevel;
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ return Zstd.compress(data, compressionLevel);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray) {
+ return Zstd.decompress(byteArray, (int) Zstd.decompressedSize(byteArray, 0, byteArray.length));
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray, int decompressedLength) {
+ return Zstd.decompress(byteArray, decompressedLength);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/ColumnHeaderConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/ColumnHeaderConstant.java
new file mode 100644
index 00000000..23a4b42c
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/ColumnHeaderConstant.java
@@ -0,0 +1,649 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.constant;
+
+
+public class ColumnHeaderConstant {
+
+ private ColumnHeaderConstant() {
+ // forbidding instantiation
+ }
+
+ // column names for query statement
+ public static final String TIME = "Time";
+ public static final String ENDTIME = "__endTime";
+ public static final String VALUE = "Value";
+ public static final String DEVICE = "Device";
+ public static final String EXPLAIN_ANALYZE = "Explain Analyze";
+
+ // column names for schema statement
+ public static final String DATABASE = "Database";
+ public static final String TIMESERIES = "Timeseries";
+ public static final String ALIAS = "Alias";
+ public static final String DATATYPE = "DataType";
+ public static final String ENCODING = "Encoding";
+ public static final String COMPRESSION = "Compression";
+ public static final String TAGS = "Tags";
+ public static final String ATTRIBUTES = "Attributes";
+ public static final String NOTES = "Notes";
+ public static final String DEADBAND = "Deadband";
+ public static final String DEADBAND_PARAMETERS = "DeadbandParameters";
+ public static final String IS_ALIGNED = "IsAligned";
+ public static final String TEMPLATE = "Template";
+
+ public static final String COUNT = "Count";
+ public static final String COLUMN_TTL = "TTL(ms)";
+ public static final String SCHEMA_REPLICATION_FACTOR = "SchemaReplicationFactor";
+ public static final String DATA_REPLICATION_FACTOR = "DataReplicationFactor";
+ public static final String TIME_PARTITION_ORIGIN = "TimePartitionOrigin";
+ public static final String TIME_PARTITION_INTERVAL = "TimePartitionInterval";
+ public static final String SCHEMA_REGION_GROUP_NUM = "SchemaRegionGroupNum";
+ public static final String MIN_SCHEMA_REGION_GROUP_NUM = "MinSchemaRegionGroupNum";
+ public static final String MAX_SCHEMA_REGION_GROUP_NUM = "MaxSchemaRegionGroupNum";
+ public static final String DATA_REGION_GROUP_NUM = "DataRegionGroupNum";
+ public static final String MIN_DATA_REGION_GROUP_NUM = "MinDataRegionGroupNum";
+ public static final String MAX_DATA_REGION_GROUP_NUM = "MaxDataRegionGroupNum";
+ public static final String CHILD_PATHS = "ChildPaths";
+ public static final String NODE_TYPES = "NodeTypes";
+ public static final String CHILD_NODES = "ChildNodes";
+ public static final String VERSION = "Version";
+ public static final String BUILD_INFO = "BuildInfo";
+ public static final String PATHS = "Paths";
+ public static final String PATH = "Path";
+ public static final String VARIABLE = "Variable";
+ public static final String SCOPE = "Scope";
+
+ // column names for count statement
+ public static final String COLUMN = "Column";
+ public static final String COUNT_DEVICES = "count(devices)";
+ public static final String COUNT_NODES = "count(nodes)";
+ public static final String COUNT_TIMESERIES = "count(timeseries)";
+ public static final String COUNT_DATABASE = "count(database)";
+
+ // column names for show cluster and show cluster details statements
+ public static final String NODE_ID = "NodeID";
+ public static final String NODE_TYPE = "NodeType";
+ public static final String STATUS = "Status";
+ public static final String INTERNAL_ADDRESS = "InternalAddress";
+ public static final String INTERNAL_PORT = "InternalPort";
+ public static final String CONFIG_CONSENSUS_PORT = "ConfigConsensusPort";
+ public static final String RPC_ADDRESS = "RpcAddress";
+ public static final String RPC_PORT = "RpcPort";
+ public static final String DATA_CONSENSUS_PORT = "DataConsensusPort";
+ public static final String SCHEMA_CONSENSUS_PORT = "SchemaConsensusPort";
+ public static final String MPP_PORT = "MppPort";
+
+ // column names for show clusterId statement
+ public static final String CLUSTER_ID = "ClusterId";
+
+ // column names for verify connection statement
+ public static final String SERVICE_PROVIDER = "ServiceProvider";
+ public static final String SENDER = "Sender";
+ public static final String CONNECTION = "Connection";
+
+ // column names for show functions statement
+ public static final String FUNCTION_NAME = "FunctionName";
+ public static final String FUNCTION_TYPE = "FunctionType";
+ public static final String CLASS_NAME_UDF = "ClassName(UDF)";
+ public static final String FUNCTION_STATE = "State";
+
+ // column names for show triggers statement
+ public static final String TRIGGER_NAME = "TriggerName";
+ public static final String EVENT = "Event";
+ public static final String STATE = "State";
+ public static final String MODEL_TYPE = "ModelType";
+ public static final String CONFIGS = "Configs";
+ public static final String PATH_PATTERN = "PathPattern";
+ public static final String CLASS_NAME = "ClassName";
+
+ // column names for show pipe plugins statement
+ public static final String PLUGIN_NAME = "PluginName";
+ public static final String PLUGIN_TYPE = "PluginType";
+ public static final String PLUGIN_JAR = "PluginJar";
+
+ // column names for show topics statement
+ public static final String TOPIC_NAME = "TopicName";
+ public static final String TOPIC_CONFIGS = "TopicConfigs";
+
+ // column names for show subscriptions statement
+ public static final String CONSUMER_GROUP_NAME = "ConsumerGroupName";
+ public static final String SUBSCRIBED_CONSUMERS = "SubscribedConsumers";
+
+ // show cluster status
+ public static final String NODE_TYPE_CONFIG_NODE = "ConfigNode";
+ public static final String NODE_TYPE_DATA_NODE = "DataNode";
+ public static final String NODE_TYPE_AI_NODE = "AINode";
+ public static final String COLUMN_CLUSTER_NAME = "ClusterName";
+ public static final String CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS =
+ "ConfigNodeConsensusProtocolClass";
+ public static final String DATA_REGION_CONSENSUS_PROTOCOL_CLASS =
+ "DataRegionConsensusProtocolClass";
+ public static final String SCHEMA_REGION_CONSENSUS_PROTOCOL_CLASS =
+ "SchemaRegionConsensusProtocolClass";
+ public static final String SERIES_SLOT_NUM = "SeriesSlotNum";
+ public static final String SERIES_SLOT_EXECUTOR_CLASS = "SeriesSlotExecutorClass";
+ public static final String SCHEMA_REGION_PER_DATA_NODE = "SchemaRegionPerDataNode";
+ public static final String DATA_REGION_PER_DATA_NODE = "DataRegionPerDataNode";
+ public static final String READ_CONSISTENCY_LEVEL = "ReadConsistencyLevel";
+ public static final String DISK_SPACE_WARNING_THRESHOLD = "DiskSpaceWarningThreshold";
+
+ public static final String TIMESTAMP_PRECISION = "TimestampPrecision";
+
+ // column names for show region statement
+ public static final String REGION_ID = "RegionId";
+ public static final String TYPE = "Type";
+ public static final String DATA_NODE_ID = "DataNodeId";
+ public static final String TIME_SLOT_NUM = "TimeSlotNum";
+ public static final String SERIES_SLOT_ID = "SeriesSlotId";
+ public static final String TIME_PARTITION = "TimePartition";
+ public static final String COUNT_TIME_PARTITION = "count(timePartition)";
+ public static final String START_TIME = "StartTime";
+ public static final String ROLE = "Role";
+ public static final String CREATE_TIME = "CreateTime";
+ public static final String TSFILE_SIZE = "TsFileSize";
+
+ // column names for show datanodes
+ public static final String SCHEMA_REGION_NUM = "SchemaRegionNum";
+ public static final String DATA_REGION_NUM = "DataRegionNum";
+
+ // column names for show device template statement
+ public static final String TEMPLATE_NAME = "TemplateName";
+
+ // column names for show pipe sink
+ public static final String NAME = "Name";
+
+ // column names for show pipe
+ public static final String ID = "ID";
+ public static final String CREATION_TIME = "CreationTime";
+ public static final String PIPE_EXTRACTOR = "PipeSource";
+ public static final String PIPE_PROCESSOR = "PipeProcessor";
+ public static final String PIPE_CONNECTOR = "PipeSink";
+ public static final String EXCEPTION_MESSAGE = "ExceptionMessage";
+ public static final String REMAINING_EVENT_COUNT = "RemainingEventCount";
+ public static final String ESTIMATED_REMAINING_SECONDS = "EstimatedRemainingSeconds";
+
+ // column names for select into
+ public static final String SOURCE_DEVICE = "SourceDevice";
+ public static final String SOURCE_COLUMN = "SourceColumn";
+ public static final String TARGET_TIMESERIES = "TargetTimeseries";
+ public static final String WRITTEN = "Written";
+
+ // column names for show cq
+ public static final String CQID = "CQId";
+ public static final String QUERY = "Query";
+
+ // column names for show query processlist
+ public static final String QUERY_ID = "QueryId";
+ public static final String ELAPSED_TIME = "ElapsedTime";
+ public static final String STATEMENT = "Statement";
+
+ public static final String QUERY_ID_TABLE_MODEL = "query_id";
+ public static final String QUERY_ID_START_TIME_TABLE_MODEL = "start_time";
+ public static final String DATA_NODE_ID_TABLE_MODEL = "datanode_id";
+ public static final String START_TIME_TABLE_MODEL = "start_time";
+ public static final String ELAPSED_TIME_TABLE_MODEL = "elapsed_time";
+
+ public static final String TABLE_NAME_TABLE_MODEL = "table_name";
+ public static final String COLUMN_NAME_TABLE_MODEL = "column_name";
+
+ public static final String SCHEMA_REPLICATION_FACTOR_TABLE_MODEL = "schema_replication_factor";
+ public static final String DATA_REPLICATION_FACTOR_TABLE_MODEL = "data_replication_factor";
+ public static final String TIME_PARTITION_INTERVAL_TABLE_MODEL = "time_partition_interval";
+ public static final String SCHEMA_REGION_GROUP_NUM_TABLE_MODEL = "schema_region_group_num";
+ public static final String DATA_REGION_GROUP_NUM_TABLE_MODEL = "data_region_group_num";
+
+ public static final String REGION_ID_TABLE_MODEL = "region_id";
+ public static final String DATANODE_ID_TABLE_MODEL = "datanode_id";
+ public static final String SERIES_SLOT_NUM_TABLE_MODEL = "series_slot_num";
+ public static final String TIME_SLOT_NUM_TABLE_MODEL = "time_slot_num";
+ public static final String RPC_ADDRESS_TABLE_MODEL = "rpc_address";
+ public static final String RPC_PORT_TABLE_MODEL = "rpc_port";
+ public static final String INTERNAL_ADDRESS_TABLE_MODEL = "internal_address";
+ public static final String CREATE_TIME_TABLE_MODEL = "create_time";
+ public static final String TS_FILE_SIZE_BYTES_TABLE_MODEL = "tsfile_size_bytes";
+
+ public static final String CREATION_TIME_TABLE_MODEL = "creation_time";
+ public static final String PIPE_SOURCE_TABLE_MODEL = "pipe_source";
+ public static final String PIPE_PROCESSOR_TABLE_MODEL = "pipe_processor";
+ public static final String PIPE_SINK_TABLE_MODEL = "pipe_sink";
+ public static final String EXCEPTION_MESSAGE_TABLE_MODEL = "exception_message";
+ public static final String REMAINING_EVENT_COUNT_TABLE_MODEL = "remaining_event_count";
+ public static final String ESTIMATED_REMAINING_SECONDS_TABLE_MODEL =
+ "estimated_remaining_seconds";
+
+ public static final String PLUGIN_NAME_TABLE_MODEL = "plugin_name";
+ public static final String PLUGIN_TYPE_TABLE_MODEL = "plugin_type";
+ public static final String CLASS_NAME_TABLE_MODEL = "class_name";
+ public static final String PLUGIN_JAR_TABLE_MODEL = "plugin_jar";
+
+ public static final String TOPIC_NAME_TABLE_MODEL = "topic_name";
+ public static final String TOPIC_CONFIGS_TABLE_MODEL = "topic_configs";
+
+ public static final String CONSUMER_GROUP_NAME_TABLE_MODEL = "consumer_group_name";
+ public static final String SUBSCRIBED_CONSUMERS_TABLE_MODEL = "subscribed_consumers";
+
+ // column names for show space quota
+ public static final String QUOTA_TYPE = "QuotaType";
+ public static final String LIMIT = "Limit";
+ public static final String USED = "Used";
+
+ // column names for show throttle quota
+ public static final String USER = "User";
+ public static final String READ_WRITE = "Read/Write";
+
+ // column names for show models/trials
+ public static final String MODEL_ID = "ModelId";
+
+ // column names for views (e.g. logical view)
+ public static final String VIEW_TYPE = "ViewType";
+ public static final String SOURCE = "Source";
+
+ // column names for show current timestamp
+ public static final String CURRENT_TIMESTAMP = "CurrentTimestamp";
+
+ // column names for table query
+ public static final String COLUMN_NAME = "ColumnName";
+ public static final String COLUMN_DATA_TYPE = "DataType";
+ public static final String COLUMN_CATEGORY = "Category";
+ public static final String TABLE_NAME = "TableName";
+ public static final String PRIVILEGES = "Privileges";
+
+ public static final String GRANT_OPTION = "GrantOption";
+
+ public static final String CURRENT_USER = "CurrentUser";
+
+ public static final String CURRENT_DATABASE = "CurrentDatabase";
+
+ public static final String CURRENT_SQL_DIALECT = "CurrentSqlDialect";
+/*
+ public static final List lastQueryColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(TIMESERIES, TSDataType.TEXT),
+ new ColumnHeader(VALUE, TSDataType.TEXT),
+ new ColumnHeader(DATATYPE, TSDataType.TEXT));
+
+ public static final List showTimeSeriesColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(TIMESERIES, TSDataType.TEXT),
+ new ColumnHeader(ALIAS, TSDataType.TEXT),
+ new ColumnHeader(DATABASE, TSDataType.TEXT),
+ new ColumnHeader(DATATYPE, TSDataType.TEXT),
+ new ColumnHeader(ENCODING, TSDataType.TEXT),
+ new ColumnHeader(COMPRESSION, TSDataType.TEXT),
+ new ColumnHeader(TAGS, TSDataType.TEXT),
+ new ColumnHeader(ATTRIBUTES, TSDataType.TEXT),
+ new ColumnHeader(DEADBAND, TSDataType.TEXT),
+ new ColumnHeader(DEADBAND_PARAMETERS, TSDataType.TEXT),
+ new ColumnHeader(VIEW_TYPE, TSDataType.TEXT));
+
+ public static final List showDevicesWithSgColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(DEVICE, TSDataType.TEXT),
+ new ColumnHeader(DATABASE, TSDataType.TEXT),
+ new ColumnHeader(IS_ALIGNED, TSDataType.TEXT),
+ new ColumnHeader(TEMPLATE, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_TTL, TSDataType.TEXT));
+
+ public static final List showDevicesColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(DEVICE, TSDataType.TEXT),
+ new ColumnHeader(IS_ALIGNED, TSDataType.TEXT),
+ new ColumnHeader(TEMPLATE, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_TTL, TSDataType.TEXT));
+ public static final List showTTLColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(DEVICE, TSDataType.TEXT), new ColumnHeader(COLUMN_TTL, TSDataType.TEXT));
+
+ public static final List showStorageGroupsColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(DATABASE, TSDataType.TEXT),
+ new ColumnHeader(SCHEMA_REPLICATION_FACTOR, TSDataType.INT32),
+ new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32),
+ new ColumnHeader(TIME_PARTITION_ORIGIN, TSDataType.INT64),
+ new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64));
+
+ public static final List showStorageGroupsDetailColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(DATABASE, TSDataType.TEXT),
+ new ColumnHeader(SCHEMA_REPLICATION_FACTOR, TSDataType.INT32),
+ new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32),
+ new ColumnHeader(TIME_PARTITION_ORIGIN, TSDataType.INT64),
+ new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64),
+ new ColumnHeader(SCHEMA_REGION_GROUP_NUM, TSDataType.INT32),
+ new ColumnHeader(MIN_SCHEMA_REGION_GROUP_NUM, TSDataType.INT32),
+ new ColumnHeader(MAX_SCHEMA_REGION_GROUP_NUM, TSDataType.INT32),
+ new ColumnHeader(DATA_REGION_GROUP_NUM, TSDataType.INT32),
+ new ColumnHeader(MIN_DATA_REGION_GROUP_NUM, TSDataType.INT32),
+ new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32));
+
+ public static final List showChildPathsColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(CHILD_PATHS, TSDataType.TEXT),
+ new ColumnHeader(NODE_TYPES, TSDataType.TEXT));
+
+ public static final List showNodesInSchemaTemplateHeaders =
+ ImmutableList.of(
+ new ColumnHeader(CHILD_NODES, TSDataType.TEXT),
+ new ColumnHeader(DATATYPE, TSDataType.TEXT),
+ new ColumnHeader(ENCODING, TSDataType.TEXT),
+ new ColumnHeader(COMPRESSION, TSDataType.TEXT));
+
+ public static final List showChildNodesColumnHeaders =
+ ImmutableList.of(new ColumnHeader(CHILD_NODES, TSDataType.TEXT));
+
+ public static final List showVersionColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(VERSION, TSDataType.TEXT),
+ new ColumnHeader(BUILD_INFO, TSDataType.TEXT));
+
+ public static final List showPathsUsingTemplateHeaders =
+ ImmutableList.of(new ColumnHeader(PATHS, TSDataType.TEXT));
+
+ public static final List showPathSetTemplateHeaders =
+ ImmutableList.of(new ColumnHeader(PATHS, TSDataType.TEXT));
+
+ public static final List countDevicesColumnHeaders =
+ ImmutableList.of(new ColumnHeader(COUNT_DEVICES, TSDataType.INT64));
+
+ public static final List countNodesColumnHeaders =
+ ImmutableList.of(new ColumnHeader(COUNT_NODES, TSDataType.INT64));
+
+ public static final List countLevelTimeSeriesColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(COLUMN, TSDataType.TEXT),
+ new ColumnHeader(COUNT_TIMESERIES, TSDataType.INT64));
+
+ public static final List countTimeSeriesColumnHeaders =
+ ImmutableList.of(new ColumnHeader(COUNT_TIMESERIES, TSDataType.INT64));
+
+ public static final List countStorageGroupColumnHeaders =
+ ImmutableList.of(new ColumnHeader(COUNT_DATABASE, TSDataType.INT32));
+
+ public static final List showRegionColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(REGION_ID, TSDataType.INT32),
+ new ColumnHeader(TYPE, TSDataType.TEXT),
+ new ColumnHeader(STATUS, TSDataType.TEXT),
+ new ColumnHeader(DATABASE, TSDataType.TEXT),
+ new ColumnHeader(SERIES_SLOT_NUM, TSDataType.INT32),
+ new ColumnHeader(TIME_SLOT_NUM, TSDataType.INT64),
+ new ColumnHeader(DATA_NODE_ID, TSDataType.INT32),
+ new ColumnHeader(RPC_ADDRESS, TSDataType.TEXT),
+ new ColumnHeader(RPC_PORT, TSDataType.INT32),
+ new ColumnHeader(INTERNAL_ADDRESS, TSDataType.TEXT),
+ new ColumnHeader(ROLE, TSDataType.TEXT),
+ new ColumnHeader(CREATE_TIME, TSDataType.TEXT),
+ new ColumnHeader(TSFILE_SIZE, TSDataType.TEXT));
+
+ public static final List showAINodesColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(NODE_ID, TSDataType.INT32),
+ new ColumnHeader(STATUS, TSDataType.TEXT),
+ new ColumnHeader(RPC_ADDRESS, TSDataType.TEXT),
+ new ColumnHeader(RPC_PORT, TSDataType.INT32));
+
+ public static final List showDataNodesColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(NODE_ID, TSDataType.INT32),
+ new ColumnHeader(STATUS, TSDataType.TEXT),
+ new ColumnHeader(RPC_ADDRESS, TSDataType.TEXT),
+ new ColumnHeader(RPC_PORT, TSDataType.INT32),
+ new ColumnHeader(DATA_REGION_NUM, TSDataType.INT32),
+ new ColumnHeader(SCHEMA_REGION_NUM, TSDataType.INT32));
+
+ public static final List showConfigNodesColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(NODE_ID, TSDataType.INT32),
+ new ColumnHeader(STATUS, TSDataType.TEXT),
+ new ColumnHeader(INTERNAL_ADDRESS, TSDataType.TEXT),
+ new ColumnHeader(INTERNAL_PORT, TSDataType.INT32),
+ new ColumnHeader(ROLE, TSDataType.TEXT));
+
+ public static final List showClusterColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(NODE_ID, TSDataType.INT32),
+ new ColumnHeader(NODE_TYPE, TSDataType.TEXT),
+ new ColumnHeader(STATUS, TSDataType.TEXT),
+ new ColumnHeader(INTERNAL_ADDRESS, TSDataType.TEXT),
+ new ColumnHeader(INTERNAL_PORT, TSDataType.INT32),
+ new ColumnHeader(VERSION, TSDataType.TEXT),
+ new ColumnHeader(BUILD_INFO, TSDataType.TEXT));
+
+ public static final List showClusterDetailsColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(NODE_ID, TSDataType.INT32),
+ new ColumnHeader(NODE_TYPE, TSDataType.TEXT),
+ new ColumnHeader(STATUS, TSDataType.TEXT),
+ new ColumnHeader(INTERNAL_ADDRESS, TSDataType.TEXT),
+ new ColumnHeader(INTERNAL_PORT, TSDataType.INT32),
+ new ColumnHeader(CONFIG_CONSENSUS_PORT, TSDataType.TEXT),
+ new ColumnHeader(RPC_ADDRESS, TSDataType.TEXT),
+ new ColumnHeader(RPC_PORT, TSDataType.TEXT),
+ new ColumnHeader(MPP_PORT, TSDataType.TEXT),
+ new ColumnHeader(SCHEMA_CONSENSUS_PORT, TSDataType.TEXT),
+ new ColumnHeader(DATA_CONSENSUS_PORT, TSDataType.TEXT),
+ new ColumnHeader(VERSION, TSDataType.TEXT),
+ new ColumnHeader(BUILD_INFO, TSDataType.TEXT));
+
+ public static final List showClusterIdColumnHeaders =
+ ImmutableList.of(new ColumnHeader(CLUSTER_ID, TSDataType.TEXT));
+
+ public static final List testConnectionColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(SERVICE_PROVIDER, TSDataType.TEXT),
+ new ColumnHeader(SENDER, TSDataType.TEXT),
+ new ColumnHeader(CONNECTION, TSDataType.TEXT));
+
+ public static final List showVariablesColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(VARIABLE, TSDataType.TEXT), new ColumnHeader(VALUE, TSDataType.TEXT));
+
+ public static final List showFunctionsColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(FUNCTION_NAME, TSDataType.TEXT),
+ new ColumnHeader(FUNCTION_TYPE, TSDataType.TEXT),
+ new ColumnHeader(CLASS_NAME_UDF, TSDataType.TEXT),
+ new ColumnHeader(FUNCTION_STATE, TSDataType.TEXT));
+
+ public static final List showTriggersColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(TRIGGER_NAME, TSDataType.TEXT),
+ new ColumnHeader(EVENT, TSDataType.TEXT),
+ new ColumnHeader(TYPE, TSDataType.TEXT),
+ new ColumnHeader(STATE, TSDataType.TEXT),
+ new ColumnHeader(PATH_PATTERN, TSDataType.TEXT),
+ new ColumnHeader(CLASS_NAME, TSDataType.TEXT),
+ new ColumnHeader(NODE_ID, TSDataType.TEXT));
+
+ public static final List showPipePluginsColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(PLUGIN_NAME, TSDataType.TEXT),
+ new ColumnHeader(PLUGIN_TYPE, TSDataType.TEXT),
+ new ColumnHeader(CLASS_NAME, TSDataType.TEXT),
+ new ColumnHeader(PLUGIN_JAR, TSDataType.TEXT));
+
+ public static final List showSchemaTemplateHeaders =
+ ImmutableList.of(new ColumnHeader(TEMPLATE_NAME, TSDataType.TEXT));
+
+ public static final List showPipeSinkTypeColumnHeaders =
+ ImmutableList.of(new ColumnHeader(TYPE, TSDataType.TEXT));
+
+ public static final List showPipeSinkColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(NAME, TSDataType.TEXT),
+ new ColumnHeader(TYPE, TSDataType.TEXT),
+ new ColumnHeader(ATTRIBUTES, TSDataType.TEXT));
+
+ public static final List showPipeColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(ID, TSDataType.TEXT),
+ new ColumnHeader(CREATION_TIME, TSDataType.TEXT),
+ new ColumnHeader(STATE, TSDataType.TEXT),
+ new ColumnHeader(PIPE_EXTRACTOR, TSDataType.TEXT),
+ new ColumnHeader(PIPE_PROCESSOR, TSDataType.TEXT),
+ new ColumnHeader(PIPE_CONNECTOR, TSDataType.TEXT),
+ new ColumnHeader(EXCEPTION_MESSAGE, TSDataType.TEXT),
+ new ColumnHeader(REMAINING_EVENT_COUNT, TSDataType.TEXT),
+ new ColumnHeader(ESTIMATED_REMAINING_SECONDS, TSDataType.TEXT));
+
+ public static final List showTopicColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(TOPIC_NAME, TSDataType.TEXT),
+ new ColumnHeader(TOPIC_CONFIGS, TSDataType.TEXT));
+
+ public static final List showSubscriptionColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(TOPIC_NAME, TSDataType.TEXT),
+ new ColumnHeader(CONSUMER_GROUP_NAME, TSDataType.TEXT),
+ new ColumnHeader(SUBSCRIBED_CONSUMERS, TSDataType.TEXT));
+
+ public static final List selectIntoColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(SOURCE_COLUMN, TSDataType.TEXT),
+ new ColumnHeader(TARGET_TIMESERIES, TSDataType.TEXT),
+ new ColumnHeader(WRITTEN, TSDataType.INT32));
+
+ public static final List selectIntoAlignByDeviceColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(SOURCE_DEVICE, TSDataType.TEXT),
+ new ColumnHeader(SOURCE_COLUMN, TSDataType.TEXT),
+ new ColumnHeader(TARGET_TIMESERIES, TSDataType.TEXT),
+ new ColumnHeader(WRITTEN, TSDataType.INT32));
+
+ public static final List getRegionIdColumnHeaders =
+ ImmutableList.of(new ColumnHeader(REGION_ID, TSDataType.INT32));
+
+ public static final List getTimeSlotListColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(TIME_PARTITION, TSDataType.INT64),
+ new ColumnHeader(START_TIME, TSDataType.TEXT));
+
+ public static final List countTimeSlotListColumnHeaders =
+ ImmutableList.of(new ColumnHeader(COUNT_TIME_PARTITION, TSDataType.INT64));
+
+ public static final List getSeriesSlotListColumnHeaders =
+ ImmutableList.of(new ColumnHeader(SERIES_SLOT_ID, TSDataType.INT32));
+
+ public static final List showContinuousQueriesColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(CQID, TSDataType.TEXT),
+ new ColumnHeader(QUERY, TSDataType.TEXT),
+ new ColumnHeader(STATE, TSDataType.TEXT));
+
+ public static final List showQueriesColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(QUERY_ID, TSDataType.TEXT),
+ new ColumnHeader(DATA_NODE_ID, TSDataType.INT32),
+ new ColumnHeader(ELAPSED_TIME, TSDataType.FLOAT),
+ new ColumnHeader(STATEMENT, TSDataType.TEXT));
+
+ public static final List showSpaceQuotaColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(DATABASE, TSDataType.TEXT),
+ new ColumnHeader(QUOTA_TYPE, TSDataType.TEXT),
+ new ColumnHeader(LIMIT, TSDataType.TEXT),
+ new ColumnHeader(USED, TSDataType.TEXT));
+
+ public static final List showThrottleQuotaColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(USER, TSDataType.TEXT),
+ new ColumnHeader(QUOTA_TYPE, TSDataType.TEXT),
+ new ColumnHeader(LIMIT, TSDataType.TEXT),
+ new ColumnHeader(READ_WRITE, TSDataType.TEXT));
+
+ public static final List showModelsColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(MODEL_ID, TSDataType.TEXT),
+ new ColumnHeader(MODEL_TYPE, TSDataType.TEXT),
+ new ColumnHeader(STATE, TSDataType.TEXT),
+ new ColumnHeader(CONFIGS, TSDataType.TEXT),
+ new ColumnHeader(NOTES, TSDataType.TEXT));
+
+ public static final List showLogicalViewColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(TIMESERIES, TSDataType.TEXT),
+ new ColumnHeader(DATABASE, TSDataType.TEXT),
+ new ColumnHeader(DATATYPE, TSDataType.TEXT),
+ new ColumnHeader(TAGS, TSDataType.TEXT),
+ new ColumnHeader(ATTRIBUTES, TSDataType.TEXT),
+ new ColumnHeader(VIEW_TYPE, TSDataType.TEXT),
+ new ColumnHeader(SOURCE, TSDataType.TEXT));
+
+ public static final List showDBColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(DATABASE, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_TTL, TSDataType.TEXT),
+ new ColumnHeader(SCHEMA_REPLICATION_FACTOR, TSDataType.INT32),
+ new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32),
+ new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64));
+
+ public static final List showDBDetailsColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(DATABASE, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_TTL, TSDataType.TEXT),
+ new ColumnHeader(SCHEMA_REPLICATION_FACTOR, TSDataType.INT32),
+ new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32),
+ new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64),
+ new ColumnHeader(SCHEMA_REGION_GROUP_NUM, TSDataType.INT32),
+ new ColumnHeader(DATA_REGION_GROUP_NUM, TSDataType.INT32));
+
+ public static final List describeTableColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(COLUMN_NAME, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_DATA_TYPE, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_CATEGORY, TSDataType.TEXT));
+
+ public static final List describeTableDetailsColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(COLUMN_NAME, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_DATA_TYPE, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_CATEGORY, TSDataType.TEXT),
+ new ColumnHeader(STATUS, TSDataType.TEXT));
+
+ public static final List showTablesColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(TABLE_NAME, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_TTL, TSDataType.TEXT));
+
+ public static final List showTablesDetailsColumnHeaders =
+ ImmutableList.of(
+ new ColumnHeader(TABLE_NAME, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_TTL, TSDataType.TEXT),
+ new ColumnHeader(STATUS, TSDataType.TEXT));
+
+ public static final List LIST_USER_OR_ROLE_PRIVILEGES_COLUMN_HEADERS =
+ ImmutableList.of(
+ new ColumnHeader(ROLE, TSDataType.TEXT),
+ new ColumnHeader(SCOPE, TSDataType.TEXT),
+ new ColumnHeader(PRIVILEGES, TSDataType.TEXT),
+ new ColumnHeader(GRANT_OPTION, TSDataType.BOOLEAN));
+
+ public static final List SHOW_CURRENT_USER_COLUMN_HEADERS =
+ ImmutableList.of(new ColumnHeader(CURRENT_USER, TSDataType.STRING));
+
+ public static final List SHOW_CURRENT_DATABASE_COLUMN_HEADERS =
+ ImmutableList.of(new ColumnHeader(CURRENT_DATABASE, TSDataType.STRING));
+
+ public static final List SHOW_CURRENT_SQL_DIALECT_COLUMN_HEADERS =
+ ImmutableList.of(new ColumnHeader(CURRENT_SQL_DIALECT, TSDataType.STRING));
+
+ public static final List SHOW_CURRENT_TIMESTAMP_COLUMN_HEADERS =
+ ImmutableList.of(new ColumnHeader(CURRENT_TIMESTAMP, TSDataType.TIMESTAMP));*/
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/IoTDBConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/IoTDBConstant.java
new file mode 100644
index 00000000..03aabc95
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/IoTDBConstant.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.constant;
+
+import java.io.File;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+public class IoTDBConstant {
+
+ private IoTDBConstant() {}
+
+ static {
+ Properties prop = new Properties();
+ String finalBuildInfo = "UNKNOWN";
+ try {
+ prop.load(
+ new InputStreamReader(
+ Objects.requireNonNull(IoTDBConstant.class.getResourceAsStream("/git.properties")),
+ StandardCharsets.UTF_8));
+ finalBuildInfo = prop.getProperty("git.commit.id.abbrev", "UNKNOWN");
+ String isDirty = prop.getProperty("git.dirty", "false");
+ if (isDirty.equalsIgnoreCase("true")) {
+ finalBuildInfo += "-dev";
+ }
+ } catch (Exception e) {
+ System.err.println("get git.properties error: " + e.getMessage());
+ }
+ BUILD_INFO = finalBuildInfo;
+ }
+
+ public static final String BUILD_INFO;
+
+ public static final String DN_ENV_FILE_NAME = "datanode-env";
+ public static final String CN_ENV_FILE_NAME = "confignode-env";
+ public static final String IOTDB_CONF = "IOTDB_CONF";
+ public static final String GLOBAL_DB_NAME = "IoTDB";
+ public static final String CN_ROLE = "confignode";
+ public static final String DN_ROLE = "datanode";
+
+ public static final String DATA_NODE_CONF_FILE_NAME = "iotdb-datanode.properties";
+
+ public static final String DN_RPC_ADDRESS = "dn_rpc_address";
+ public static final String DN_RPC_PORT = "dn_rpc_port";
+
+ public static final String CN_INTERNAL_ADDRESS = "cn_internal_address";
+ public static final String DN_INTERNAL_ADDRESS = "dn_internal_address";
+
+ public static final String CN_METRIC_PROMETHEUS_REPORTER_PORT =
+ "cn_metric_prometheus_reporter_port";
+ public static final String DN_METRIC_PROMETHEUS_REPORTER_PORT =
+ "dn_metric_prometheus_reporter_port";
+
+ public static final String CN_INTERNAL_PORT = "cn_internal_port";
+ public static final String DN_INTERNAL_PORT = "dn_internal_port";
+ public static final String CN_CONSENSUS_PORT = "cn_consensus_port";
+
+ public static final String CN_SEED_CONFIG_NODE = "cn_seed_config_node";
+ public static final String CN_TARGET_CONFIG_NODE_LIST = "cn_target_config_node_list";
+ public static final String DN_SEED_CONFIG_NODE = "dn_seed_config_node";
+ public static final String DN_TARGET_CONFIG_NODE_LIST = "dn_target_config_node_list";
+
+ public static final String CLUSTER_NAME = "cluster_name";
+ public static final String DEFAULT_CLUSTER_NAME = "defaultCluster";
+ public static final String LOGO =
+ ""
+ + " _____ _________ ______ ______ \n"
+ + "|_ _| | _ _ ||_ _ `.|_ _ \\ \n"
+ + " | | .--.|_/ | | \\_| | | `. \\ | |_) | \n"
+ + " | | / .'`\\ \\ | | | | | | | __'. \n"
+ + " _| |_| \\__. | _| |_ _| |_.' /_| |__) | \n"
+ + "|_____|'.__.' |_____| |______.'|_______/ ";
+
+ // when running the program in IDE, we can not get the version info using
+ // getImplementationVersion()
+ public static final String VERSION =
+ IoTDBConstant.class.getPackage().getImplementationVersion() != null
+ ? IoTDBConstant.class.getPackage().getImplementationVersion()
+ : "UNKNOWN";
+ public static final String MAJOR_VERSION =
+ "UNKNOWN".equals(VERSION)
+ ? "UNKNOWN"
+ : VERSION.split("\\.")[0] + "." + VERSION.split("\\.")[1];
+ public static final String VERSION_WITH_BUILD = VERSION + " (Build: " + BUILD_INFO + ")";
+
+ public static final String AUDIT_LOGGER_NAME = "IoTDB_AUDIT_LOGGER";
+ public static final String SLOW_SQL_LOGGER_NAME = "SLOW_SQL";
+ public static final String SAMPLED_QUERIES_LOGGER_NAME = "SAMPLED_QUERIES";
+
+ public static final String COMPACTION_LOGGER_NAME = "COMPACTION";
+ public static final String EXPLAIN_ANALYZE_LOGGER_NAME = "EXPLAIN_ANALYZE";
+
+ public static final String IOTDB_JMX_LOCAL = "iotdb.jmx.local";
+ public static final String IOTDB_JMX_PORT = "com.sun.management.jmxremote.port";
+
+ public static final String IOTDB_SERVICE_JMX_NAME = "org.apache.iotdb.service";
+ public static final String IOTDB_THREADPOOL_JMX_NAME = "org.apache.iotdb.threadpool";
+ public static final String JMX_TYPE = "type";
+
+ public static final long PB = 1L << 50;
+ public static final long TB = 1L << 40;
+ public static final long GB = 1L << 30;
+ public static final long MB = 1L << 20;
+ public static final long KB = 1L << 10;
+
+ public static final String IOTDB_HOME = "IOTDB_HOME";
+
+ public static final String IOTDB_DATA_HOME = "IOTDB_DATA_HOME";
+
+ public static final String SEQFILE_LOG_NODE_SUFFIX = "-seq";
+ public static final String UNSEQFILE_LOG_NODE_SUFFIX = "-unseq";
+
+ public static final String PATH_ROOT = "root";
+ public static final char PATH_SEPARATOR = '.';
+ public static final String PROFILE_SUFFIX = ".profile";
+ public static final String MAX_TIME = "max_time";
+ public static final String MIN_TIME = "min_time";
+ public static final String LAST_VALUE = "last_value";
+ public static final int MIN_SUPPORTED_JDK_VERSION = 8;
+ public static final Set reservedWords = new HashSet<>();
+
+ static {
+ reservedWords.add("TIME");
+ reservedWords.add("TIMESTAMP");
+ reservedWords.add("ROOT");
+ }
+
+ // show info
+ public static final String COLUMN_ITEM = " item";
+ public static final String COLUMN_VALUE = "value";
+ public static final String COLUMN_VERSION = "version";
+ public static final String COLUMN_BUILD_INFO = "build info";
+ public static final String COLUMN_TIMESERIES = "timeseries";
+ public static final String COLUMN_TIMESERIES_ALIAS = "alias";
+ public static final String COLUMN_TIMESERIES_DATATYPE = "dataType";
+ public static final String COLUMN_TIMESERIES_ENCODING = "encoding";
+ public static final String COLUMN_TIMESERIES_COMPRESSION = "compression";
+ public static final String COLUMN_TIMESERIES_COMPRESSOR = "compressor";
+ public static final String COLUMN_CHILD_PATHS = "child paths";
+ public static final String COLUMN_CHILD_PATHS_TYPES = "node types";
+ public static final String COLUMN_CHILD_NODES = "child nodes";
+ public static final String COLUMN_DEVICES = "devices";
+ public static final String COLUMN_DELETED_DEVICE_NUM = "num_of_deleted_devices";
+ public static final String COLUMN_COLUMN = "column";
+ public static final String COLUMN_COUNT = "count";
+ public static final String COLUMN_TAGS = "tags";
+ public static final String COLUMN_ATTRIBUTES = "attributes";
+ public static final String COLUMN_IS_ALIGNED = "isAligned";
+ public static final String COLUMN_DISTRIBUTION_PLAN = "distribution plan";
+ public static final String QUERY_ID = "queryId";
+ public static final String STATEMENT = "statement";
+
+ public static final String COLUMN_DATABASE = "database";
+
+ public static final String COLUMN_FUNCTION_NAME = "function name";
+ public static final String COLUMN_FUNCTION_TYPE = "function type";
+ public static final String COLUMN_FUNCTION_CLASS = "class name (UDF)";
+
+ public static final String COLUMN_SCHEMA_TEMPLATE = "template name";
+
+ // for tree model
+ public static final String FUNCTION_TYPE_NATIVE = "native";
+ public static final String FUNCTION_TYPE_BUILTIN_SCALAR = "built-in scalar";
+ public static final String FUNCTION_TYPE_BUILTIN_UDAF = "built-in UDAF";
+ public static final String FUNCTION_TYPE_BUILTIN_UDTF = "built-in UDTF";
+ public static final String FUNCTION_TYPE_EXTERNAL_UDAF = "external UDAF";
+ public static final String FUNCTION_TYPE_EXTERNAL_UDTF = "external UDTF";
+ // for table model
+ public static final String FUNCTION_TYPE_BUILTIN_SCALAR_FUNC = "built-in scalar function";
+ public static final String FUNCTION_TYPE_BUILTIN_AGG_FUNC = "built-in aggregate function";
+ public static final String FUNCTION_TYPE_BUILTIN_TABLE_FUNC = "built-in table function";
+ public static final String FUNCTION_TYPE_USER_DEFINED_SCALAR_FUNC =
+ "user-defined scalar function";
+ public static final String FUNCTION_TYPE_USER_DEFINED_AGG_FUNC =
+ "user-defined aggregate function";
+ public static final String FUNCTION_TYPE_USER_DEFINED_TABLE_FUNC = "user-defined table function";
+ // common
+ public static final String FUNCTION_TYPE_UNKNOWN = "UNKNOWN";
+ public static final String FUNCTION_STATE_AVAILABLE = "AVAILABLE";
+ public static final String FUNCTION_STATE_UNAVAILABLE = "UNAVAILABLE";
+
+ public static final String COLUMN_TRIGGER_NAME = "trigger name";
+ public static final String COLUMN_TRIGGER_STATUS = "status";
+ public static final String COLUMN_TRIGGER_EVENT = "event";
+ public static final String COLUMN_TRIGGER_PATH = "path";
+ public static final String COLUMN_TRIGGER_CLASS = "class name";
+ public static final String COLUMN_TRIGGER_ATTRIBUTES = "attributes";
+
+ public static final String COLUMN_TRIGGER_STATUS_STARTED = "started";
+ public static final String COLUMN_TRIGGER_STATUS_STOPPED = "stopped";
+
+ public static final String ONE_LEVEL_PATH_WILDCARD = "*";
+ public static final String MULTI_LEVEL_PATH_WILDCARD = "**";
+ public static final String TIME = "time";
+
+ // sdt parameters
+ public static final String LOSS = "loss";
+ public static final String SDT = "sdt";
+ public static final String SDT_COMP_DEV = "compdev";
+ public static final String SDT_COMP_MIN_TIME = "compmintime";
+ public static final String SDT_COMP_MAX_TIME = "compmaxtime";
+ public static final String[] SDT_PARAMETERS =
+ new String[] {SDT_COMP_DEV, SDT_COMP_MIN_TIME, SDT_COMP_MAX_TIME};
+
+ public static final String DEADBAND = "deadband";
+ public static final String MAX_POINT_NUMBER = "max_point_number";
+ public static final String MAX_STRING_LENGTH = "max_string_length";
+ public static final Set ALLOWED_SCHEMA_PROPS =
+ new HashSet<>(
+ Arrays.asList(
+ DEADBAND,
+ LOSS,
+ SDT,
+ SDT_COMP_DEV,
+ SDT_COMP_MIN_TIME,
+ SDT_COMP_MAX_TIME,
+ MAX_POINT_NUMBER,
+ MAX_STRING_LENGTH));
+
+ // default base dir, stores all IoTDB runtime files
+ public static final String CN_DEFAULT_DATA_DIR = "data" + File.separator + CN_ROLE;
+ public static final String DN_DEFAULT_DATA_DIR = "data" + File.separator + DN_ROLE;
+
+ // data folder name
+ public static final String DATA_FOLDER_NAME = "data";
+ public static final String SEQUENCE_FOLDER_NAME = "sequence";
+ public static final String UNSEQUENCE_FOLDER_NAME = "unsequence";
+ public static final String FILE_NAME_SEPARATOR = "-";
+ public static final String CONSENSUS_FOLDER_NAME = "consensus";
+ public static final String DATA_REGION_FOLDER_NAME = "data_region";
+ public static final String INVALID_DATA_REGION_FOLDER_NAME = "invalid_data_region";
+ public static final String SCHEMA_REGION_FOLDER_NAME = "schema_region";
+ public static final String SNAPSHOT_FOLDER_NAME = "snapshot";
+
+ // system folder name
+ public static final String SYSTEM_FOLDER_NAME = "system";
+ public static final String SCHEMA_FOLDER_NAME = "schema";
+ public static final String LOAD_TSFILE_FOLDER_NAME = "load";
+ public static final String LOAD_TSFILE_ACTIVE_LISTENING_PENDING_FOLDER_NAME = "pending";
+ public static final String LOAD_TSFILE_ACTIVE_LISTENING_FAILED_FOLDER_NAME = "failed";
+ public static final String SYNC_FOLDER_NAME = "sync";
+ public static final String QUERY_FOLDER_NAME = "query";
+ public static final String EXT_FOLDER_NAME = "ext";
+ public static final String UDF_FOLDER_NAME = "udf";
+ public static final String TRIGGER_FOLDER_NAME = "trigger";
+ public static final String PIPE_FOLDER_NAME = "pipe";
+ public static final String TMP_FOLDER_NAME = "tmp";
+ public static final String DELETION_FOLDER_NAME = "deletion";
+
+ public static final String MQTT_FOLDER_NAME = "mqtt";
+ public static final String WAL_FOLDER_NAME = "wal";
+ public static final String EXT_PIPE_FOLDER_NAME = "extPipe";
+
+ // mqtt
+ public static final String ENABLE_MQTT = "enable_mqtt_service";
+ public static final String MQTT_HOST_NAME = "mqtt_host";
+ public static final String MQTT_PORT_NAME = "mqtt_port";
+ public static final String MQTT_HANDLER_POOL_SIZE_NAME = "mqtt_handler_pool_size";
+ public static final String MQTT_PAYLOAD_FORMATTER_NAME = "mqtt_payload_formatter";
+ public static final String MQTT_DATA_PATH = "mqtt_data_path";
+ public static final String MQTT_MAX_MESSAGE_SIZE = "mqtt_max_message_size";
+
+ // thrift
+ public static final int LEFT_SIZE_IN_REQUEST = 4 * 1024 * 1024;
+ public static final int DEFAULT_FETCH_SIZE = 5000;
+ public static final int DEFAULT_CONNECTION_TIMEOUT_MS = 0;
+
+ // ratis
+ public static final int RAFT_LOG_BASIC_SIZE = 48;
+
+ // inner space compaction
+ public static final String INNER_COMPACTION_TMP_FILE_SUFFIX = ".inner";
+
+ // cross space compaction
+ public static final String CROSS_COMPACTION_TMP_FILE_SUFFIX = ".cross";
+
+ public static final String SETTLE_SUFFIX = ".settle";
+ public static final String MODS_SETTLE_FILE_SUFFIX = ".mods.settle";
+ public static final String BLANK = "";
+
+ // write ahead log
+ public static final String WAL_FILE_PREFIX = "_";
+ public static final String WAL_FILE_SUFFIX = ".wal";
+ public static final String WAL_CHECKPOINT_FILE_SUFFIX = ".checkpoint";
+ public static final String WAL_VERSION_ID = "versionId";
+ public static final String WAL_START_SEARCH_INDEX = "startSearchIndex";
+ public static final String WAL_STATUS_CODE = "statusCode";
+
+ public static final String IOTDB_FOREGROUND = "iotdb-foreground";
+ public static final String IOTDB_PIDFILE = "iotdb-pidfile";
+
+ // quota
+ public static final String SPACE_QUOTA_DISK = "disk";
+ public static final String QUOTA_UNLIMITED = "unlimited";
+ public static final String REQUEST_NUM_PER_UNIT_TIME = "request";
+ public static final String REQUEST_SIZE_PER_UNIT_TIME = "size";
+ public static final String MEMORY_SIZE_PER_READ = "mem";
+ public static final String CPU_NUMBER_PER_READ = "cpu";
+ public static final String REQUEST_TYPE = "type";
+ public static final String REQUEST_TYPE_READ = "read";
+ public static final String REQUEST_TYPE_WRITE = "write";
+ public static final String REQ_UNIT = "req";
+ public static final String REQ_SPLIT_UNIT = "req/";
+ public static final int UNLIMITED_VALUE = -1;
+ public static final int DEFAULT_VALUE = 0;
+ public static final float B_FLOAT = 1024.0F;
+
+ // SizeUnit
+ public static final String B_UNIT = "B";
+ public static final String KB_UNIT = "K";
+ public static final String MB_UNIT = "M";
+ public static final String GB_UNIT = "G";
+ public static final String TB_UNIT = "T";
+ public static final String PB_UNIT = "P";
+
+ // Time
+ public static final int SEC = 1000;
+ public static final int MIN = 60 * 1000;
+ public static final int HOUR = 60 * 60 * 1000;
+ public static final int DAY = 24 * 60 * 60 * 1000;
+
+ // TimeUnit
+ public static final String SEC_UNIT = "sec";
+ public static final String MIN_UNIT = "min";
+ public static final String HOUR_UNIT = "hour";
+ public static final String DAY_UNIT = "day";
+
+ // client version number
+ public enum ClientVersion {
+ V_0_12,
+ V_0_13,
+ V_1_0
+ }
+
+ // select into
+ public static final Pattern LEVELED_PATH_TEMPLATE_PATTERN = Pattern.compile("\\$\\{\\w+}");
+ public static final String DOUBLE_COLONS = "::";
+
+ public static final int MAX_DATABASE_NAME_LENGTH = 64;
+
+ public static final String TIER_SEPARATOR = ";";
+
+ public static final String OBJECT_STORAGE_DIR = "object_storage";
+
+ public static final String TTL_INFINITE = "INF";
+
+ public static final String INTEGRATION_TEST_KILL_POINTS = "integrationTestKillPoints";
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeConnectorConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeConnectorConstant.java
new file mode 100644
index 00000000..706a2ae2
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeConnectorConstant.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.constant;
+
+import com.github.luben.zstd.Zstd;
+import org.apache.iotdb.collector.config.PipeOptions;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static io.moquette.BrokerConstants.MB;
+
+public class PipeConnectorConstant {
+
+ public static final String CONNECTOR_KEY = "connector";
+ public static final String SINK_KEY = "sink";
+
+ public static final String CONNECTOR_IOTDB_IP_KEY = "connector.ip";
+ public static final String SINK_IOTDB_IP_KEY = "sink.ip";
+ public static final String CONNECTOR_IOTDB_HOST_KEY = "connector.host";
+ public static final String SINK_IOTDB_HOST_KEY = "sink.host";
+ public static final String CONNECTOR_IOTDB_PORT_KEY = "connector.port";
+ public static final String SINK_IOTDB_PORT_KEY = "sink.port";
+ public static final String CONNECTOR_IOTDB_NODE_URLS_KEY = "connector.node-urls";
+ public static final String SINK_IOTDB_NODE_URLS_KEY = "sink.node-urls";
+
+ public static final String SINK_IOTDB_SSL_ENABLE_KEY = "sink.ssl.enable";
+ public static final String SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY = "sink.ssl.trust-store-path";
+ public static final String SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY = "sink.ssl.trust-store-pwd";
+
+ public static final String CONNECTOR_IOTDB_PARALLEL_TASKS_KEY = "connector.parallel.tasks";
+ public static final String SINK_IOTDB_PARALLEL_TASKS_KEY = "sink.parallel.tasks";
+ public static final int CONNECTOR_IOTDB_PARALLEL_TASKS_DEFAULT_VALUE =
+ PipeOptions.PIPE_SUBTASK_EXECUTOR_MAX_THREAD_NUM.value();
+
+ public static final String CONNECTOR_REALTIME_FIRST_KEY = "connector.realtime-first";
+ public static final String SINK_REALTIME_FIRST_KEY = "sink.realtime-first";
+ public static final boolean CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY = "connector.batch.enable";
+ public static final String SINK_IOTDB_BATCH_MODE_ENABLE_KEY = "sink.batch.enable";
+ public static final boolean CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_IOTDB_BATCH_DELAY_KEY = "connector.batch.max-delay-seconds";
+ public static final String SINK_IOTDB_BATCH_DELAY_KEY = "sink.batch.max-delay-seconds";
+ public static final int CONNECTOR_IOTDB_PLAIN_BATCH_DELAY_DEFAULT_VALUE = 1;
+ public static final int CONNECTOR_IOTDB_TS_FILE_BATCH_DELAY_DEFAULT_VALUE = 5;
+
+ public static final String CONNECTOR_IOTDB_BATCH_SIZE_KEY = "connector.batch.size-bytes";
+ public static final String SINK_IOTDB_BATCH_SIZE_KEY = "sink.batch.size-bytes";
+ public static final long CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE = 16 * MB;
+ public static final long CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE = 80 * MB;
+
+ public static final String CONNECTOR_IOTDB_USER_KEY = "connector.user";
+ public static final String SINK_IOTDB_USER_KEY = "sink.user";
+ public static final String CONNECTOR_IOTDB_USERNAME_KEY = "connector.username";
+ public static final String SINK_IOTDB_USERNAME_KEY = "sink.username";
+ public static final String CONNECTOR_IOTDB_USER_DEFAULT_VALUE = "root";
+
+ public static final String CONNECTOR_IOTDB_PASSWORD_KEY = "connector.password";
+ public static final String SINK_IOTDB_PASSWORD_KEY = "sink.password";
+ public static final String CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE = "root";
+
+ public static final String CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY =
+ "connector.exception.data.convert-on-type-mismatch";
+ public static final String SINK_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY =
+ "sink.exception.data.convert-on-type-mismatch";
+ public static final boolean CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE =
+ true;
+
+ public static final String CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY =
+ "connector.exception.conflict.resolve-strategy";
+ public static final String SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY =
+ "sink.exception.conflict.resolve-strategy";
+ public static final String CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE = "retry";
+
+ public static final String CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_KEY =
+ "connector.exception.conflict.retry-max-time-seconds";
+ public static final String SINK_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_KEY =
+ "sink.exception.conflict.retry-max-time-seconds";
+ public static final long CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE = 60;
+
+ public static final String CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_KEY =
+ "connector.exception.conflict.record-ignored-data";
+ public static final String SINK_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_KEY =
+ "sink.exception.conflict.record-ignored-data";
+ public static final boolean CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_KEY =
+ "connector.exception.others.retry-max-time-seconds";
+ public static final String SINK_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_KEY =
+ "sink.exception.others.retry-max-time-seconds";
+ public static final long CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE = -1;
+
+ public static final String CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_KEY =
+ "connector.exception.others.record-ignored-data";
+ public static final String SINK_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_KEY =
+ "sink.exception.others.record-ignored-data";
+ public static final boolean CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_AIR_GAP_E_LANGUAGE_ENABLE_KEY =
+ "connector.air-gap.e-language.enable";
+ public static final String SINK_AIR_GAP_E_LANGUAGE_ENABLE_KEY = "sink.air-gap.e-language.enable";
+ public static final boolean CONNECTOR_AIR_GAP_E_LANGUAGE_ENABLE_DEFAULT_VALUE = false;
+
+ public static final String CONNECTOR_AIR_GAP_HANDSHAKE_TIMEOUT_MS_KEY =
+ "connector.air-gap.handshake-timeout-ms";
+ public static final String SINK_AIR_GAP_HANDSHAKE_TIMEOUT_MS_KEY =
+ "sink.air-gap.handshake-timeout-ms";
+ public static final int CONNECTOR_AIR_GAP_HANDSHAKE_TIMEOUT_MS_DEFAULT_VALUE = 5000;
+
+ public static final String CONNECTOR_IOTDB_SYNC_CONNECTOR_VERSION_KEY = "connector.version";
+ public static final String SINK_IOTDB_SYNC_CONNECTOR_VERSION_KEY = "sink.version";
+ public static final String CONNECTOR_IOTDB_SYNC_CONNECTOR_VERSION_DEFAULT_VALUE = "1.1";
+
+ public static final String CONNECTOR_WEBSOCKET_PORT_KEY = "connector.websocket.port";
+ public static final String SINK_WEBSOCKET_PORT_KEY = "sink.websocket.port";
+ public static final int CONNECTOR_WEBSOCKET_PORT_DEFAULT_VALUE = 8080;
+
+ public static final String CONNECTOR_OPC_UA_MODEL_KEY = "connector.opcua.model";
+ public static final String SINK_OPC_UA_MODEL_KEY = "sink.opcua.model";
+ public static final String CONNECTOR_OPC_UA_MODEL_CLIENT_SERVER_VALUE = "client-server";
+ public static final String CONNECTOR_OPC_UA_MODEL_PUB_SUB_VALUE = "pub-sub";
+ public static final String CONNECTOR_OPC_UA_MODEL_DEFAULT_VALUE =
+ CONNECTOR_OPC_UA_MODEL_CLIENT_SERVER_VALUE;
+
+ public static final String CONNECTOR_OPC_UA_TCP_BIND_PORT_KEY = "connector.opcua.tcp.port";
+ public static final String SINK_OPC_UA_TCP_BIND_PORT_KEY = "sink.opcua.tcp.port";
+ public static final int CONNECTOR_OPC_UA_TCP_BIND_PORT_DEFAULT_VALUE = 12686;
+
+ public static final String CONNECTOR_OPC_UA_HTTPS_BIND_PORT_KEY = "connector.opcua.https.port";
+ public static final String SINK_OPC_UA_HTTPS_BIND_PORT_KEY = "sink.opcua.https.port";
+ public static final int CONNECTOR_OPC_UA_HTTPS_BIND_PORT_DEFAULT_VALUE = 8443;
+
+ public static final String CONNECTOR_OPC_UA_SECURITY_DIR_KEY = "connector.opcua.security.dir";
+ public static final String SINK_OPC_UA_SECURITY_DIR_KEY = "sink.opcua.security.dir";
+ public static final String CONNECTOR_OPC_UA_SECURITY_DIR_DEFAULT_VALUE =
+ getConfDir() != null
+ ? getConfDir() + File.separatorChar + "opc_security"
+ : System.getProperty("user.home") + File.separatorChar + "iotdb_opc_security";
+
+ public static final String CONNECTOR_OPC_UA_ENABLE_ANONYMOUS_ACCESS_KEY =
+ "connector.opcua.enable-anonymous-access";
+ public static final String SINK_OPC_UA_ENABLE_ANONYMOUS_ACCESS_KEY =
+ "sink.opcua.enable-anonymous-access";
+ public static final boolean CONNECTOR_OPC_UA_ENABLE_ANONYMOUS_ACCESS_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_OPC_UA_PLACEHOLDER_KEY = "connector.opcua.placeholder";
+ public static final String SINK_OPC_UA_PLACEHOLDER_KEY = "sink.opcua.placeholder";
+ public static final String CONNECTOR_OPC_UA_PLACEHOLDER_DEFAULT_VALUE = "null";
+
+ public static final String CONNECTOR_LEADER_CACHE_ENABLE_KEY = "connector.leader-cache.enable";
+ public static final String SINK_LEADER_CACHE_ENABLE_KEY = "sink.leader-cache.enable";
+ public static final boolean CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_LOAD_BALANCE_STRATEGY_KEY =
+ "connector.load-balance-strategy";
+ public static final String SINK_LOAD_BALANCE_STRATEGY_KEY = "sink.load-balance-strategy";
+ public static final String CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY = "round-robin";
+ public static final String CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY = "random";
+ public static final String CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY = "priority";
+ public static final Set CONNECTOR_LOAD_BALANCE_STRATEGY_SET =
+ Collections.unmodifiableSet(
+ new HashSet<>(
+ Arrays.asList(
+ CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY,
+ CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY,
+ CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY)));
+
+ public static final String CONNECTOR_COMPRESSOR_KEY = "connector.compressor";
+ public static final String SINK_COMPRESSOR_KEY = "sink.compressor";
+ public static final String CONNECTOR_COMPRESSOR_DEFAULT_VALUE = "";
+ public static final String CONNECTOR_COMPRESSOR_SNAPPY = "snappy";
+ public static final String CONNECTOR_COMPRESSOR_GZIP = "gzip";
+ public static final String CONNECTOR_COMPRESSOR_LZ4 = "lz4";
+ public static final String CONNECTOR_COMPRESSOR_ZSTD = "zstd";
+ public static final String CONNECTOR_COMPRESSOR_LZMA2 = "lzma2";
+ public static final Set CONNECTOR_COMPRESSOR_SET =
+ Collections.unmodifiableSet(
+ new HashSet<>(
+ Arrays.asList(
+ CONNECTOR_COMPRESSOR_SNAPPY,
+ CONNECTOR_COMPRESSOR_GZIP,
+ CONNECTOR_COMPRESSOR_LZ4,
+ CONNECTOR_COMPRESSOR_ZSTD,
+ CONNECTOR_COMPRESSOR_LZMA2)));
+
+ public static final String CONNECTOR_COMPRESSOR_ZSTD_LEVEL_KEY =
+ "connector.compressor.zstd.level";
+ public static final String SINK_COMPRESSOR_ZSTD_LEVEL_KEY = "sink.compressor.zstd.level";
+ public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE =
+ Zstd.defaultCompressionLevel();
+ public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE = Zstd.minCompressionLevel();
+ public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE = Zstd.maxCompressionLevel();
+
+ public static final String CONNECTOR_RATE_LIMIT_KEY = "connector.rate-limit-bytes-per-second";
+ public static final String SINK_RATE_LIMIT_KEY = "sink.rate-limit-bytes-per-second";
+ public static final double CONNECTOR_RATE_LIMIT_DEFAULT_VALUE = -1;
+
+ public static final String CONNECTOR_FORMAT_KEY = "connector.format";
+ public static final String SINK_FORMAT_KEY = "sink.format";
+ public static final String CONNECTOR_FORMAT_TABLET_VALUE = "tablet";
+ public static final String CONNECTOR_FORMAT_TS_FILE_VALUE = "tsfile";
+ public static final String CONNECTOR_FORMAT_HYBRID_VALUE = "hybrid";
+
+ public static final String SINK_TOPIC_KEY = "sink.topic";
+ public static final String SINK_CONSUMER_GROUP_KEY = "sink.consumer-group";
+
+ public static final String CONNECTOR_CONSENSUS_GROUP_ID_KEY = "connector.consensus.group-id";
+ public static final String CONNECTOR_CONSENSUS_PIPE_NAME = "connector.consensus.pipe-name";
+
+ public static final String CONNECTOR_LOAD_TSFILE_STRATEGY_KEY = "connector.load-tsfile-strategy";
+ public static final String SINK_LOAD_TSFILE_STRATEGY_KEY = "sink.load-tsfile-strategy";
+ public static final String CONNECTOR_LOAD_TSFILE_STRATEGY_ASYNC_VALUE = "async";
+ public static final String CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE = "sync";
+ public static final Set CONNECTOR_LOAD_TSFILE_STRATEGY_SET =
+ Collections.unmodifiableSet(
+ new HashSet<>(
+ Arrays.asList(
+ CONNECTOR_LOAD_TSFILE_STRATEGY_ASYNC_VALUE,
+ CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE)));
+
+ public static final String CONNECTOR_LOAD_TSFILE_VALIDATION_KEY =
+ "connector.load-tsfile-validation";
+ public static final String SINK_LOAD_TSFILE_VALIDATION_KEY = "sink.load-tsfile-validation";
+ public static final boolean CONNECTOR_LOAD_TSFILE_VALIDATION_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_MARK_AS_PIPE_REQUEST_KEY = "connector.mark-as-pipe-request";
+ public static final String SINK_MARK_AS_PIPE_REQUEST_KEY = "sink.mark-as-pipe-request";
+ public static final boolean CONNECTOR_MARK_AS_PIPE_REQUEST_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_OPC_DA_CLSID_KEY = "connector.opcda.clsid";
+ public static final String SINK_OPC_DA_CLSID_KEY = "sink.opcda.clsid";
+
+ public static final String CONNECTOR_OPC_DA_PROGID_KEY = "connector.opcda.progid";
+ public static final String SINK_OPC_DA_PROGID_KEY = "sink.opcda.progid";
+
+ public static String getConfDir() {
+ // Check if a config-directory was specified first.
+ String confString = System.getProperty(IoTDBConstant.IOTDB_CONF, null);
+ // If it wasn't, check if a home directory was provided (This usually contains a config)
+ if (confString == null) {
+ confString = System.getProperty(IoTDBConstant.IOTDB_HOME, null);
+ if (confString != null) {
+ confString = confString + File.separatorChar + "conf";
+ }
+ }
+ return confString;
+ }
+
+ private PipeConnectorConstant() {
+ throw new IllegalStateException("Utility class");
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeExtractorConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeExtractorConstant.java
new file mode 100644
index 00000000..10b0e4ca
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeExtractorConstant.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.constant;
+
+public class PipeExtractorConstant {
+
+ public static final String EXTRACTOR_KEY = "extractor";
+ public static final String SOURCE_KEY = "source";
+
+ public static final String EXTRACTOR_CAPTURE_TREE_KEY = "extractor.capture.tree";
+ public static final String SOURCE_CAPTURE_TREE_KEY = "source.capture.tree";
+ public static final String EXTRACTOR_CAPTURE_TABLE_KEY = "extractor.capture.table";
+ public static final String SOURCE_CAPTURE_TABLE_KEY = "source.capture.table";
+
+ public static final String EXTRACTOR_INCLUSION_KEY = "extractor.inclusion";
+ public static final String SOURCE_INCLUSION_KEY = "source.inclusion";
+ public static final String EXTRACTOR_INCLUSION_DEFAULT_VALUE = "data.insert";
+
+ public static final String EXTRACTOR_EXCLUSION_KEY = "extractor.inclusion.exclusion";
+ public static final String SOURCE_EXCLUSION_KEY = "source.inclusion.exclusion";
+ public static final String EXTRACTOR_EXCLUSION_DEFAULT_VALUE = "";
+
+ public static final String EXTRACTOR_MODE_KEY = "extractor.mode";
+ public static final String SOURCE_MODE_KEY = "source.mode";
+ public static final String EXTRACTOR_MODE_QUERY_VALUE = "query";
+ public static final String EXTRACTOR_MODE_SNAPSHOT_VALUE = "snapshot";
+ public static final String EXTRACTOR_MODE_SUBSCRIBE_VALUE = "subscribe";
+ public static final String EXTRACTOR_MODE_LIVE_VALUE = "live";
+ public static final String EXTRACTOR_MODE_DEFAULT_VALUE = EXTRACTOR_MODE_LIVE_VALUE;
+
+ public static final String EXTRACTOR_PATTERN_KEY = "extractor.pattern";
+ public static final String SOURCE_PATTERN_KEY = "source.pattern";
+ public static final String EXTRACTOR_PATH_KEY = "extractor.path";
+ public static final String SOURCE_PATH_KEY = "source.path";
+ public static final String EXTRACTOR_PATTERN_FORMAT_KEY = "extractor.pattern.format";
+ public static final String SOURCE_PATTERN_FORMAT_KEY = "source.pattern.format";
+ public static final String EXTRACTOR_PATTERN_FORMAT_PREFIX_VALUE = "prefix";
+ public static final String EXTRACTOR_PATTERN_FORMAT_IOTDB_VALUE = "iotdb";
+ public static final String EXTRACTOR_PATTERN_PREFIX_DEFAULT_VALUE = "root";
+ public static final String EXTRACTOR_PATTERN_IOTDB_DEFAULT_VALUE = "root.**";
+ public static final String EXTRACTOR_DATABASE_NAME_KEY = "extractor.database-name";
+ public static final String SOURCE_DATABASE_NAME_KEY = "source.database-name";
+ public static final String EXTRACTOR_TABLE_NAME_KEY = "extractor.table-name";
+ public static final String SOURCE_TABLE_NAME_KEY = "source.table-name";
+ public static final String EXTRACTOR_DATABASE_NAME_DEFAULT_VALUE = ".*";
+ public static final String EXTRACTOR_TABLE_NAME_DEFAULT_VALUE = ".*";
+ public static final String EXTRACTOR_DATABASE_KEY = "extractor.database";
+ public static final String SOURCE_DATABASE_KEY = "source.database";
+ public static final String EXTRACTOR_TABLE_KEY = "extractor.table";
+ public static final String SOURCE_TABLE_KEY = "source.table";
+
+ public static final String EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY =
+ "extractor.forwarding-pipe-requests";
+ public static final String SOURCE_FORWARDING_PIPE_REQUESTS_KEY =
+ "source.forwarding-pipe-requests";
+ public static final boolean EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE = true;
+
+ public static final String EXTRACTOR_HISTORY_ENABLE_KEY = "extractor.history.enable";
+ public static final String SOURCE_HISTORY_ENABLE_KEY = "source.history.enable";
+ public static final boolean EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE = true;
+ public static final String EXTRACTOR_HISTORY_START_TIME_KEY = "extractor.history.start-time";
+ public static final String SOURCE_HISTORY_START_TIME_KEY = "source.history.start-time";
+ public static final String EXTRACTOR_HISTORY_END_TIME_KEY = "extractor.history.end-time";
+ public static final String SOURCE_HISTORY_END_TIME_KEY = "source.history.end-time";
+ public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_KEY = "extractor.history.loose-range";
+ public static final String SOURCE_HISTORY_LOOSE_RANGE_KEY = "source.history.loose-range";
+ public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_TIME_VALUE = "time";
+ public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_PATH_VALUE = "path";
+ public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_ALL_VALUE = "all";
+ public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE = "";
+ public static final String EXTRACTOR_MODS_ENABLE_KEY = "extractor.mods.enable";
+ public static final String SOURCE_MODS_ENABLE_KEY = "source.mods.enable";
+ public static final boolean EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE = false;
+ public static final String EXTRACTOR_MODS_KEY = "extractor.mods";
+ public static final String SOURCE_MODS_KEY = "source.mods";
+ public static final boolean EXTRACTOR_MODS_DEFAULT_VALUE = EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE;
+
+ public static final String EXTRACTOR_REALTIME_ENABLE_KEY = "extractor.realtime.enable";
+ public static final String SOURCE_REALTIME_ENABLE_KEY = "source.realtime.enable";
+ public static final boolean EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE = true;
+ public static final String EXTRACTOR_REALTIME_MODE_KEY = "extractor.realtime.mode";
+ public static final String SOURCE_REALTIME_MODE_KEY = "source.realtime.mode";
+ public static final String EXTRACTOR_REALTIME_MODE_HYBRID_VALUE = "hybrid";
+ public static final String EXTRACTOR_REALTIME_MODE_FILE_VALUE = "file";
+ public static final String EXTRACTOR_REALTIME_MODE_LOG_VALUE = "log";
+ public static final String EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE = "forced-log";
+ public static final String EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE = "stream";
+ public static final String EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE = "batch";
+ public static final String EXTRACTOR_REALTIME_LOOSE_RANGE_KEY = "extractor.realtime.loose-range";
+ public static final String SOURCE_REALTIME_LOOSE_RANGE_KEY = "source.realtime.loose-range";
+ public static final String EXTRACTOR_REALTIME_LOOSE_RANGE_TIME_VALUE = "time";
+ public static final String EXTRACTOR_REALTIME_LOOSE_RANGE_PATH_VALUE = "path";
+ public static final String EXTRACTOR_REALTIME_LOOSE_RANGE_ALL_VALUE = "all";
+ public static final String EXTRACTOR_REALTIME_LOOSE_RANGE_DEFAULT_VALUE = "";
+
+ public static final String EXTRACTOR_MODE_STREAMING_KEY = "extractor.mode.streaming";
+ public static final String SOURCE_MODE_STREAMING_KEY = "source.mode.streaming";
+ public static final boolean EXTRACTOR_MODE_STREAMING_DEFAULT_VALUE = true;
+ public static final String EXTRACTOR_MODE_STRICT_KEY = "extractor.mode.strict";
+ public static final String SOURCE_MODE_STRICT_KEY = "source.mode.strict";
+ public static final boolean EXTRACTOR_MODE_STRICT_DEFAULT_VALUE = true;
+ public static final String EXTRACTOR_MODE_SNAPSHOT_KEY = "extractor.mode.snapshot";
+ public static final String SOURCE_MODE_SNAPSHOT_KEY = "source.mode.snapshot";
+ public static final boolean EXTRACTOR_MODE_SNAPSHOT_DEFAULT_VALUE = false;
+ public static final String EXTRACTOR_MODE_DOUBLE_LIVING_KEY = "extractor.mode.double-living";
+ public static final String SOURCE_MODE_DOUBLE_LIVING_KEY = "source.mode.double-living";
+ public static final boolean EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE = false;
+
+ public static final String EXTRACTOR_START_TIME_KEY = "extractor.start-time";
+ public static final String SOURCE_START_TIME_KEY = "source.start-time";
+ public static final String EXTRACTOR_END_TIME_KEY = "extractor.end-time";
+ public static final String SOURCE_END_TIME_KEY = "source.end-time";
+ public static final String NOW_TIME_VALUE = "now";
+
+ public static final String _EXTRACTOR_WATERMARK_INTERVAL_KEY = "extractor.watermark-interval-ms";
+ public static final String _SOURCE_WATERMARK_INTERVAL_KEY = "source.watermark-interval-ms";
+ public static final long EXTRACTOR_WATERMARK_INTERVAL_DEFAULT_VALUE = -1; // -1 means no watermark
+ public static final String EXTRACTOR_WATERMARK_INTERVAL_KEY = "extractor.watermark.interval-ms";
+ public static final String SOURCE_WATERMARK_INTERVAL_KEY = "source.watermark.interval-ms";
+
+ ///////////////////// pipe consensus /////////////////////
+
+ public static final String EXTRACTOR_CONSENSUS_GROUP_ID_KEY = "extractor.consensus.group-id";
+ public static final String EXTRACTOR_CONSENSUS_SENDER_DATANODE_ID_KEY =
+ "extractor.consensus.sender-dn-id";
+ public static final String EXTRACTOR_CONSENSUS_RECEIVER_DATANODE_ID_KEY =
+ "extractor.consensus.receiver-dn-id";
+
+ private PipeExtractorConstant() {
+ throw new IllegalStateException("Utility class");
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeTransferHandshakeConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeTransferHandshakeConstant.java
new file mode 100644
index 00000000..1adcfabb
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeTransferHandshakeConstant.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.constant;
+
+public class PipeTransferHandshakeConstant {
+
+ public static final String HANDSHAKE_KEY_TIME_PRECISION = "timestampPrecision";
+ public static final String HANDSHAKE_KEY_CLUSTER_ID = "clusterID";
+ public static final String HANDSHAKE_KEY_CONVERT_ON_TYPE_MISMATCH = "convertOnTypeMismatch";
+ public static final String HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY = "loadTsFileStrategy";
+ public static final String HANDSHAKE_KEY_USERNAME = "username";
+ public static final String HANDSHAKE_KEY_PASSWORD = "password";
+ public static final String HANDSHAKE_KEY_VALIDATE_TSFILE = "validateTsFile";
+ public static final String HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST = "markAsPipeRequest";
+
+ private PipeTransferHandshakeConstant() {
+ // Utility class
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/SystemConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/SystemConstant.java
new file mode 100644
index 00000000..c196336c
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/SystemConstant.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.constant;
+
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class SystemConstant {
+
+ public static final String SYSTEM_PREFIX_KEY = "__system";
+
+ public static final String RESTART_KEY = "__system.restart";
+ public static final boolean RESTART_DEFAULT_VALUE = false;
+
+ public static final String SQL_DIALECT_KEY = "__system.sql-dialect";
+ public static final String SQL_DIALECT_TREE_VALUE = "tree";
+ public static final String SQL_DIALECT_TABLE_VALUE = "table";
+
+ /////////////////////////////////// Utility ///////////////////////////////////
+
+ public static final Set SYSTEM_KEYS = new HashSet<>();
+
+ static {
+ SYSTEM_KEYS.add(RESTART_KEY);
+ SYSTEM_KEYS.add(SQL_DIALECT_KEY);
+ }
+
+ public static PipeParameters addSystemKeysIfNecessary(final PipeParameters givenPipeParameters) {
+ final Map attributes = new HashMap<>(givenPipeParameters.getAttribute());
+ attributes.putIfAbsent(SQL_DIALECT_KEY, SQL_DIALECT_TREE_VALUE);
+ return new PipeParameters(attributes);
+ }
+
+ /////////////////////////////////// Private Constructor ///////////////////////////////////
+
+ private SystemConstant() {
+ throw new IllegalStateException("Utility class");
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeBinaryTransformer.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeBinaryTransformer.java
new file mode 100644
index 00000000..06c3fdcf
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeBinaryTransformer.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.event.row;
+
+public class PipeBinaryTransformer {
+
+ public static org.apache.tsfile.utils.Binary transformToBinary(
+ org.apache.iotdb.pipe.api.type.Binary binary) {
+ return binary == null ? null : new org.apache.tsfile.utils.Binary(binary.getValues());
+ }
+
+ public static org.apache.iotdb.pipe.api.type.Binary transformToPipeBinary(
+ org.apache.tsfile.utils.Binary binary) {
+ return binary == null ? null : new org.apache.iotdb.pipe.api.type.Binary(binary.getValues());
+ }
+
+ private PipeBinaryTransformer() {
+ // util class
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeDataTypeTransformer.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeDataTypeTransformer.java
new file mode 100644
index 00000000..a4ee4813
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeDataTypeTransformer.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.event.row;
+
+import org.apache.iotdb.pipe.api.type.Type;
+import org.apache.tsfile.enums.TSDataType;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/** Transform between {@link TSDataType} and {@link Type}. */
+public class PipeDataTypeTransformer {
+
+ public static List transformToPipeDataTypeList(final List tsDataTypeList) {
+ return tsDataTypeList == null
+ ? null
+ : tsDataTypeList.stream()
+ .map(PipeDataTypeTransformer::transformToPipeDataType)
+ .collect(Collectors.toList());
+ }
+
+ public static Type transformToPipeDataType(final TSDataType tsDataType) {
+ return tsDataType == null ? null : getPipeDataType(tsDataType.getType());
+ }
+
+ private static Type getPipeDataType(final byte type) {
+ switch (type) {
+ case 0:
+ return Type.BOOLEAN;
+ case 1:
+ return Type.INT32;
+ case 2:
+ return Type.INT64;
+ case 3:
+ return Type.FLOAT;
+ case 4:
+ return Type.DOUBLE;
+ case 5:
+ return Type.TEXT;
+ case 8:
+ return Type.TIMESTAMP;
+ case 9:
+ return Type.DATE;
+ case 10:
+ return Type.BLOB;
+ case 11:
+ return Type.STRING;
+ default:
+ throw new IllegalArgumentException("Invalid input: " + type);
+ }
+ }
+
+ private PipeDataTypeTransformer() {
+ // util class
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeResetTabletRow.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeResetTabletRow.java
new file mode 100644
index 00000000..a30aca16
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeResetTabletRow.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.event.row;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+/**
+ * The pipe framework will reset a new {@link Tablet} when this kind of {@link PipeRow} is
+ * encountered.
+ */
+public class PipeResetTabletRow extends PipeRow {
+
+ public PipeResetTabletRow(
+ int rowIndex,
+ String deviceId,
+ boolean isAligned,
+ MeasurementSchema[] measurementSchemaList,
+ long[] timestampColumn,
+ TSDataType[] valueColumnTypes,
+ Object[] valueColumns,
+ BitMap[] bitMaps,
+ String[] columnNameStringList) {
+ super(
+ rowIndex,
+ deviceId,
+ isAligned,
+ measurementSchemaList,
+ timestampColumn,
+ valueColumnTypes,
+ valueColumns,
+ bitMaps,
+ columnNameStringList);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeRow.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeRow.java
new file mode 100644
index 00000000..84d074e7
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeRow.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.event.row;
+
+import org.apache.iotdb.pipe.api.access.Row;
+import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
+import org.apache.iotdb.pipe.api.type.Binary;
+import org.apache.iotdb.pipe.api.type.Type;
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+
+import java.time.LocalDate;
+import java.util.Arrays;
+import java.util.List;
+
+public class PipeRow implements Row {
+
+ protected final int rowIndex;
+
+ protected final String deviceId;
+ protected final boolean isAligned;
+ protected final IMeasurementSchema[] measurementSchemaList;
+
+ protected final long[] timestampColumn;
+ protected final TSDataType[] valueColumnTypes;
+ protected final Object[] valueColumns;
+ protected final BitMap[] bitMaps;
+
+ protected final String[] columnNameStringList;
+
+ public PipeRow(
+ final int rowIndex,
+ final String deviceId,
+ final boolean isAligned,
+ final IMeasurementSchema[] measurementSchemaList,
+ final long[] timestampColumn,
+ final TSDataType[] valueColumnTypes,
+ final Object[] valueColumns,
+ final BitMap[] bitMaps,
+ final String[] columnNameStringList) {
+ this.rowIndex = rowIndex;
+ this.deviceId = deviceId;
+ this.isAligned = isAligned;
+ this.measurementSchemaList = measurementSchemaList;
+ this.timestampColumn = timestampColumn;
+ this.valueColumnTypes = valueColumnTypes;
+ this.valueColumns = valueColumns;
+ this.bitMaps = bitMaps;
+ this.columnNameStringList = columnNameStringList;
+ }
+
+ @Override
+ public long getTime() {
+ return timestampColumn[rowIndex];
+ }
+
+ @Override
+ public int getInt(final int columnIndex) {
+ return ((int[]) valueColumns[columnIndex])[rowIndex];
+ }
+
+ @Override
+ public LocalDate getDate(final int columnIndex) {
+ return ((LocalDate[]) valueColumns[columnIndex])[rowIndex];
+ }
+
+ @Override
+ public long getLong(final int columnIndex) {
+ return ((long[]) valueColumns[columnIndex])[rowIndex];
+ }
+
+ @Override
+ public float getFloat(final int columnIndex) {
+ return ((float[]) valueColumns[columnIndex])[rowIndex];
+ }
+
+ @Override
+ public double getDouble(final int columnIndex) {
+ return ((double[]) valueColumns[columnIndex])[rowIndex];
+ }
+
+ @Override
+ public boolean getBoolean(final int columnIndex) {
+ return ((boolean[]) valueColumns[columnIndex])[rowIndex];
+ }
+
+ @Override
+ public Binary getBinary(final int columnIndex) {
+ return PipeBinaryTransformer.transformToPipeBinary(
+ ((org.apache.tsfile.utils.Binary[]) valueColumns[columnIndex])[rowIndex]);
+ }
+
+ @Override
+ public String getString(final int columnIndex) {
+ final org.apache.tsfile.utils.Binary binary =
+ ((org.apache.tsfile.utils.Binary[]) valueColumns[columnIndex])[rowIndex];
+ return binary == null ? null : binary.getStringValue(TSFileConfig.STRING_CHARSET);
+ }
+
+ @Override
+ public Object getObject(final int columnIndex) {
+ switch (getDataType(columnIndex)) {
+ case INT32:
+ return getInt(columnIndex);
+ case DATE:
+ return getDate(columnIndex);
+ case INT64:
+ case TIMESTAMP:
+ return getLong(columnIndex);
+ case FLOAT:
+ return getFloat(columnIndex);
+ case DOUBLE:
+ return getDouble(columnIndex);
+ case BOOLEAN:
+ return getBoolean(columnIndex);
+ case TEXT:
+ case BLOB:
+ case STRING:
+ return getBinary(columnIndex);
+ default:
+ throw new UnsupportedOperationException(
+ String.format(
+ "unsupported data type %s for column %s",
+ getDataType(columnIndex), columnNameStringList[columnIndex]));
+ }
+ }
+
+ @Override
+ public Type getDataType(final int columnIndex) {
+ return PipeDataTypeTransformer.transformToPipeDataType(valueColumnTypes[columnIndex]);
+ }
+
+ @Override
+ public boolean isNull(final int columnIndex) {
+ return bitMaps[columnIndex].isMarked(rowIndex);
+ }
+
+ @Override
+ public int size() {
+ return valueColumns.length;
+ }
+
+ @Override
+ public int getColumnIndex(final Path columnName) throws PipeParameterNotValidException {
+ for (int i = 0; i < columnNameStringList.length; i++) {
+ if (columnNameStringList[i].equals(columnName.getFullPath())) {
+ return i;
+ }
+ }
+ throw new PipeParameterNotValidException(
+ String.format("column %s not found", columnName.getFullPath()));
+ }
+
+ @Override
+ public String getColumnName(final int columnIndex) {
+ return columnNameStringList[columnIndex];
+ }
+
+ @Override
+ public List getColumnTypes() {
+ return PipeDataTypeTransformer.transformToPipeDataTypeList(Arrays.asList(valueColumnTypes));
+ }
+
+ @Override
+ public String getDeviceId() {
+ return deviceId;
+ }
+
+ public boolean isAligned() {
+ return isAligned;
+ }
+
+ public int getCurrentRowSize() {
+ int rowSize = 0;
+ rowSize += 8; // timestamp
+ for (int i = 0; i < valueColumnTypes.length; i++) {
+ if (valueColumnTypes[i] != null) {
+ if (valueColumnTypes[i].isBinary()) {
+ rowSize += getBinary(i) != null ? getBinary(i).getLength() : 0;
+ } else {
+ rowSize += valueColumnTypes[i].getDataTypeSize();
+ }
+ }
+ }
+ return rowSize;
+ }
+
+ public IMeasurementSchema[] getMeasurementSchemaList() {
+ return measurementSchemaList;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeRowCollector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeRowCollector.java
new file mode 100644
index 00000000..38d8258f
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/row/PipeRowCollector.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.event.row;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.collector.plugin.builtin.sink.utils.PipeMemoryWeightUtil;
+import org.apache.iotdb.pipe.api.access.Row;
+import org.apache.iotdb.pipe.api.collector.RowCollector;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+import org.apache.iotdb.pipe.api.type.Binary;
+import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class PipeRowCollector implements RowCollector {
+
+ private final List tabletInsertionEventList = new ArrayList<>();
+ private Tablet tablet = null;
+ private boolean isAligned = false;
+ private final PipeRawTabletInsertionEvent sourceEvent; // Used to report progress
+ private final String sourceEventDataBaseName;
+ private final Boolean isTableModel;
+
+ public PipeRowCollector(PipeRawTabletInsertionEvent sourceEvent) {
+ this.sourceEvent = sourceEvent;
+ if (sourceEvent instanceof PipeInsertionEvent) {
+ sourceEventDataBaseName =
+ ((PipeInsertionEvent) sourceEvent).getSourceDatabaseNameFromDataRegion();
+ isTableModel = ((PipeInsertionEvent) sourceEvent).getRawIsTableModelEvent();
+ } else {
+ sourceEventDataBaseName = null;
+ isTableModel = null;
+ }
+ }
+
+ public PipeRowCollector(
+ PipeRawTabletInsertionEvent sourceEvent,
+ String sourceEventDataBase,
+ Boolean isTableModel) {
+ this.sourceEvent = sourceEvent;
+ this.sourceEventDataBaseName = sourceEventDataBase;
+ this.isTableModel = isTableModel;
+ }
+
+ @Override
+ public void collectRow(Row row) {
+ if (!(row instanceof PipeRow)) {
+ throw new PipeException("Row can not be customized");
+ }
+
+ final PipeRow pipeRow = (PipeRow) row;
+ final IMeasurementSchema[] measurementSchemaArray = pipeRow.getMeasurementSchemaList();
+
+ // Trigger collection when a PipeResetTabletRow is encountered
+ if (row instanceof PipeResetTabletRow) {
+ collectTabletInsertionEvent();
+ }
+
+ if (tablet == null) {
+ final String deviceId = pipeRow.getDeviceId();
+ final List measurementSchemaList =
+ new ArrayList<>(Arrays.asList(measurementSchemaArray));
+ // Calculate row count and memory size of the tablet based on the first row
+ Pair rowCountAndMemorySize =
+ PipeMemoryWeightUtil.calculateTabletRowCountAndMemory(pipeRow);
+ tablet = new Tablet(deviceId, measurementSchemaList, rowCountAndMemorySize.getLeft());
+ tablet.initBitMaps();
+ isAligned = pipeRow.isAligned();
+ }
+
+ final int rowIndex = tablet.getRowSize();
+ tablet.addTimestamp(rowIndex, row.getTime());
+ for (int i = 0; i < row.size(); i++) {
+ final Object value = row.getObject(i);
+ if (value instanceof Binary) {
+ tablet.addValue(
+ measurementSchemaArray[i].getMeasurementName(),
+ rowIndex,
+ PipeBinaryTransformer.transformToBinary((Binary) value));
+ } else {
+ tablet.addValue(measurementSchemaArray[i].getMeasurementName(), rowIndex, value);
+ }
+ if (row.isNull(i)) {
+ tablet.getBitMaps()[i].mark(rowIndex);
+ }
+ }
+
+ if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
+ collectTabletInsertionEvent();
+ }
+ }
+
+ private void collectTabletInsertionEvent() {
+ if (tablet != null) {
+ // TODO: non-PipeInsertionEvent sourceEvent is not supported?
+ final PipeInsertionEvent pipeInsertionEvent =
+ sourceEvent instanceof PipeInsertionEvent ? ((PipeInsertionEvent) sourceEvent) : null;
+ tabletInsertionEventList.add(
+ new PipeRawTabletInsertionEvent(
+ isTableModel,
+ sourceEventDataBaseName,
+ pipeInsertionEvent == null ? null : pipeInsertionEvent.getRawTableModelDataBase(),
+ pipeInsertionEvent == null ? null : pipeInsertionEvent.getRawTreeModelDataBase(),
+ tablet,
+ isAligned,
+ sourceEvent == null ? null : sourceEvent.getPipeName(),
+ sourceEvent == null ? 0 : sourceEvent.getCreationTime(),
+ sourceEvent,
+ false));
+ }
+ this.tablet = null;
+ }
+
+ public List convertToTabletInsertionEvents(final boolean shouldReport) {
+ collectTabletInsertionEvent();
+
+ final int eventListSize = tabletInsertionEventList.size();
+ if (eventListSize > 0 && shouldReport) { // The last event should report progress
+ ((PipeRawTabletInsertionEvent) tabletInsertionEventList.get(eventListSize - 1))
+ .markAsNeedToReport();
+ }
+ return tabletInsertionEventList;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/tablet/PipeRawTabletInsertionEvent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/tablet/PipeRawTabletInsertionEvent.java
new file mode 100644
index 00000000..f249143b
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/tablet/PipeRawTabletInsertionEvent.java
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.event.tablet;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.datastructure.pattern.TablePattern;
+import org.apache.iotdb.collector.plugin.builtin.sink.datastructure.pattern.TreePattern;
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tablet.parser.TabletInsertionEventParser;
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tablet.parser.TabletInsertionEventTablePatternParser;
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tablet.parser.TabletInsertionEventTreePatternParser;
+import org.apache.iotdb.collector.plugin.builtin.sink.resource.ref.PipePhantomReferenceManager;
+import org.apache.iotdb.pipe.api.access.Row;
+import org.apache.iotdb.pipe.api.collector.RowCollector;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.tsfile.utils.RamUsageEstimator;
+import org.apache.tsfile.write.record.Tablet;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiConsumer;
+
+public class PipeRawTabletInsertionEvent extends PipeInsertionEvent
+ implements TabletInsertionEvent, ReferenceTrackableEvent, AutoCloseable {
+
+ // For better calculation
+ private static final long INSTANCE_SIZE =
+ RamUsageEstimator.shallowSizeOfInstance(PipeRawTabletInsertionEvent.class);
+ private Tablet tablet;
+ private String deviceId; // Only used when the tablet is released.
+ private final boolean isAligned;
+
+ private final PipeRawTabletInsertionEvent sourceEvent;
+ private boolean needToReport;
+
+ // private final PipeTabletMemoryBlock allocatedMemoryBlock;
+
+ private TabletInsertionEventParser eventParser;
+
+ private PipeRawTabletInsertionEvent(
+ final Boolean isTableModelEvent,
+ final String databaseName,
+ final String tableModelDataBaseName,
+ final String treeModelDataBaseName,
+ final Tablet tablet,
+ final boolean isAligned,
+ final PipeRawTabletInsertionEvent sourceEvent,
+ final boolean needToReport,
+ final String pipeName,
+ final long creationTime,
+ final TreePattern treePattern,
+ final TablePattern tablePattern,
+ final long startTime,
+ final long endTime) {
+ super(
+ pipeName,
+ creationTime,
+ treePattern,
+ tablePattern,
+ startTime,
+ endTime,
+ isTableModelEvent,
+ databaseName,
+ tableModelDataBaseName,
+ treeModelDataBaseName);
+ this.tablet = Objects.requireNonNull(tablet);
+ this.isAligned = isAligned;
+ this.sourceEvent = sourceEvent;
+ this.needToReport = needToReport;
+
+ // Allocate empty memory block, will be resized later.
+ // this.allocatedMemoryBlock =
+ // PipeDataNodeResourceManager.memory().forceAllocateForTabletWithRetry(0);
+ }
+
+ public PipeRawTabletInsertionEvent(
+ final Boolean isTableModelEvent,
+ final String databaseName,
+ final String tableModelDataBaseName,
+ final String treeModelDataBaseName,
+ final Tablet tablet,
+ final boolean isAligned,
+ final String pipeName,
+ final long creationTime,
+ final PipeRawTabletInsertionEvent sourceEvent,
+ final boolean needToReport) {
+ this(
+ isTableModelEvent,
+ databaseName,
+ tableModelDataBaseName,
+ treeModelDataBaseName,
+ tablet,
+ isAligned,
+ sourceEvent,
+ needToReport,
+ pipeName,
+ creationTime,
+ null,
+ null,
+ Long.MIN_VALUE,
+ Long.MAX_VALUE);
+ }
+
+ @Override
+ public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) {
+ // PipeDataNodeResourceManager.memory()
+ // .forceResize(
+ // allocatedMemoryBlock,
+ // PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet) + INSTANCE_SIZE);
+ return true;
+ }
+
+ @Override
+ public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) {
+ // allocatedMemoryBlock.close();
+
+ // Record the deviceId before the memory is released,
+ // for later possibly updating the leader cache.
+ deviceId = tablet.getDeviceId();
+
+ // Actually release the occupied memory.
+ tablet = null;
+ eventParser = null;
+ return true;
+ }
+
+ @Override
+ public PipeRawTabletInsertionEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport(
+ final String pipeName,
+ final long creationTime,
+ final TreePattern treePattern,
+ final TablePattern tablePattern,
+ final long startTime,
+ final long endTime) {
+ return new PipeRawTabletInsertionEvent(
+ getRawIsTableModelEvent(),
+ getSourceDatabaseNameFromDataRegion(),
+ getRawTableModelDataBase(),
+ getRawTreeModelDataBase(),
+ tablet,
+ isAligned,
+ sourceEvent,
+ needToReport,
+ pipeName,
+ creationTime,
+ treePattern,
+ tablePattern,
+ startTime,
+ endTime);
+ }
+
+ @Override
+ public boolean isGeneratedByPipe() {
+ throw new UnsupportedOperationException("isGeneratedByPipe() is not supported!");
+ }
+
+ @Override
+ public boolean mayEventTimeOverlappedWithTimeRange() {
+ final long[] timestamps = tablet.getTimestamps();
+ if (Objects.isNull(timestamps) || timestamps.length == 0) {
+ return false;
+ }
+ // We assume that `timestamps` is ordered.
+ return startTime <= timestamps[timestamps.length - 1] && timestamps[0] <= endTime;
+ }
+
+ @Override
+ public boolean mayEventPathsOverlappedWithPattern() {
+ return sourceEvent == null || sourceEvent.mayEventPathsOverlappedWithPattern();
+ }
+
+ public void markAsNeedToReport() {
+ this.needToReport = true;
+ }
+
+ public String getDeviceId() {
+ // NonNull indicates that the internallyDecreaseResourceReferenceCount has not been called.
+ return Objects.nonNull(tablet) ? tablet.getDeviceId() : deviceId;
+ }
+
+ public PipeRawTabletInsertionEvent getSourceEvent() {
+ return sourceEvent;
+ }
+
+ /////////////////////////// TabletInsertionEvent ///////////////////////////
+
+ @Override
+ public Iterable processRowByRow(
+ final BiConsumer consumer) {
+ return initEventParser().processRowByRow(consumer);
+ }
+
+ @Override
+ public Iterable processTablet(
+ final BiConsumer consumer) {
+ return initEventParser().processTablet(consumer);
+ }
+
+ /////////////////////////// convertToTablet ///////////////////////////
+
+ public boolean isAligned() {
+ return isAligned;
+ }
+
+ public Tablet convertToTablet() {
+ if (!shouldParseTimeOrPattern()) {
+ return tablet;
+ }
+ return initEventParser().convertToTablet();
+ }
+
+ /////////////////////////// event parser ///////////////////////////
+
+ private TabletInsertionEventParser initEventParser() {
+ if (eventParser == null) {
+ eventParser =
+ tablet.getDeviceId().startsWith("root.")
+ ? new TabletInsertionEventTreePatternParser(this, tablet, isAligned, treePattern)
+ : new TabletInsertionEventTablePatternParser(this, tablet, isAligned, tablePattern);
+ }
+ return eventParser;
+ }
+
+ public long count() {
+ final Tablet covertedTablet = shouldParseTimeOrPattern() ? convertToTablet() : tablet;
+ return (long) covertedTablet.getRowSize() * covertedTablet.getSchemas().size();
+ }
+
+ /////////////////////////// parsePatternOrTime ///////////////////////////
+
+ public PipeRawTabletInsertionEvent parseEventWithPatternOrTime() {
+ return new PipeRawTabletInsertionEvent(
+ getRawIsTableModelEvent(),
+ getSourceDatabaseNameFromDataRegion(),
+ getRawTableModelDataBase(),
+ getRawTreeModelDataBase(),
+ convertToTablet(),
+ isAligned,
+ pipeName,
+ creationTime,
+ this,
+ needToReport);
+ }
+
+ public boolean hasNoNeedParsingAndIsEmpty() {
+ return !shouldParseTimeOrPattern() && isTabletEmpty(tablet);
+ }
+
+ public static boolean isTabletEmpty(final Tablet tablet) {
+ return Objects.isNull(tablet)
+ || tablet.getRowSize() == 0
+ || Objects.isNull(tablet.getSchemas())
+ || tablet.getSchemas().isEmpty();
+ }
+
+ /////////////////////////// Object ///////////////////////////
+
+ @Override
+ public String toString() {
+ return String.format(
+ "PipeRawTabletInsertionEvent{tablet=%s, isAligned=%s, sourceEvent=%s, needToReport=%s, eventParser=%s}",
+ tablet, isAligned, sourceEvent, needToReport, eventParser)
+ + " - "
+ + super.toString();
+ }
+
+ @Override
+ public String coreReportMessage() {
+ return String.format(
+ "PipeRawTabletInsertionEvent{tablet=%s, isAligned=%s, sourceEvent=%s, needToReport=%s}",
+ tablet,
+ isAligned,
+ sourceEvent == null ? "null" : sourceEvent.coreReportMessage(),
+ needToReport)
+ + " - "
+ + super.coreReportMessage();
+ }
+
+ /////////////////////////// ReferenceTrackableEvent ///////////////////////////
+
+ @Override
+ protected void trackResource() {
+ // PipeDataNodeResourceManager.ref().trackPipeEventResource(this, eventResourceBuilder());
+ }
+
+ @Override
+ public PipePhantomReferenceManager.PipeEventResource eventResourceBuilder() {
+ return new PipeRawTabletInsertionEventResource(this.isReleased, this.referenceCount);
+ }
+
+ private static class PipeRawTabletInsertionEventResource
+ extends PipePhantomReferenceManager.PipeEventResource {
+
+ // private final PipeTabletMemoryBlock allocatedMemoryBlock;
+
+ private PipeRawTabletInsertionEventResource(
+ final AtomicBoolean isReleased, final AtomicInteger referenceCount) {
+ super(isReleased, referenceCount);
+ }
+
+ @Override
+ protected void finalizeResource() {
+ // allocatedMemoryBlock.close();
+ }
+ }
+
+ /////////////////////////// AutoCloseable ///////////////////////////
+
+ @Override
+ public void close() {
+ // The semantic of close is to release the memory occupied by parsing, this method does nothing
+ // to unify the external close semantic:
+ // 1. PipeRawTabletInsertionEvent: the tablet occupying memory upon construction, even when
+ // parsing is involved.
+ // 2. PipeInsertNodeTabletInsertionEvent: the tablet is only constructed when it's actually
+ // involved in parsing.
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/tsfile/PipeTsFileInsertionEvent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/tsfile/PipeTsFileInsertionEvent.java
new file mode 100644
index 00000000..e3832849
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/tsfile/PipeTsFileInsertionEvent.java
@@ -0,0 +1,644 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.event.tsfile;
+
+import static org.apache.tsfile.common.constant.TsFileConstant.PATH_ROOT;
+import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.datastructure.pattern.TablePattern;
+import org.apache.iotdb.collector.plugin.builtin.sink.datastructure.pattern.TreePattern;
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tsfile.aggregator.TsFileInsertionPointCounter;
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tsfile.parser.TsFileInsertionEventParser;
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tsfile.parser.TsFileInsertionEventParserProvider;
+import org.apache.iotdb.collector.plugin.builtin.sink.resource.ref.PipePhantomReferenceManager.PipeEventResource;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.PlainDeviceID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PipeTsFileInsertionEvent extends PipeInsertionEvent
+ implements TsFileInsertionEvent, ReferenceTrackableEvent {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileInsertionEvent.class);
+
+ private static final String TREE_MODEL_EVENT_TABLE_NAME_PREFIX = PATH_ROOT + PATH_SEPARATOR;
+
+ private final TsFileResource resource;
+ private File tsFile;
+
+ // This is true iff the modFile exists and should be transferred
+ private boolean isWithMod;
+ private File modFile;
+ private final File sharedModFile;
+
+ private final boolean isLoaded;
+ private final boolean isGeneratedByPipe;
+ private final boolean isGeneratedByPipeConsensus;
+ private final boolean isGeneratedByHistoricalExtractor;
+
+ private final AtomicBoolean isClosed;
+ private final AtomicReference eventParser;
+
+ // The point count of the TsFile. Used for metrics on PipeConsensus' receiver side.
+ // May be updated after it is flushed. Should be negative if not set.
+ // private long flushPointCount = TsFileProcessor.FLUSH_POINT_COUNT_NOT_SET;
+
+ // public PipeTsFileInsertionEvent(
+ // final Boolean isTableModelEvent,
+ // final String databaseNameFromDataRegion,
+ // final TsFileResource resource,
+ // final boolean isLoaded,
+ // final boolean isGeneratedByPipe,
+ // final boolean isGeneratedByHistoricalExtractor) {
+ // // The modFile must be copied before the event is assigned to the listening pipes
+ // this(
+ // isTableModelEvent,
+ // databaseNameFromDataRegion,
+ // resource,
+ // true,
+ // isLoaded,
+ // isGeneratedByPipe,
+ // isGeneratedByHistoricalExtractor,
+ // null,
+ // 0,
+ // null,
+ // null,
+ // Long.MIN_VALUE,
+ // Long.MAX_VALUE);
+ // }
+
+ public PipeTsFileInsertionEvent(
+ final Boolean isTableModelEvent,
+ final String databaseNameFromDataRegion,
+ final TsFileResource resource,
+ final boolean isWithMod,
+ final boolean isLoaded,
+ final boolean isGeneratedByPipe,
+ final boolean isGeneratedByHistoricalExtractor,
+ final String pipeName,
+ final long creationTime,
+ final TreePattern treePattern,
+ final TablePattern tablePattern,
+ final long startTime,
+ final long endTime) {
+ super(
+ pipeName,
+ creationTime,
+ treePattern,
+ tablePattern,
+ startTime,
+ endTime,
+ isTableModelEvent,
+ databaseNameFromDataRegion);
+
+ this.resource = resource;
+ tsFile = resource.getTsFile();
+
+ this.isWithMod = isWithMod && resource.anyModFileExists();
+ this.modFile = this.isWithMod ? resource.getExclusiveModFile().getFile() : null;
+ // TODO: process the shared mod file
+ this.sharedModFile =
+ resource.getSharedModFile() != null ? resource.getSharedModFile().getFile() : null;
+
+ this.isLoaded = isLoaded;
+ this.isGeneratedByPipe = isGeneratedByPipe;
+ this.isGeneratedByPipeConsensus = resource.isGeneratedByPipeConsensus();
+ this.isGeneratedByHistoricalExtractor = isGeneratedByHistoricalExtractor;
+
+ isClosed = new AtomicBoolean(resource.isClosed());
+ // Register close listener if TsFile is not closed
+ // if (!isClosed.get()) {
+ // final TsFileProcessor processor = resource.getProcessor();
+ // if (processor != null) {
+ // processor.addCloseFileListener(
+ // o -> {
+ // synchronized (isClosed) {
+ // isClosed.set(true);
+ // isClosed.notifyAll();
+ // // Update flushPointCount after TsFile is closed
+ // flushPointCount = processor.getMemTableFlushPointCount();
+ // }
+ // });
+ // }
+ // }
+ // Check again after register close listener in case TsFile is closed during the process
+ // TsFile flushing steps:
+ // 1. Flush tsFile
+ // 2. First listener (Set resource status "closed" -> Set processor == null -> processor == null
+ // is seen)
+ // 3. Other listeners (Set "closed" status for events)
+ // Then we can imply that:
+ // 1. If the listener cannot be executed because all listeners passed, then resources status is
+ // set "closed" and can be set here
+ // 2. If the listener cannot be executed because processor == null is seen, then resources
+ // status is set "closed" and can be set here
+ // Then we know:
+ // 1. The status in the event can be closed eventually.
+ // 2. If the status is "closed", then the resource status is "closed".
+ // Then we know:
+ // If the status is "closed", then the resource status is "closed", the tsFile won't be altered
+ // and can be sent.
+ isClosed.set(resource.isClosed());
+
+ this.eventParser = new AtomicReference<>(null);
+ }
+
+ /**
+ * @return {@code false} if this file can't be sent by pipe because it is empty. {@code true}
+ * otherwise.
+ */
+ public boolean waitForTsFileClose() throws InterruptedException {
+ if (!isClosed.get()) {
+ isClosed.set(resource.isClosed());
+
+ synchronized (isClosed) {
+ while (!isClosed.get()) {
+ isClosed.wait(100);
+
+ final boolean isClosedNow = resource.isClosed();
+ if (isClosedNow) {
+ isClosed.set(true);
+ isClosed.notifyAll();
+
+ // Update flushPointCount after TsFile is closed
+ // final TsFileProcessor processor = resource.getProcessor();
+ // if (processor != null) {
+ // flushPointCount = processor.getMemTableFlushPointCount();
+ // }
+
+ break;
+ }
+ }
+ }
+ }
+
+ // From illustrations above we know If the status is "closed", then the tsFile is flushed
+ // And here we guarantee that the isEmpty() is set before flushing if tsFile is empty
+ // Then we know: "isClosed" --> tsFile flushed --> (isEmpty() <--> tsFile is empty)
+ return !resource.isEmpty();
+ }
+
+ public File getTsFile() {
+ return tsFile;
+ }
+
+ public File getModFile() {
+ return modFile;
+ }
+
+ public File getSharedModFile() {
+ return sharedModFile;
+ }
+
+ public boolean isWithMod() {
+ return isWithMod;
+ }
+
+ // If the previous "isWithMod" is false, the modFile has been set to "null", then the isWithMod
+ // can't be set to true
+ public void disableMod4NonTransferPipes(final boolean isWithMod) {
+ this.isWithMod = isWithMod && this.isWithMod;
+ }
+
+ public boolean isLoaded() {
+ return isLoaded;
+ }
+
+ public long getFileStartTime() {
+ return resource.getFileStartTime();
+ }
+
+ /**
+ * Only used for metrics on PipeConsensus' receiver side. If the event is recovered after data
+ * node's restart, the flushPointCount can be not set. It's totally fine for the PipeConsensus'
+ * receiver side. The receiver side will count the actual point count from the TsFile.
+ *
+ * If you want to get the actual point count with no risk, you can call {@link
+ * #count(boolean)}.
+ */
+ // public long getFlushPointCount() {
+ // return flushPointCount;
+ // }
+
+ public long getTimePartitionId() {
+ return resource.getTimePartition();
+ }
+
+ /////////////////////////// PipeRawTabletInsertionEvent ///////////////////////////
+
+ @Override
+ public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) {
+ try {
+ // tsFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(tsFile, true, resource);
+ // if (isWithMod) {
+ // modFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(modFile, false, null);
+ // }
+ return true;
+ } catch (final Exception e) {
+ LOGGER.warn(
+ String.format(
+ "Increase reference count for TsFile %s or modFile %s error. Holder Message: %s",
+ tsFile, modFile, holderMessage),
+ e);
+ return false;
+ }
+ }
+
+ @Override
+ public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) {
+ try {
+ // PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile);
+ // if (isWithMod) {
+ // PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile);
+ // }
+ close();
+ return true;
+ } catch (final Exception e) {
+ LOGGER.warn(
+ String.format(
+ "Decrease reference count for TsFile %s error. Holder Message: %s",
+ tsFile.getPath(), holderMessage),
+ e);
+ return false;
+ }
+ }
+
+ @Override
+ public PipeTsFileInsertionEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport(
+ final String pipeName,
+ final long creationTime,
+ final TreePattern treePattern,
+ final TablePattern tablePattern,
+ final long startTime,
+ final long endTime) {
+ return new PipeTsFileInsertionEvent(
+ getRawIsTableModelEvent(),
+ getSourceDatabaseNameFromDataRegion(),
+ resource,
+ isWithMod,
+ isLoaded,
+ isGeneratedByPipe,
+ isGeneratedByHistoricalExtractor,
+ pipeName,
+ creationTime,
+ treePattern,
+ tablePattern,
+ startTime,
+ endTime);
+ }
+
+ @Override
+ public boolean isGeneratedByPipe() {
+ return isGeneratedByPipe;
+ }
+
+ @Override
+ public boolean mayEventTimeOverlappedWithTimeRange() {
+ // If the tsFile is not closed the resource.getFileEndTime() will be Long.MIN_VALUE
+ // In that case we only judge the resource.getFileStartTime() to avoid losing data
+ return isClosed.get()
+ ? startTime <= resource.getFileEndTime() && resource.getFileStartTime() <= endTime
+ : resource.getFileStartTime() <= endTime;
+ }
+
+ @Override
+ public boolean mayEventPathsOverlappedWithPattern() {
+ if (!resource.isClosed()) {
+ return true;
+ }
+
+ try {
+ final Map deviceIsAlignedMap = null;
+ // PipeDataNodeResourceManager.tsfile()
+ // .getDeviceIsAlignedMapFromCache(
+ // PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile()),
+ // false);
+ final Set deviceSet =
+ Objects.nonNull(deviceIsAlignedMap) ? deviceIsAlignedMap.keySet() : resource.getDevices();
+ return deviceSet.stream()
+ .anyMatch(
+ deviceID -> {
+ // Tree model
+ if (Boolean.FALSE.equals(getRawIsTableModelEvent())
+ || deviceID instanceof PlainDeviceID
+ || deviceID.getTableName().startsWith(TREE_MODEL_EVENT_TABLE_NAME_PREFIX)
+ || deviceID.getTableName().equals(PATH_ROOT)) {
+ markAsTreeModelEvent();
+ return treePattern.mayOverlapWithDevice(deviceID);
+ }
+
+ // Table model
+ markAsTableModelEvent();
+ return true;
+ });
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Pipe {}: failed to get devices from TsFile {}, extract it anyway",
+ pipeName,
+ resource.getTsFilePath(),
+ e);
+ return true;
+ }
+ }
+
+ /////////////////////////// PipeInsertionEvent ///////////////////////////
+
+ @Override
+ public boolean isTableModelEvent() {
+ if (getRawIsTableModelEvent() == null) {
+ if (getSourceDatabaseNameFromDataRegion() != null) {
+ return super.isTableModelEvent();
+ }
+
+ try {
+ final Map deviceIsAlignedMap = null;
+ // PipeDataNodeResourceManager.tsfile()
+ // .getDeviceIsAlignedMapFromCache(
+ // PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+ // resource.getTsFile()),
+ // false);
+ final Set deviceSet =
+ Objects.nonNull(deviceIsAlignedMap)
+ ? deviceIsAlignedMap.keySet()
+ : resource.getDevices();
+ for (final IDeviceID deviceID : deviceSet) {
+ if (deviceID instanceof PlainDeviceID
+ || deviceID.getTableName().startsWith(TREE_MODEL_EVENT_TABLE_NAME_PREFIX)
+ || deviceID.getTableName().equals(PATH_ROOT)) {
+ markAsTreeModelEvent();
+ } else {
+ markAsTableModelEvent();
+ }
+ break;
+ }
+ } catch (final Exception e) {
+ throw new PipeException(
+ String.format(
+ "Pipe %s: failed to judge whether TsFile %s is table model or tree model",
+ pipeName, resource.getTsFilePath()),
+ e);
+ }
+ }
+
+ return getRawIsTableModelEvent();
+ }
+
+ /////////////////////////// TsFileInsertionEvent ///////////////////////////
+
+ @Override
+ public Iterable toTabletInsertionEvents() throws PipeException {
+ return toTabletInsertionEvents(Long.MAX_VALUE);
+ }
+
+ public Iterable toTabletInsertionEvents(final long timeoutMs)
+ throws PipeException {
+ try {
+ if (!waitForTsFileClose()) {
+ LOGGER.warn(
+ "Pipe skipping temporary TsFile's parsing which shouldn't be transferred: {}", tsFile);
+ return Collections.emptyList();
+ }
+ waitForResourceEnough4Parsing(timeoutMs);
+ return initEventParser().toTabletInsertionEvents();
+ } catch (final Exception e) {
+ close();
+
+ // close() should be called before re-interrupting the thread
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+
+ final String errorMsg =
+ e instanceof InterruptedException
+ ? String.format(
+ "Interrupted when waiting for closing TsFile %s.", resource.getTsFilePath())
+ : String.format(
+ "Parse TsFile %s error. Because: %s", resource.getTsFilePath(), e.getMessage());
+ LOGGER.warn(errorMsg, e);
+ throw new PipeException(errorMsg);
+ }
+ }
+
+ private void waitForResourceEnough4Parsing(final long timeoutMs) throws InterruptedException {
+ // final PipeMemoryManager memoryManager = PipeDataNodeResourceManager.memory();
+ // if (memoryManager.isEnough4TabletParsing()) {
+ // return;
+ // }
+
+ final long startTime = System.currentTimeMillis();
+ // long lastRecordTime = startTime;
+ //
+ // final long memoryCheckIntervalMs = PipeOptions.PIPE_CHECK_MEMORY_ENOUGH_INTERVAL_MS.value();
+ // while (!memoryManager.isEnough4TabletParsing()) {
+ // Thread.sleep(memoryCheckIntervalMs);
+ //
+ // final long currentTime = System.currentTimeMillis();
+ // final double elapsedRecordTimeSeconds = (currentTime - lastRecordTime) / 1000.0;
+ // final double waitTimeSeconds = (currentTime - startTime) / 1000.0;
+ // if (elapsedRecordTimeSeconds > 10.0) {
+ // LOGGER.info(
+ // "Wait for resource enough for parsing {} for {} seconds.",
+ // resource != null ? resource.getTsFilePath() : "tsfile",
+ // waitTimeSeconds);
+ // lastRecordTime = currentTime;
+ // } else if (LOGGER.isDebugEnabled()) {
+ // LOGGER.debug(
+ // "Wait for resource enough for parsing {} for {} seconds.",
+ // resource != null ? resource.getTsFilePath() : "tsfile",
+ // waitTimeSeconds);
+ // }
+ //
+ // if (waitTimeSeconds * 1000 > timeoutMs) {
+ // // should contain 'TimeoutException' in exception message
+ // throw new PipeException(
+ // String.format("TimeoutException: Waited %s seconds", waitTimeSeconds));
+ // }
+ // }
+
+ final long currentTime = System.currentTimeMillis();
+ final double waitTimeSeconds = (currentTime - startTime) / 1000.0;
+ LOGGER.info(
+ "Wait for resource enough for parsing {} for {} seconds.",
+ resource != null ? resource.getTsFilePath() : "tsfile",
+ waitTimeSeconds);
+ }
+
+ /** The method is used to prevent circular replication in PipeConsensus */
+ public boolean isGeneratedByPipeConsensus() {
+ return isGeneratedByPipeConsensus;
+ }
+
+ public boolean isGeneratedByHistoricalExtractor() {
+ return isGeneratedByHistoricalExtractor;
+ }
+
+ private TsFileInsertionEventParser initEventParser() {
+ try {
+ eventParser.compareAndSet(
+ null,
+ new TsFileInsertionEventParserProvider(
+ tsFile, treePattern, tablePattern, startTime, endTime, this)
+ .provide());
+ return eventParser.get();
+ } catch (final IOException e) {
+ close();
+
+ final String errorMsg = String.format("Read TsFile %s error.", resource.getTsFilePath());
+ LOGGER.warn(errorMsg, e);
+ throw new PipeException(errorMsg);
+ }
+ }
+
+ public long count(final boolean skipReportOnCommit) throws IOException {
+ long count = 0;
+
+ if (shouldParseTime()) {
+ try {
+ for (final TabletInsertionEvent event : toTabletInsertionEvents()) {
+ final PipeRawTabletInsertionEvent rawEvent = ((PipeRawTabletInsertionEvent) event);
+ count += rawEvent.count();
+ // if (skipReportOnCommit) {
+ // rawEvent.skipReportOnCommit();
+ // }
+ }
+ return count;
+ } finally {
+ close();
+ }
+ }
+
+ try (final TsFileInsertionPointCounter counter =
+ new TsFileInsertionPointCounter(tsFile, treePattern)) {
+ return counter.count();
+ }
+ }
+
+ /** Release the resource of {@link TsFileInsertionEventParser}. */
+ @Override
+ public void close() {
+ eventParser.getAndUpdate(
+ parser -> {
+ if (Objects.nonNull(parser)) {
+ parser.close();
+ }
+ return null;
+ });
+ }
+
+ /////////////////////////// Object ///////////////////////////
+
+ @Override
+ public String toString() {
+ return String.format(
+ "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s, isClosed=%s, eventParser=%s}",
+ resource, tsFile, isLoaded, isGeneratedByPipe, isClosed.get(), eventParser)
+ + " - "
+ + super.toString();
+ }
+
+ @Override
+ public String coreReportMessage() {
+ return String.format(
+ "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s, isClosed=%s}",
+ resource, tsFile, isLoaded, isGeneratedByPipe, isClosed.get())
+ + " - "
+ + super.coreReportMessage();
+ }
+
+ /////////////////////////// ReferenceTrackableEvent ///////////////////////////
+
+ @Override
+ public void trackResource() {
+ // PipeDataNodeResourceManager.ref().trackPipeEventResource(this, eventResourceBuilder());
+ }
+
+ @Override
+ public PipeEventResource eventResourceBuilder() {
+ return new PipeTsFileInsertionEventResource(
+ this.isReleased,
+ this.referenceCount,
+ this.tsFile,
+ this.isWithMod,
+ this.modFile,
+ this.sharedModFile,
+ this.eventParser);
+ }
+
+ private static class PipeTsFileInsertionEventResource extends PipeEventResource {
+
+ private final File tsFile;
+ private final boolean isWithMod;
+ private final File modFile;
+ private final File sharedModFile; // unused now
+ private final AtomicReference eventParser;
+
+ private PipeTsFileInsertionEventResource(
+ final AtomicBoolean isReleased,
+ final AtomicInteger referenceCount,
+ final File tsFile,
+ final boolean isWithMod,
+ final File modFile,
+ final File sharedModFile,
+ final AtomicReference eventParser) {
+ super(isReleased, referenceCount);
+ this.tsFile = tsFile;
+ this.isWithMod = isWithMod;
+ this.modFile = modFile;
+ this.sharedModFile = sharedModFile;
+ this.eventParser = eventParser;
+ }
+
+ @Override
+ protected void finalizeResource() {
+ try {
+ // decrease reference count
+ // PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile);
+ // if (isWithMod) {
+ // PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile);
+ // }
+
+ // close event parser
+ eventParser.getAndUpdate(
+ parser -> {
+ if (Objects.nonNull(parser)) {
+ parser.close();
+ }
+ return null;
+ });
+ } catch (final Exception e) {
+ LOGGER.warn("Decrease reference count for TsFile {} error.", tsFile.getPath(), e);
+ }
+ }
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeConnectorCriticalException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeConnectorCriticalException.java
new file mode 100644
index 00000000..f84a263e
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeConnectorCriticalException.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception.pipe;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeRuntimeConnectorCriticalException extends PipeRuntimeCriticalException {
+
+ public PipeRuntimeConnectorCriticalException(final String message) {
+ super(message);
+ }
+
+ public PipeRuntimeConnectorCriticalException(final String message, final long timeStamp) {
+ super(message, timeStamp);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeRuntimeConnectorCriticalException
+ && Objects.equals(getMessage(), ((PipeRuntimeConnectorCriticalException) obj).getMessage())
+ && Objects.equals(getTimeStamp(), ((PipeRuntimeException) obj).getTimeStamp());
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public void serialize(final ByteBuffer byteBuffer) {
+ PipeRuntimeExceptionType.CONNECTOR_CRITICAL_EXCEPTION.serialize(byteBuffer);
+ ReadWriteIOUtils.write(getMessage(), byteBuffer);
+ ReadWriteIOUtils.write(getTimeStamp(), byteBuffer);
+ }
+
+ @Override
+ public void serialize(final OutputStream stream) throws IOException {
+ PipeRuntimeExceptionType.CONNECTOR_CRITICAL_EXCEPTION.serialize(stream);
+ ReadWriteIOUtils.write(getMessage(), stream);
+ ReadWriteIOUtils.write(getTimeStamp(), stream);
+ }
+
+ @Override
+ public String toString() {
+ return "PipeRuntimeConnectorCriticalException{"
+ + "message='"
+ + getMessage()
+ + "', timeStamp="
+ + getTimeStamp()
+ + "}";
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeConnectorRetryTimesConfigurableException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeConnectorRetryTimesConfigurableException.java
new file mode 100644
index 00000000..8d215d8b
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeConnectorRetryTimesConfigurableException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception.pipe;
+
+
+public class PipeRuntimeConnectorRetryTimesConfigurableException
+ extends PipeRuntimeConnectorCriticalException {
+
+ private final int retryTimes;
+
+ public PipeRuntimeConnectorRetryTimesConfigurableException(
+ final String message, final int retryTimes) {
+ super(message);
+ this.retryTimes = retryTimes;
+ }
+
+ public int getRetryTimes() {
+ return retryTimes;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeCriticalException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeCriticalException.java
new file mode 100644
index 00000000..fda9c614
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeCriticalException.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception.pipe;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeRuntimeCriticalException extends PipeRuntimeException {
+
+ public PipeRuntimeCriticalException(final String message) {
+ super(message);
+ }
+
+ public PipeRuntimeCriticalException(final String message, final long timeStamp) {
+ super(message, timeStamp);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeRuntimeCriticalException
+ && Objects.equals(getMessage(), ((PipeRuntimeCriticalException) obj).getMessage())
+ && Objects.equals(getTimeStamp(), ((PipeRuntimeException) obj).getTimeStamp());
+ }
+
+ @Override
+ public void serialize(final ByteBuffer byteBuffer) {
+ PipeRuntimeExceptionType.CRITICAL_EXCEPTION.serialize(byteBuffer);
+ ReadWriteIOUtils.write(getMessage(), byteBuffer);
+ ReadWriteIOUtils.write(getTimeStamp(), byteBuffer);
+ }
+
+ @Override
+ public void serialize(final OutputStream stream) throws IOException {
+ PipeRuntimeExceptionType.CRITICAL_EXCEPTION.serialize(stream);
+ ReadWriteIOUtils.write(getMessage(), stream);
+ ReadWriteIOUtils.write(getTimeStamp(), stream);
+ }
+
+ @Override
+ public String toString() {
+ return "PipeRuntimeCriticalException{"
+ + "message='"
+ + getMessage()
+ + "', timeStamp="
+ + getTimeStamp()
+ + "}";
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeException.java
new file mode 100644
index 00000000..8e85ce86
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeException.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception.pipe;
+
+import org.apache.iotdb.pipe.api.exception.PipeException;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public abstract class PipeRuntimeException extends PipeException {
+
+ protected PipeRuntimeException(final String message) {
+ super(message);
+ }
+
+ protected PipeRuntimeException(final String message, final long timeStamp) {
+ super(message, timeStamp);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeRuntimeException
+ && Objects.equals(getMessage(), ((PipeRuntimeException) obj).getMessage())
+ && Objects.equals(getTimeStamp(), ((PipeRuntimeException) obj).getTimeStamp());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getMessage(), getTimeStamp());
+ }
+
+ public abstract void serialize(final ByteBuffer byteBuffer);
+
+ public abstract void serialize(final OutputStream stream) throws IOException;
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeExceptionType.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeExceptionType.java
new file mode 100644
index 00000000..9f39441d
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeExceptionType.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception.pipe;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+public enum PipeRuntimeExceptionType {
+ NON_CRITICAL_EXCEPTION((short) 1),
+ CRITICAL_EXCEPTION((short) 2),
+ CONNECTOR_CRITICAL_EXCEPTION((short) 3),
+ OUT_OF_MEMORY_CRITICAL_EXCEPTION((short) 4),
+ ;
+
+ private final short type;
+
+ PipeRuntimeExceptionType(short type) {
+ this.type = type;
+ }
+
+ public short getType() {
+ return type;
+ }
+
+ public void serialize(ByteBuffer byteBuffer) {
+ ReadWriteIOUtils.write(type, byteBuffer);
+ }
+
+ public void serialize(OutputStream stream) throws IOException {
+ ReadWriteIOUtils.write(type, stream);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeNonCriticalException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeNonCriticalException.java
new file mode 100644
index 00000000..a539d2e3
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeNonCriticalException.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception.pipe;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeRuntimeNonCriticalException extends PipeRuntimeException {
+
+ public PipeRuntimeNonCriticalException(String message) {
+ super(message);
+ }
+
+ public PipeRuntimeNonCriticalException(String message, long timeStamp) {
+ super(message, timeStamp);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeRuntimeNonCriticalException
+ && Objects.equals(getMessage(), ((PipeRuntimeNonCriticalException) obj).getMessage())
+ && Objects.equals(getTimeStamp(), ((PipeRuntimeException) obj).getTimeStamp());
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public void serialize(ByteBuffer byteBuffer) {
+ PipeRuntimeExceptionType.NON_CRITICAL_EXCEPTION.serialize(byteBuffer);
+ ReadWriteIOUtils.write(getMessage(), byteBuffer);
+ ReadWriteIOUtils.write(getTimeStamp(), byteBuffer);
+ }
+
+ @Override
+ public void serialize(OutputStream stream) throws IOException {
+ PipeRuntimeExceptionType.NON_CRITICAL_EXCEPTION.serialize(stream);
+ ReadWriteIOUtils.write(getMessage(), stream);
+ ReadWriteIOUtils.write(getTimeStamp(), stream);
+ }
+
+ @Override
+ public String toString() {
+ return "PipeRuntimeNonCriticalException{"
+ + "message='"
+ + getMessage()
+ + "', timeStamp="
+ + getTimeStamp()
+ + "}";
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeOutOfMemoryCriticalException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeOutOfMemoryCriticalException.java
new file mode 100644
index 00000000..82cc89bf
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/pipe/PipeRuntimeOutOfMemoryCriticalException.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception.pipe;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeRuntimeOutOfMemoryCriticalException extends PipeRuntimeCriticalException {
+
+ public PipeRuntimeOutOfMemoryCriticalException(String message) {
+ super(message);
+ }
+
+ public PipeRuntimeOutOfMemoryCriticalException(String message, long timeStamp) {
+ super(message, timeStamp);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeRuntimeOutOfMemoryCriticalException
+ && Objects.equals(
+ getMessage(), ((PipeRuntimeOutOfMemoryCriticalException) obj).getMessage())
+ && Objects.equals(getTimeStamp(), ((PipeRuntimeException) obj).getTimeStamp());
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public void serialize(ByteBuffer byteBuffer) {
+ PipeRuntimeExceptionType.OUT_OF_MEMORY_CRITICAL_EXCEPTION.serialize(byteBuffer);
+ ReadWriteIOUtils.write(getMessage(), byteBuffer);
+ ReadWriteIOUtils.write(getTimeStamp(), byteBuffer);
+ }
+
+ @Override
+ public void serialize(OutputStream stream) throws IOException {
+ PipeRuntimeExceptionType.OUT_OF_MEMORY_CRITICAL_EXCEPTION.serialize(stream);
+ ReadWriteIOUtils.write(getMessage(), stream);
+ ReadWriteIOUtils.write(getTimeStamp(), stream);
+ }
+
+ @Override
+ public String toString() {
+ return "PipeRuntimeOutOfMemoryException{"
+ + "message='"
+ + getMessage()
+ + "', timeStamp="
+ + getTimeStamp()
+ + "}";
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/limiter/GlobalRateLimiter.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/limiter/GlobalRateLimiter.java
new file mode 100644
index 00000000..fbd7062a
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/limiter/GlobalRateLimiter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.limiter;
+
+import com.google.common.util.concurrent.AtomicDouble;
+import com.google.common.util.concurrent.RateLimiter;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.iotdb.collector.config.PipeOptions;
+
+/** This is a global rate limiter for all connectors. */
+public class GlobalRateLimiter {
+
+ private final AtomicDouble throughputBytesPerSecond =
+ new AtomicDouble(PipeOptions.PIPE_ALL_SINK_RATE_LIMIT_BYTES_PER_SECOND.value());
+ private final RateLimiter rateLimiter;
+
+ public GlobalRateLimiter() {
+ final double throughputBytesPerSecondLimit = throughputBytesPerSecond.get();
+ rateLimiter =
+ throughputBytesPerSecondLimit <= 0
+ ? RateLimiter.create(Double.MAX_VALUE)
+ : RateLimiter.create(throughputBytesPerSecondLimit);
+ }
+
+ public void acquire(long bytes) {
+ if (reloadParams()) {
+ return;
+ }
+
+ while (bytes > 0) {
+ if (bytes > Integer.MAX_VALUE) {
+ tryAcquireWithRateCheck(Integer.MAX_VALUE);
+ bytes -= Integer.MAX_VALUE;
+ } else {
+ tryAcquireWithRateCheck((int) bytes);
+ return;
+ }
+ }
+ }
+
+ private void tryAcquireWithRateCheck(final int bytes) {
+ while (!rateLimiter.tryAcquire(
+ bytes,
+ PipeOptions.RATE_LIMITER_HOT_RELOAD_CHECK_INTERVAL_MS.value(),
+ TimeUnit.MILLISECONDS)) {
+ if (reloadParams()) {
+ return;
+ }
+ }
+ }
+
+ private boolean reloadParams() {
+ final double throughputBytesPerSecondLimit =
+ PipeOptions.PIPE_ALL_SINK_RATE_LIMIT_BYTES_PER_SECOND.value();
+
+ if (throughputBytesPerSecond.get() != throughputBytesPerSecondLimit) {
+ throughputBytesPerSecond.set(throughputBytesPerSecondLimit);
+ rateLimiter.setRate(
+ // if throughput <= 0, disable rate limiting
+ throughputBytesPerSecondLimit <= 0 ? Double.MAX_VALUE : throughputBytesPerSecondLimit);
+ }
+
+ // For performance, we don't need to acquire rate limiter if throughput <= 0
+ return throughputBytesPerSecondLimit <= 0;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/limiter/PipeEndPointRateLimiter.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/limiter/PipeEndPointRateLimiter.java
new file mode 100644
index 00000000..39ac4ba2
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/limiter/PipeEndPointRateLimiter.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.limiter;
+
+import com.google.common.util.concurrent.RateLimiter;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.iotdb.collector.config.PipeOptions;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+
+public class PipeEndPointRateLimiter {
+
+ // The task agent is used to check if the pipe is still alive
+
+ // private final String pipeName;
+ // private final long creationTime;
+
+ private final double bytesPerSecondLimit;
+
+ private final ConcurrentMap endPointRateLimiterMap;
+
+ public PipeEndPointRateLimiter(
+ final String pipeName, final long creationTime, final double bytesPerSecondLimit) {
+ // this.pipeName = pipeName;
+ // this.creationTime = creationTime;
+ this.bytesPerSecondLimit = bytesPerSecondLimit;
+ endPointRateLimiterMap = new ConcurrentHashMap<>();
+ }
+
+ public void acquire(final TEndPoint endPoint, long bytes) {
+ if (endPoint == null) {
+ return;
+ }
+
+ final RateLimiter rateLimiter =
+ endPointRateLimiterMap.computeIfAbsent(
+ endPoint, e -> RateLimiter.create(bytesPerSecondLimit));
+
+ while (bytes > 0) {
+ if (bytes > Integer.MAX_VALUE) {
+ if (!tryAcquireWithPipeCheck(rateLimiter, Integer.MAX_VALUE)) {
+ return;
+ }
+ bytes -= Integer.MAX_VALUE;
+ } else {
+ tryAcquireWithPipeCheck(rateLimiter, (int) bytes);
+ return;
+ }
+ }
+ }
+
+ private boolean tryAcquireWithPipeCheck(final RateLimiter rateLimiter, final int bytes) {
+ while (!rateLimiter.tryAcquire(
+ bytes,
+ PipeOptions.RATE_LIMITER_HOT_RELOAD_CHECK_INTERVAL_MS.value(),
+ TimeUnit.MILLISECONDS)) {
+ }
+ return true;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/airgap/AirGapELanguageConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/airgap/AirGapELanguageConstant.java
new file mode 100644
index 00000000..3544652d
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/airgap/AirGapELanguageConstant.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.airgap;
+
+import java.nio.charset.StandardCharsets;
+
+public class AirGapELanguageConstant {
+ public static final byte[] E_LANGUAGE_PREFIX =
+ ("" + "\n" + "" + "\n")
+ .getBytes(StandardCharsets.UTF_8);
+ public static final byte[] E_LANGUAGE_SUFFIX =
+ ("\n" + "").getBytes(StandardCharsets.UTF_8);
+
+ private AirGapELanguageConstant() {
+ // Utility class
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/airgap/AirGapOneByteResponse.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/airgap/AirGapOneByteResponse.java
new file mode 100644
index 00000000..3ff4a9bd
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/airgap/AirGapOneByteResponse.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.airgap;
+
+public class AirGapOneByteResponse {
+
+ public static final byte[] OK = new byte[] {0};
+ public static final byte[] FAIL = new byte[] {(byte) 0xFF};
+
+ private AirGapOneByteResponse() {
+ // Utility class
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/airgap/AirGapPseudoTPipeTransferRequest.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/airgap/AirGapPseudoTPipeTransferRequest.java
new file mode 100644
index 00000000..0e80a038
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/airgap/AirGapPseudoTPipeTransferRequest.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.airgap;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+public class AirGapPseudoTPipeTransferRequest extends TPipeTransferReq {}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventBatch.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventBatch.java
new file mode 100644
index 00000000..716d119e
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventBatch.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.collector.plugin.builtin.sink.protocol.thrift.async.IoTDBDataRegionAsyncConnector;
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class PipeTabletEventBatch implements AutoCloseable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventBatch.class);
+
+ protected final List events = new ArrayList<>();
+
+ private final int maxDelayInMs;
+ private long firstEventProcessingTime = Long.MIN_VALUE;
+
+ protected long totalBufferSize = 0;
+ // private final PipeMemoryBlock allocatedMemoryBlock;
+
+ protected volatile boolean isClosed = false;
+
+ protected PipeTabletEventBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) {
+ this.maxDelayInMs = maxDelayInMs;
+
+ // limit in buffer size
+ // this.allocatedMemoryBlock =
+ // PipeDataNodeResourceManager.memory()
+ // .tryAllocate(requestMaxBatchSizeInBytes)
+ // .setShrinkMethod(oldMemory -> Math.max(oldMemory / 2, 0))
+ // .setShrinkCallback(
+ // (oldMemory, newMemory) ->
+ // LOGGER.info(
+ // "The batch size limit has shrunk from {} to {}.", oldMemory, newMemory))
+ // .setExpandMethod(
+ // oldMemory -> Math.min(Math.max(oldMemory, 1) * 2, requestMaxBatchSizeInBytes))
+ // .setExpandCallback(
+ // (oldMemory, newMemory) ->
+ // LOGGER.info(
+ // "The batch size limit has expanded from {} to {}.", oldMemory, newMemory));
+
+ if (getMaxBatchSizeInBytes() != requestMaxBatchSizeInBytes) {
+ LOGGER.info(
+ "PipeTabletEventBatch: the max batch size is adjusted from {} to {} due to the "
+ + "memory restriction",
+ requestMaxBatchSizeInBytes,
+ getMaxBatchSizeInBytes());
+ }
+ }
+
+ /**
+ * Try offer {@link Event} into batch if the given {@link Event} is not duplicated.
+ *
+ * @param event the given {@link Event}
+ * @return {@code true} if the batch can be transferred
+ */
+ public synchronized boolean onEvent(final TabletInsertionEvent event)
+ throws IOException {
+ if (isClosed || !(event instanceof PipeRawTabletInsertionEvent)) {
+ return false;
+ }
+
+ // The deduplication logic here is to avoid the accumulation of
+ // the same event in a batch when retrying.
+ if (events.isEmpty() || !Objects.equals(events.get(events.size() - 1), event)) {
+ if (constructBatch(event)) {
+ events.add((PipeRawTabletInsertionEvent) event);
+ }
+
+ if (firstEventProcessingTime == Long.MIN_VALUE) {
+ firstEventProcessingTime = System.currentTimeMillis();
+ }
+ }
+
+ return shouldEmit();
+ }
+
+ /**
+ * Added an {@link TabletInsertionEvent} into batch.
+ *
+ * @param event the {@link TabletInsertionEvent} in batch
+ * @return {@code true} if the event is calculated into batch, {@code false} if the event is
+ * cached and not emitted in this batch. If there are failure encountered, just throw
+ * exceptions and do not return {@code false} here.
+ */
+ protected abstract boolean constructBatch(final TabletInsertionEvent event)
+ throws IOException;
+
+ public boolean shouldEmit() {
+ return totalBufferSize >= getMaxBatchSizeInBytes()
+ || System.currentTimeMillis() - firstEventProcessingTime >= maxDelayInMs;
+ }
+
+ private long getMaxBatchSizeInBytes() {
+ // return allocatedMemoryBlock.getMemoryUsageInBytes();
+ return 0;
+ }
+
+ public synchronized void onSuccess() {
+ events.clear();
+
+ totalBufferSize = 0;
+
+ firstEventProcessingTime = Long.MIN_VALUE;
+ }
+
+ @Override
+ public synchronized void close() {
+ isClosed = true;
+
+ events.clear();
+ }
+
+ public List deepCopyEvents() {
+ return new ArrayList<>(events);
+ }
+
+ public boolean isEmpty() {
+ return events.isEmpty();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java
new file mode 100644
index 00000000..03e536bc
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request.PipeTransferTabletBatchReqV2;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PipeTabletEventPlainBatch extends PipeTabletEventBatch {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventPlainBatch.class);
+
+ private final List binaryBuffers = new ArrayList<>();
+ private final List insertNodeBuffers = new ArrayList<>();
+ private final List tabletBuffers = new ArrayList<>();
+
+ private static final String TREE_MODEL_DATABASE_PLACEHOLDER = null;
+ private final List binaryDataBases = new ArrayList<>();
+ private final List insertNodeDataBases = new ArrayList<>();
+ private final List tabletDataBases = new ArrayList<>();
+
+ // Used to rate limit when transferring data
+ private final Map, Long> pipe2BytesAccumulated = new HashMap<>();
+
+ PipeTabletEventPlainBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) {
+ super(maxDelayInMs, requestMaxBatchSizeInBytes);
+ }
+
+ @Override
+ protected boolean constructBatch(final TabletInsertionEvent event)
+ throws IOException {
+ final int bufferSize = buildTabletInsertionBuffer(event);
+ totalBufferSize += bufferSize;
+ pipe2BytesAccumulated.compute(
+ new Pair<>(
+ ((PipeRawTabletInsertionEvent) event).getPipeName(), ((PipeRawTabletInsertionEvent) event).getCreationTime()),
+ (pipeName, bytesAccumulated) ->
+ bytesAccumulated == null ? bufferSize : bytesAccumulated + bufferSize);
+ return true;
+ }
+
+ @Override
+ public synchronized void onSuccess() {
+ super.onSuccess();
+
+ binaryBuffers.clear();
+ insertNodeBuffers.clear();
+ tabletBuffers.clear();
+
+ binaryDataBases.clear();
+ insertNodeDataBases.clear();
+ tabletDataBases.clear();
+
+ pipe2BytesAccumulated.clear();
+ }
+
+ public PipeTransferTabletBatchReqV2 toTPipeTransferReq() throws IOException {
+ return PipeTransferTabletBatchReqV2.toTPipeTransferReq(
+ binaryBuffers,
+ insertNodeBuffers,
+ tabletBuffers,
+ binaryDataBases,
+ insertNodeDataBases,
+ tabletDataBases);
+ }
+
+ public Map, Long> deepCopyPipeName2BytesAccumulated() {
+ return new HashMap<>(pipe2BytesAccumulated);
+ }
+
+ public Map, Long> getPipe2BytesAccumulated() {
+ return pipe2BytesAccumulated;
+ }
+
+ private int buildTabletInsertionBuffer(final TabletInsertionEvent event)
+ throws IOException {
+ int databaseEstimateSize = 0;
+ final ByteBuffer buffer;
+ // if (event instanceof PipeInsertNodeTabletInsertionEvent) {
+ // final PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletInsertionEvent =
+ // (PipeInsertNodeTabletInsertionEvent) event;
+ // // Read the bytebuffer from the wal file and transfer it directly without serializing or
+ // // deserializing if possible
+ // final InsertNode insertNode =
+ // pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ // if (Objects.isNull(insertNode)) {
+ // buffer = pipeInsertNodeTabletInsertionEvent.getByteBuffer();
+ // binaryBuffers.add(buffer);
+ // if (pipeInsertNodeTabletInsertionEvent.isTableModelEvent()) {
+ // databaseEstimateSize =
+ // pipeInsertNodeTabletInsertionEvent.getTableModelDatabaseName().length();
+ // binaryDataBases.add(pipeInsertNodeTabletInsertionEvent.getTableModelDatabaseName());
+ // } else {
+ // databaseEstimateSize = 4;
+ // binaryDataBases.add(TREE_MODEL_DATABASE_PLACEHOLDER);
+ // }
+ // } else {
+ // buffer = insertNode.serializeToByteBuffer();
+ // insertNodeBuffers.add(buffer);
+ // if (pipeInsertNodeTabletInsertionEvent.isTableModelEvent()) {
+ // databaseEstimateSize =
+ // pipeInsertNodeTabletInsertionEvent.getTableModelDatabaseName().length();
+ // insertNodeDataBases.add(pipeInsertNodeTabletInsertionEvent.getTableModelDatabaseName());
+ // } else {
+ // databaseEstimateSize = 4;
+ // insertNodeDataBases.add(TREE_MODEL_DATABASE_PLACEHOLDER);
+ // }
+ // }
+ // } else
+ {
+ final PipeRawTabletInsertionEvent pipeRawTabletInsertionEvent =
+ (PipeRawTabletInsertionEvent) event;
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ pipeRawTabletInsertionEvent.convertToTablet().serialize(outputStream);
+ ReadWriteIOUtils.write(pipeRawTabletInsertionEvent.isAligned(), outputStream);
+ buffer = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+ tabletBuffers.add(buffer);
+ if (pipeRawTabletInsertionEvent.isTableModelEvent()) {
+ databaseEstimateSize = pipeRawTabletInsertionEvent.getTableModelDatabaseName().length();
+ tabletDataBases.add(pipeRawTabletInsertionEvent.getTableModelDatabaseName());
+ } else {
+ databaseEstimateSize = 4;
+ tabletDataBases.add(TREE_MODEL_DATABASE_PLACEHOLDER);
+ }
+ }
+ return buffer.limit() + databaseEstimateSize;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
new file mode 100644
index 00000000..f1805cc3
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.collector.plugin.builtin.sink.utils.PipeMemoryWeightUtil;
+import org.apache.iotdb.collector.plugin.builtin.sink.utils.builder.PipeTableModeTsFileBuilder;
+import org.apache.iotdb.collector.plugin.builtin.sink.utils.builder.PipeTreeModelTsFileBuilder;
+import org.apache.iotdb.collector.plugin.builtin.sink.utils.builder.PipeTsFileBuilder;
+import org.apache.iotdb.collector.plugin.builtin.sink.utils.sorter.PipeTableModelTabletEventSorter;
+import org.apache.iotdb.collector.plugin.builtin.sink.utils.sorter.PipeTreeModelTabletEventSorter;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.write.record.Tablet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PipeTabletEventTsFileBatch extends PipeTabletEventBatch {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventTsFileBatch.class);
+
+ private static final AtomicLong BATCH_ID_GENERATOR = new AtomicLong(0);
+ private final AtomicLong currentBatchId = new AtomicLong(BATCH_ID_GENERATOR.incrementAndGet());
+
+ private final PipeTsFileBuilder treeModeTsFileBuilder;
+ private final PipeTsFileBuilder tableModeTsFileBuilder;
+
+ private final Map, Double> pipeName2WeightMap = new HashMap<>();
+
+ public PipeTabletEventTsFileBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) {
+ super(maxDelayInMs, requestMaxBatchSizeInBytes);
+
+ final AtomicLong tsFileIdGenerator = new AtomicLong(0);
+ treeModeTsFileBuilder = new PipeTreeModelTsFileBuilder(currentBatchId, tsFileIdGenerator);
+ tableModeTsFileBuilder = new PipeTableModeTsFileBuilder(currentBatchId, tsFileIdGenerator);
+ }
+
+ @Override
+ protected boolean constructBatch(final TabletInsertionEvent event) {
+ // if (event instanceof PipeInsertNodeTabletInsertionEvent) {
+ // final PipeInsertNodeTabletInsertionEvent insertNodeTabletInsertionEvent =
+ // (PipeInsertNodeTabletInsertionEvent) event;
+ // final boolean isTableModel = insertNodeTabletInsertionEvent.isTableModelEvent();
+ // final List tablets = insertNodeTabletInsertionEvent.convertToTablets();
+ // for (int i = 0; i < tablets.size(); ++i) {
+ // final Tablet tablet = tablets.get(i);
+ // if (tablet.getRowSize() == 0) {
+ // continue;
+ // }
+ // if (isTableModel) {
+ // // table Model
+ // bufferTableModelTablet(
+ // insertNodeTabletInsertionEvent.getPipeName(),
+ // insertNodeTabletInsertionEvent.getCreationTime(),
+ // tablet,
+ // insertNodeTabletInsertionEvent.getTableModelDatabaseName());
+ // } else {
+ // // tree Model
+ // bufferTreeModelTablet(
+ // insertNodeTabletInsertionEvent.getPipeName(),
+ // insertNodeTabletInsertionEvent.getCreationTime(),
+ // tablet,
+ // insertNodeTabletInsertionEvent.isAligned(i));
+ // }
+ // }
+ // } else
+ if (event instanceof PipeRawTabletInsertionEvent) {
+ final PipeRawTabletInsertionEvent rawTabletInsertionEvent =
+ (PipeRawTabletInsertionEvent) event;
+ final Tablet tablet = rawTabletInsertionEvent.convertToTablet();
+ if (tablet.getRowSize() == 0) {
+ return true;
+ }
+ if (rawTabletInsertionEvent.isTableModelEvent()) {
+ // table Model
+ bufferTableModelTablet(
+ rawTabletInsertionEvent.getPipeName(),
+ rawTabletInsertionEvent.getCreationTime(),
+ tablet,
+ rawTabletInsertionEvent.getTableModelDatabaseName());
+ } else {
+ // tree Model
+ bufferTreeModelTablet(
+ rawTabletInsertionEvent.getPipeName(),
+ rawTabletInsertionEvent.getCreationTime(),
+ tablet,
+ rawTabletInsertionEvent.isAligned());
+ }
+ } else {
+ LOGGER.warn(
+ "Batch id = {}: Unsupported event {} type {} when constructing tsfile batch",
+ currentBatchId.get(),
+ event,
+ event.getClass());
+ }
+ return true;
+ }
+
+ private void bufferTreeModelTablet(
+ final String pipeName,
+ final long creationTime,
+ final Tablet tablet,
+ final boolean isAligned) {
+ new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary();
+
+ totalBufferSize += PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet);
+
+ pipeName2WeightMap.compute(
+ new Pair<>(pipeName, creationTime),
+ (pipe, weight) -> Objects.nonNull(weight) ? ++weight : 1);
+
+ treeModeTsFileBuilder.bufferTreeModelTablet(tablet, isAligned);
+ }
+
+ private void bufferTableModelTablet(
+ final String pipeName, final long creationTime, final Tablet tablet, final String dataBase) {
+ new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByDevIdTimestamp();
+
+ totalBufferSize += PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet);
+
+ pipeName2WeightMap.compute(
+ new Pair<>(pipeName, creationTime),
+ (pipe, weight) -> Objects.nonNull(weight) ? ++weight : 1);
+
+ tableModeTsFileBuilder.bufferTableModelTablet(dataBase, tablet);
+ }
+
+ public Map, Double> deepCopyPipe2WeightMap() {
+ final double sum = pipeName2WeightMap.values().stream().reduce(Double::sum).orElse(0.0);
+ if (sum == 0.0) {
+ return Collections.emptyMap();
+ }
+ pipeName2WeightMap.entrySet().forEach(entry -> entry.setValue(entry.getValue() / sum));
+ return new HashMap<>(pipeName2WeightMap);
+ }
+
+ /**
+ * Converts a Tablet to a TSFile and returns the generated TSFile along with its corresponding
+ * database name.
+ *
+ * @return a list of pairs containing the database name and the generated TSFile
+ * @throws IOException if an I/O error occurs during the conversion process
+ * @throws WriteProcessException if an error occurs during the write process
+ */
+ public synchronized List> sealTsFiles()
+ throws IOException, WriteProcessException {
+ if (isClosed) {
+ return Collections.emptyList();
+ }
+
+ final List> list = new ArrayList<>();
+ if (!treeModeTsFileBuilder.isEmpty()) {
+ list.addAll(treeModeTsFileBuilder.convertTabletToTsFileWithDBInfo());
+ }
+ if (!tableModeTsFileBuilder.isEmpty()) {
+ list.addAll(tableModeTsFileBuilder.convertTabletToTsFileWithDBInfo());
+ }
+ return list;
+ }
+
+ @Override
+ public synchronized void onSuccess() {
+ super.onSuccess();
+
+ pipeName2WeightMap.clear();
+ tableModeTsFileBuilder.onSuccess();
+ treeModeTsFileBuilder.onSuccess();
+ }
+
+ @Override
+ public synchronized void close() {
+ super.close();
+
+ pipeName2WeightMap.clear();
+
+ tableModeTsFileBuilder.close();
+ treeModeTsFileBuilder.close();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java
new file mode 100644
index 00000000..5bbfffc2
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch;
+
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.event.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+// import org.apache.iotdb.db.pipe.connector.client.IoTDBDataNodeCacheLeaderClientManager;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_DELAY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_SIZE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_TS_FILE_BATCH_DELAY_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_FORMAT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_DELAY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_SIZE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LEADER_CACHE_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_DELAY_DEFAULT_VALUE;
+
+public class PipeTransferBatchReqBuilder implements AutoCloseable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTransferBatchReqBuilder.class);
+
+ private final boolean useLeaderCache;
+
+ private final int requestMaxDelayInMs;
+ private final long requestMaxBatchSizeInBytes;
+
+ // If the leader cache is disabled (or unable to find the endpoint of event in the leader cache),
+ // the event will be stored in the default batch.
+ private final PipeTabletEventBatch defaultBatch;
+ // If the leader cache is enabled, the batch will be divided by the leader endpoint,
+ // each endpoint has a batch.
+ // This is only used in plain batch since tsfile does not return redirection info.
+ private final Map endPointToBatch = new HashMap<>();
+
+ public PipeTransferBatchReqBuilder(final PipeParameters parameters) {
+ final boolean usingTsFileBatch =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(CONNECTOR_FORMAT_KEY, SINK_FORMAT_KEY), CONNECTOR_FORMAT_HYBRID_VALUE)
+ .equals(CONNECTOR_FORMAT_TS_FILE_VALUE);
+
+ useLeaderCache =
+ !usingTsFileBatch
+ && parameters.getBooleanOrDefault(
+ Arrays.asList(SINK_LEADER_CACHE_ENABLE_KEY, CONNECTOR_LEADER_CACHE_ENABLE_KEY),
+ CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE);
+
+ final int requestMaxDelayInSeconds;
+ if (usingTsFileBatch) {
+ requestMaxDelayInSeconds =
+ parameters.getIntOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_DELAY_KEY, SINK_IOTDB_BATCH_DELAY_KEY),
+ CONNECTOR_IOTDB_TS_FILE_BATCH_DELAY_DEFAULT_VALUE);
+ requestMaxDelayInMs =
+ requestMaxDelayInSeconds < 0 ? Integer.MAX_VALUE : requestMaxDelayInSeconds * 1000;
+ requestMaxBatchSizeInBytes =
+ parameters.getLongOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY),
+ CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE);
+ this.defaultBatch =
+ new PipeTabletEventTsFileBatch(requestMaxDelayInMs, requestMaxBatchSizeInBytes);
+ } else {
+ requestMaxDelayInSeconds =
+ parameters.getIntOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_DELAY_KEY, SINK_IOTDB_BATCH_DELAY_KEY),
+ CONNECTOR_IOTDB_PLAIN_BATCH_DELAY_DEFAULT_VALUE);
+ requestMaxDelayInMs =
+ requestMaxDelayInSeconds < 0 ? Integer.MAX_VALUE : requestMaxDelayInSeconds * 1000;
+ requestMaxBatchSizeInBytes =
+ parameters.getLongOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY),
+ CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE);
+ this.defaultBatch =
+ new PipeTabletEventPlainBatch(requestMaxDelayInMs, requestMaxBatchSizeInBytes);
+ }
+ }
+
+ /**
+ * Try offer {@link Event} into the corresponding batch if the given {@link Event} is not
+ * duplicated.
+ *
+ * @param event the given {@link Event}
+ * @return {@link Pair}<{@link TEndPoint}, {@link PipeTabletEventPlainBatch}> not null means this
+ * {@link PipeTabletEventPlainBatch} can be transferred. the first element is the leader
+ * endpoint to transfer to (might be null), the second element is the batch to be transferred.
+ */
+ public synchronized Pair onEvent(
+ final TabletInsertionEvent event)
+ throws IOException, WALPipeException, WriteProcessException {
+ if (!(event instanceof PipeRawTabletInsertionEvent)) {
+ LOGGER.warn(
+ "Unsupported event {} type {} when building transfer request", event, event.getClass());
+ return null;
+ }
+
+ if (!useLeaderCache) {
+ return defaultBatch.onEvent(event) ? new Pair<>(null, defaultBatch) : null;
+ }
+
+ String deviceId = null;
+ if (event instanceof PipeRawTabletInsertionEvent) {
+ deviceId = ((PipeRawTabletInsertionEvent) event).getDeviceId();
+ }
+ // else if (event instanceof PipeInsertNodeTabletInsertionEvent) {
+ // deviceId = ((PipeInsertNodeTabletInsertionEvent) event).getDeviceId();
+ // }
+
+ if (Objects.isNull(deviceId)) {
+ return defaultBatch.onEvent(event) ? new Pair<>(null, defaultBatch) : null;
+ }
+
+ final TEndPoint endPoint = null;
+ // IoTDBDataNodeCacheLeaderClientManager.LEADER_CACHE_MANAGER.getLeaderEndPoint(deviceId);
+ if (Objects.isNull(endPoint)) {
+ return defaultBatch.onEvent(event) ? new Pair<>(null, defaultBatch) : null;
+ }
+
+ final PipeTabletEventPlainBatch batch =
+ endPointToBatch.computeIfAbsent(
+ endPoint,
+ k -> new PipeTabletEventPlainBatch(requestMaxDelayInMs, requestMaxBatchSizeInBytes));
+ return batch.onEvent(event) ? new Pair<>(endPoint, batch) : null;
+ }
+
+ /** Get all batches that have at least 1 event. */
+ public synchronized List> getAllNonEmptyBatches() {
+ final List> nonEmptyBatches = new ArrayList<>();
+ if (!defaultBatch.isEmpty()) {
+ nonEmptyBatches.add(new Pair<>(null, defaultBatch));
+ }
+ endPointToBatch.forEach(
+ (endPoint, batch) -> {
+ if (!batch.isEmpty()) {
+ nonEmptyBatches.add(new Pair<>(endPoint, batch));
+ }
+ });
+ return nonEmptyBatches;
+ }
+
+ public boolean isEmpty() {
+ return defaultBatch.isEmpty()
+ && endPointToBatch.values().stream().allMatch(PipeTabletEventPlainBatch::isEmpty);
+ }
+
+ public synchronized void discardEventsOfPipe(final String pipeNameToDrop, final int regionId) {
+ defaultBatch.discardEventsOfPipe(pipeNameToDrop, regionId);
+ endPointToBatch.values().forEach(batch -> batch.discardEventsOfPipe(pipeNameToDrop, regionId));
+ }
+
+ @Override
+ public synchronized void close() {
+ defaultBatch.close();
+ endPointToBatch.values().forEach(PipeTabletEventPlainBatch::close);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV1Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV1Req.java
new file mode 100644
index 00000000..e277e155
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV1Req.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.IOException;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferHandshakeV1Req;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+public class PipeTransferDataNodeHandshakeV1Req extends PipeTransferHandshakeV1Req {
+
+ private PipeTransferDataNodeHandshakeV1Req() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.HANDSHAKE_DATANODE_V1;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferDataNodeHandshakeV1Req toTPipeTransferReq(
+ final String timestampPrecision) throws IOException {
+ return (PipeTransferDataNodeHandshakeV1Req)
+ new PipeTransferDataNodeHandshakeV1Req().convertToTPipeTransferReq(timestampPrecision);
+ }
+
+ public static PipeTransferDataNodeHandshakeV1Req fromTPipeTransferReq(
+ final TPipeTransferReq transferReq) {
+ return (PipeTransferDataNodeHandshakeV1Req)
+ new PipeTransferDataNodeHandshakeV1Req().translateFromTPipeTransferReq(transferReq);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(final String timestampPrecision) throws IOException {
+ return new PipeTransferDataNodeHandshakeV1Req()
+ .convertToTransferHandshakeBytes(timestampPrecision);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeTransferDataNodeHandshakeV1Req && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.java
new file mode 100644
index 00000000..207de7d2
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferHandshakeV2Req;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+public class PipeTransferDataNodeHandshakeV2Req extends PipeTransferHandshakeV2Req {
+
+ private PipeTransferDataNodeHandshakeV2Req() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.HANDSHAKE_DATANODE_V2;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferDataNodeHandshakeV2Req toTPipeTransferReq(Map params)
+ throws IOException {
+ return (PipeTransferDataNodeHandshakeV2Req)
+ new PipeTransferDataNodeHandshakeV2Req().convertToTPipeTransferReq(params);
+ }
+
+ public static PipeTransferDataNodeHandshakeV2Req fromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+ return (PipeTransferDataNodeHandshakeV2Req)
+ new PipeTransferDataNodeHandshakeV2Req().translateFromTPipeTransferReq(transferReq);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(Map params) throws IOException {
+ return new PipeTransferDataNodeHandshakeV2Req().convertToTransferHandshakeBytes(params);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeTransferDataNodeHandshakeV2Req && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotPieceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotPieceReq.java
new file mode 100644
index 00000000..beb41190
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotPieceReq.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.IOException;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFilePieceReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+public class PipeTransferSchemaSnapshotPieceReq extends PipeTransferFilePieceReq {
+
+ private PipeTransferSchemaSnapshotPieceReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferSchemaSnapshotPieceReq toTPipeTransferReq(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return (PipeTransferSchemaSnapshotPieceReq)
+ new PipeTransferSchemaSnapshotPieceReq()
+ .convertToTPipeTransferReq(fileName, startWritingOffset, filePiece);
+ }
+
+ public static PipeTransferSchemaSnapshotPieceReq fromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+ return (PipeTransferSchemaSnapshotPieceReq)
+ new PipeTransferSchemaSnapshotPieceReq().translateFromTPipeTransferReq(transferReq);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return new PipeTransferSchemaSnapshotPieceReq()
+ .convertToTPipeTransferBytes(fileName, startWritingOffset, filePiece);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeTransferSchemaSnapshotPieceReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotSealReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotSealReq.java
new file mode 100644
index 00000000..b3b7d065
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotSealReq.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.constant.ColumnHeaderConstant;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFileSealReqV2;
+import org.apache.iotdb.collector.plugin.builtin.sink.protocol.session.IClientSession;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+public class PipeTransferSchemaSnapshotSealReq extends PipeTransferFileSealReqV2 {
+
+ private PipeTransferSchemaSnapshotSealReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferSchemaSnapshotSealReq toTPipeTransferReq(
+ final String treePattern,
+ final String tablePatternDatabase,
+ final String tablePatternTable,
+ final boolean isTreeCaptured,
+ final boolean isTableCaptured,
+ final String mTreeSnapshotName,
+ final long mTreeSnapshotLength,
+ final String tLogName,
+ final long tLogLength,
+ final String attributeSnapshotName,
+ final long attributeSnapshotLength,
+ final String databaseName,
+ final String typeString)
+ throws IOException {
+ final Map parameters = new HashMap<>();
+ parameters.put(ColumnHeaderConstant.PATH_PATTERN, treePattern);
+ parameters.put(DATABASE_PATTERN, tablePatternDatabase);
+ parameters.put(ColumnHeaderConstant.TABLE_NAME, tablePatternTable);
+ if (isTreeCaptured) {
+ parameters.put(IClientSession.SqlDialect.TREE.toString(), "");
+ }
+ if (isTableCaptured) {
+ parameters.put(IClientSession.SqlDialect.TABLE.toString(), "");
+ }
+ parameters.put(ColumnHeaderConstant.DATABASE, databaseName);
+ parameters.put(ColumnHeaderConstant.TYPE, typeString);
+
+ final List fileNameList;
+ final List fileLengthList;
+
+ // Tree model sync
+ if (Objects.isNull(attributeSnapshotName)) {
+ fileNameList =
+ Objects.nonNull(tLogName)
+ ? Arrays.asList(mTreeSnapshotName, tLogName)
+ : Collections.singletonList(mTreeSnapshotName);
+ fileLengthList =
+ Objects.nonNull(tLogName)
+ ? Arrays.asList(mTreeSnapshotLength, tLogLength)
+ : Collections.singletonList(mTreeSnapshotLength);
+ } else {
+ fileNameList = Arrays.asList(mTreeSnapshotName, tLogName, attributeSnapshotName);
+ fileLengthList = Arrays.asList(mTreeSnapshotLength, tLogLength, attributeSnapshotLength);
+ }
+
+ return (PipeTransferSchemaSnapshotSealReq)
+ new PipeTransferSchemaSnapshotSealReq()
+ .convertToTPipeTransferReq(fileNameList, fileLengthList, parameters);
+ }
+
+ public static PipeTransferSchemaSnapshotSealReq fromTPipeTransferReq(final TPipeTransferReq req) {
+ return (PipeTransferSchemaSnapshotSealReq)
+ new PipeTransferSchemaSnapshotSealReq().translateFromTPipeTransferReq(req);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ final String treePattern,
+ final String tablePatternDatabase,
+ final String tablePatternTable,
+ final boolean isTreeCaptured,
+ final boolean isTableCaptured,
+ final String mTreeSnapshotName,
+ final long mTreeSnapshotLength,
+ final String tLogName,
+ final long tLogLength,
+ final String attributeSnapshotName,
+ final long attributeSnapshotLength,
+ final String databaseName,
+ final String typeString)
+ throws IOException {
+ final Map parameters = new HashMap<>();
+ parameters.put(ColumnHeaderConstant.PATH_PATTERN, treePattern);
+ parameters.put(DATABASE_PATTERN, tablePatternDatabase);
+ parameters.put(ColumnHeaderConstant.TABLE_NAME, tablePatternTable);
+ if (isTreeCaptured) {
+ parameters.put(TREE, "");
+ }
+ if (isTableCaptured) {
+ parameters.put(TABLE, "");
+ }
+ parameters.put(ColumnHeaderConstant.DATABASE, databaseName);
+ parameters.put(ColumnHeaderConstant.TYPE, typeString);
+
+ final List fileNameList;
+ final List fileLengthList;
+
+ // Tree model sync
+ if (Objects.isNull(attributeSnapshotName)) {
+ fileNameList =
+ Objects.nonNull(tLogName)
+ ? Arrays.asList(mTreeSnapshotName, tLogName)
+ : Collections.singletonList(mTreeSnapshotName);
+ fileLengthList =
+ Objects.nonNull(tLogName)
+ ? Arrays.asList(mTreeSnapshotLength, tLogLength)
+ : Collections.singletonList(mTreeSnapshotLength);
+ } else {
+ fileNameList = Arrays.asList(mTreeSnapshotName, tLogName, attributeSnapshotName);
+ fileLengthList = Arrays.asList(mTreeSnapshotLength, tLogLength, attributeSnapshotLength);
+ }
+
+ return new PipeTransferSchemaSnapshotSealReq()
+ .convertToTPipeTransferSnapshotSealBytes(fileNameList, fileLengthList, parameters);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeTransferSchemaSnapshotSealReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReq.java
new file mode 100644
index 00000000..cf154193
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReq.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.collector.plugin.builtin.sink.utils.TestOnly;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public class PipeTransferTabletBatchReq extends TPipeTransferReq {
+
+ private final transient List binaryReqs = new ArrayList<>();
+ // private final transient List insertNodeReqs = new ArrayList<>();
+ private final transient List tabletReqs = new ArrayList<>();
+
+ private PipeTransferTabletBatchReq() {
+ // Empty constructor
+ }
+
+ // public Pair constructStatements() {
+ // final InsertRowsStatement insertRowsStatement = new InsertRowsStatement();
+ // final InsertMultiTabletsStatement insertMultiTabletsStatement =
+ // new InsertMultiTabletsStatement();
+ //
+ // final List insertRowStatementList = new ArrayList<>();
+ // final List insertTabletStatementList = new ArrayList<>();
+ //
+ // for (final PipeTransferTabletBinaryReq binaryReq : binaryReqs) {
+ // final InsertBaseStatement statement = binaryReq.constructStatement();
+ // if (statement.isEmpty()) {
+ // continue;
+ // }
+ // if (statement instanceof InsertRowStatement) {
+ // insertRowStatementList.add((InsertRowStatement) statement);
+ // } else if (statement instanceof InsertTabletStatement) {
+ // insertTabletStatementList.add((InsertTabletStatement) statement);
+ // } else if (statement instanceof InsertRowsStatement) {
+ // insertRowStatementList.addAll(
+ // ((InsertRowsStatement) statement).getInsertRowStatementList());
+ // } else {
+ // throw new UnsupportedOperationException(
+ // String.format(
+ // "unknown InsertBaseStatement %s constructed from PipeTransferTabletBinaryReq.",
+ // binaryReq));
+ // }
+ // }
+ //
+ // for (final PipeTransferTabletInsertNodeReq insertNodeReq : insertNodeReqs) {
+ // final InsertBaseStatement statement = insertNodeReq.constructStatement();
+ // if (statement.isEmpty()) {
+ // continue;
+ // }
+ // if (statement instanceof InsertRowStatement) {
+ // insertRowStatementList.add((InsertRowStatement) statement);
+ // } else if (statement instanceof InsertTabletStatement) {
+ // insertTabletStatementList.add((InsertTabletStatement) statement);
+ // } else if (statement instanceof InsertRowsStatement) {
+ // insertRowStatementList.addAll(
+ // ((InsertRowsStatement) statement).getInsertRowStatementList());
+ // } else {
+ // throw new UnsupportedOperationException(
+ // String.format(
+ // "Unknown InsertBaseStatement %s constructed from PipeTransferTabletInsertNodeReq.",
+ // statement));
+ // }
+ // }
+ //
+ // for (final PipeTransferTabletRawReq tabletReq : tabletReqs) {
+ // final InsertTabletStatement statement = tabletReq.constructStatement();
+ // if (statement.isEmpty()) {
+ // continue;
+ // }
+ // insertTabletStatementList.add(statement);
+ // }
+ //
+ // insertRowsStatement.setInsertRowStatementList(insertRowStatementList);
+ // insertMultiTabletsStatement.setInsertTabletStatementList(insertTabletStatementList);
+ // return new Pair<>(insertRowsStatement, insertMultiTabletsStatement);
+ // }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletBatchReq toTPipeTransferReq(
+ final List binaryBuffers,
+ final List insertNodeBuffers,
+ final List tabletBuffers)
+ throws IOException {
+ final PipeTransferTabletBatchReq batchReq = new PipeTransferTabletBatchReq();
+
+ // batchReq.binaryReqs, batchReq.insertNodeReqs, batchReq.tabletReqs are empty
+ // when this method is called from PipeTransferTabletBatchReqBuilder.toTPipeTransferReq()
+
+ batchReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ batchReq.type = PipeRequestType.TRANSFER_TABLET_BATCH.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(binaryBuffers.size(), outputStream);
+ for (final ByteBuffer binaryBuffer : binaryBuffers) {
+ ReadWriteIOUtils.write(binaryBuffer.limit(), outputStream);
+ outputStream.write(binaryBuffer.array(), 0, binaryBuffer.limit());
+ }
+
+ ReadWriteIOUtils.write(insertNodeBuffers.size(), outputStream);
+ for (final ByteBuffer insertNodeBuffer : insertNodeBuffers) {
+ outputStream.write(insertNodeBuffer.array(), 0, insertNodeBuffer.limit());
+ }
+
+ ReadWriteIOUtils.write(tabletBuffers.size(), outputStream);
+ for (final ByteBuffer tabletBuffer : tabletBuffers) {
+ outputStream.write(tabletBuffer.array(), 0, tabletBuffer.limit());
+ }
+
+ batchReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return batchReq;
+ }
+
+ // public static PipeTransferTabletBatchReq fromTPipeTransferReq(
+ // final TPipeTransferReq transferReq) {
+ // final PipeTransferTabletBatchReq batchReq = new PipeTransferTabletBatchReq();
+ //
+ // int size = ReadWriteIOUtils.readInt(transferReq.body);
+ // for (int i = 0; i < size; ++i) {
+ // final int length = ReadWriteIOUtils.readInt(transferReq.body);
+ // final byte[] body = new byte[length];
+ // transferReq.body.get(body);
+ // batchReq.binaryReqs.add(
+ // PipeTransferTabletBinaryReq.toTPipeTransferReq(ByteBuffer.wrap(body)));
+ // }
+ //
+ // size = ReadWriteIOUtils.readInt(transferReq.body);
+ // for (int i = 0; i < size; ++i) {
+ // batchReq.insertNodeReqs.add(
+ // PipeTransferTabletInsertNodeReq.toTPipeTransferRawReq(
+ // (InsertNode) PlanFragment.deserializeHelper(transferReq.body, null)));
+ // }
+ //
+ // size = ReadWriteIOUtils.readInt(transferReq.body);
+ // for (int i = 0; i < size; ++i) {
+ // batchReq.tabletReqs.add(
+ // PipeTransferTabletRawReq.toTPipeTransferRawReq(
+ // Tablet.deserialize(transferReq.body), ReadWriteIOUtils.readBool(transferReq.body)));
+ // }
+ //
+ // batchReq.version = transferReq.version;
+ // batchReq.type = transferReq.type;
+ // batchReq.body = transferReq.body;
+ //
+ // return batchReq;
+ // }
+
+ /////////////////////////////// TestOnly ///////////////////////////////
+
+ @TestOnly
+ public List getBinaryReqs() {
+ return binaryReqs;
+ }
+
+ // @TestOnly
+ // public List getInsertNodeReqs() {
+ // return insertNodeReqs;
+ // }
+
+ @TestOnly
+ public List getTabletReqs() {
+ return tabletReqs;
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeTransferTabletBatchReq that = (PipeTransferTabletBatchReq) obj;
+ return binaryReqs.equals(that.binaryReqs)
+ // && insertNodeReqs.equals(that.insertNodeReqs)
+ && tabletReqs.equals(that.tabletReqs)
+ && version == that.version
+ && type == that.type
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(binaryReqs, /*insertNodeReqs,*/ tabletReqs, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java
new file mode 100644
index 00000000..573c2290
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.collector.plugin.builtin.sink.utils.TestOnly;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public class PipeTransferTabletBatchReqV2 extends TPipeTransferReq {
+
+ private final transient List binaryReqs = new ArrayList<>();
+ private final transient List insertNodeReqs =
+ new ArrayList<>();
+ private final transient List tabletReqs = new ArrayList<>();
+
+ private PipeTransferTabletBatchReqV2() {
+ // Empty constructor
+ }
+
+ // public List constructStatements() {
+ // final List statements = new ArrayList<>();
+ //
+ // final InsertRowsStatement insertRowsStatement = new InsertRowsStatement();
+ // final InsertMultiTabletsStatement insertMultiTabletsStatement =
+ // new InsertMultiTabletsStatement();
+ //
+ // final List insertRowStatementList = new ArrayList<>();
+ // final List insertTabletStatementList = new ArrayList<>();
+ //
+ // for (final PipeTransferTabletBinaryReqV2 binaryReq : binaryReqs) {
+ // final InsertBaseStatement statement = binaryReq.constructStatement();
+ // if (statement.isEmpty()) {
+ // continue;
+ // }
+ // if (statement.isWriteToTable()) {
+ // statements.add(statement);
+ // continue;
+ // }
+ // if (statement instanceof InsertRowStatement) {
+ // insertRowStatementList.add((InsertRowStatement) statement);
+ // } else if (statement instanceof InsertTabletStatement) {
+ // insertTabletStatementList.add((InsertTabletStatement) statement);
+ // } else if (statement instanceof InsertRowsStatement) {
+ // insertRowStatementList.addAll(
+ // ((InsertRowsStatement) statement).getInsertRowStatementList());
+ // } else {
+ // throw new UnsupportedOperationException(
+ // String.format(
+ // "unknown InsertBaseStatement %s constructed from PipeTransferTabletBinaryReqV2.",
+ // binaryReq));
+ // }
+ // }
+ //
+ // for (final PipeTransferTabletInsertNodeReqV2 insertNodeReq : insertNodeReqs) {
+ // final InsertBaseStatement statement = insertNodeReq.constructStatement();
+ // if (statement.isEmpty()) {
+ // continue;
+ // }
+ // if (statement.isWriteToTable()) {
+ // statements.add(statement);
+ // continue;
+ // }
+ // if (statement instanceof InsertRowStatement) {
+ // insertRowStatementList.add((InsertRowStatement) statement);
+ // } else if (statement instanceof InsertTabletStatement) {
+ // insertTabletStatementList.add((InsertTabletStatement) statement);
+ // } else if (statement instanceof InsertRowsStatement) {
+ // insertRowStatementList.addAll(
+ // ((InsertRowsStatement) statement).getInsertRowStatementList());
+ // } else {
+ // throw new UnsupportedOperationException(
+ // String.format(
+ // "Unknown InsertBaseStatement %s constructed from PipeTransferTabletInsertNodeReqV2.",
+ // statement));
+ // }
+ // }
+ //
+ // for (final PipeTransferTabletRawReqV2 tabletReq : tabletReqs) {
+ // final InsertTabletStatement statement = tabletReq.constructStatement();
+ // if (statement.isEmpty()) {
+ // continue;
+ // }
+ // if (statement.isWriteToTable()) {
+ // statements.add(statement);
+ // continue;
+ // }
+ // insertTabletStatementList.add(statement);
+ // }
+ //
+ // insertRowsStatement.setInsertRowStatementList(insertRowStatementList);
+ // insertMultiTabletsStatement.setInsertTabletStatementList(insertTabletStatementList);
+ // if (!insertRowsStatement.isEmpty()) {
+ // statements.add(insertRowsStatement);
+ // }
+ // if (!insertMultiTabletsStatement.isEmpty()) {
+ // statements.add(insertMultiTabletsStatement);
+ // }
+ // return statements;
+ // }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletBatchReqV2 toTPipeTransferReq(
+ final List binaryBuffers,
+ final List insertNodeBuffers,
+ final List tabletBuffers,
+ final List binaryDataBases,
+ final List insertNodeDataBases,
+ final List tabletDataBases)
+ throws IOException {
+ final PipeTransferTabletBatchReqV2 batchReq = new PipeTransferTabletBatchReqV2();
+
+ batchReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ batchReq.type = PipeRequestType.TRANSFER_TABLET_BATCH_V2.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(binaryBuffers.size(), outputStream);
+ for (int i = 0; i < binaryBuffers.size(); i++) {
+ final ByteBuffer binaryBuffer = binaryBuffers.get(i);
+ ReadWriteIOUtils.write(binaryBuffer.limit(), outputStream);
+ outputStream.write(binaryBuffer.array(), 0, binaryBuffer.limit());
+ ReadWriteIOUtils.write(binaryDataBases.get(i), outputStream);
+ }
+
+ ReadWriteIOUtils.write(insertNodeBuffers.size(), outputStream);
+ for (int i = 0; i < insertNodeBuffers.size(); i++) {
+ final ByteBuffer insertNodeBuffer = insertNodeBuffers.get(i);
+ outputStream.write(insertNodeBuffer.array(), 0, insertNodeBuffer.limit());
+ ReadWriteIOUtils.write(insertNodeDataBases.get(i), outputStream);
+ }
+
+ ReadWriteIOUtils.write(tabletBuffers.size(), outputStream);
+ for (int i = 0; i < tabletBuffers.size(); i++) {
+ final ByteBuffer tabletBuffer = tabletBuffers.get(i);
+ outputStream.write(tabletBuffer.array(), 0, tabletBuffer.limit());
+ ReadWriteIOUtils.write(tabletDataBases.get(i), outputStream);
+ }
+
+ batchReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return batchReq;
+ }
+
+ // public static PipeTransferTabletBatchReqV2 fromTPipeTransferReq(
+ // final TPipeTransferReq transferReq) {
+ // final PipeTransferTabletBatchReqV2 batchReq = new PipeTransferTabletBatchReqV2();
+ //
+ // int size = ReadWriteIOUtils.readInt(transferReq.body);
+ // for (int i = 0; i < size; ++i) {
+ // final int length = ReadWriteIOUtils.readInt(transferReq.body);
+ // final byte[] body = new byte[length];
+ // transferReq.body.get(body);
+ // batchReq.binaryReqs.add(
+ // PipeTransferTabletBinaryReqV2.toTPipeTransferBinaryReq(
+ // ByteBuffer.wrap(body), ReadWriteIOUtils.readString(transferReq.body)));
+ // }
+ //
+ // size = ReadWriteIOUtils.readInt(transferReq.body);
+ // for (int i = 0; i < size; ++i) {
+ // batchReq.insertNodeReqs.add(
+ // PipeTransferTabletInsertNodeReqV2.toTabletInsertNodeReq(
+ // (InsertNode) PlanFragment.deserializeHelper(transferReq.body, null),
+ // ReadWriteIOUtils.readString(transferReq.body)));
+ // }
+ //
+ // size = ReadWriteIOUtils.readInt(transferReq.body);
+ // for (int i = 0; i < size; ++i) {
+ // batchReq.tabletReqs.add(
+ // PipeTransferTabletRawReqV2.toTPipeTransferRawReq(
+ // Tablet.deserialize(transferReq.body),
+ // ReadWriteIOUtils.readBool(transferReq.body),
+ // ReadWriteIOUtils.readString(transferReq.body)));
+ // }
+ //
+ // batchReq.version = transferReq.version;
+ // batchReq.type = transferReq.type;
+ // batchReq.body = transferReq.body;
+ //
+ // return batchReq;
+ // }
+
+ /////////////////////////////// TestOnly ///////////////////////////////
+
+ @TestOnly
+ public List getBinaryReqs() {
+ return binaryReqs;
+ }
+
+ @TestOnly
+ public List getInsertNodeReqs() {
+ return insertNodeReqs;
+ }
+
+ @TestOnly
+ public List getTabletReqs() {
+ return tabletReqs;
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeTransferTabletBatchReqV2 that = (PipeTransferTabletBatchReqV2) obj;
+ return Objects.equals(binaryReqs, that.binaryReqs)
+ && Objects.equals(insertNodeReqs, that.insertNodeReqs)
+ && Objects.equals(tabletReqs, that.tabletReqs)
+ && version == that.version
+ && type == that.type
+ && Objects.equals(body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(binaryReqs, insertNodeReqs, tabletReqs, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java
new file mode 100644
index 00000000..e1d31020
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.BytesUtils;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public class PipeTransferTabletBinaryReq extends TPipeTransferReq {
+
+ protected transient ByteBuffer byteBuffer;
+
+ protected PipeTransferTabletBinaryReq() {
+ // Do nothing
+ }
+
+ public ByteBuffer getByteBuffer() {
+ return byteBuffer;
+ }
+
+ // public InsertBaseStatement constructStatement() {
+ // final InsertNode insertNode = parseByteBuffer();
+ //
+ // if (!(insertNode instanceof InsertRowNode
+ // || insertNode instanceof InsertTabletNode
+ // || insertNode instanceof InsertRowsNode)) {
+ // throw new UnsupportedOperationException(
+ // String.format(
+ // "Unknown InsertNode type %s when constructing statement from insert node.",
+ // insertNode));
+ // }
+ //
+ // return (InsertBaseStatement)
+ // IoTDBDataNodeReceiver.PLAN_TO_STATEMENT_VISITOR.process(insertNode, null);
+ // }
+ //
+ // protected InsertNode parseByteBuffer() {
+ // final PlanNode node = WALEntry.deserializeForConsensus(byteBuffer);
+ // return node instanceof InsertNode ? (InsertNode) node : null;
+ // }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletBinaryReq toTPipeTransferReq(final ByteBuffer byteBuffer) {
+ final PipeTransferTabletBinaryReq req = new PipeTransferTabletBinaryReq();
+ req.byteBuffer = byteBuffer;
+
+ req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ req.type = PipeRequestType.TRANSFER_TABLET_BINARY.getType();
+ req.body = byteBuffer;
+
+ return req;
+ }
+
+ public static PipeTransferTabletBinaryReq fromTPipeTransferReq(
+ final TPipeTransferReq transferReq) {
+ final PipeTransferTabletBinaryReq binaryReq = new PipeTransferTabletBinaryReq();
+ binaryReq.byteBuffer = transferReq.body;
+
+ binaryReq.version = transferReq.version;
+ binaryReq.type = transferReq.type;
+ binaryReq.body = transferReq.body;
+
+ return binaryReq;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(final ByteBuffer byteBuffer) throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_BINARY.getType(), outputStream);
+ return BytesUtils.concatByteArray(byteArrayOutputStream.toByteArray(), byteBuffer.array());
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeTransferTabletBinaryReq that = (PipeTransferTabletBinaryReq) obj;
+ return byteBuffer.equals(that.byteBuffer)
+ && version == that.version
+ && type == that.type
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(byteBuffer, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java
new file mode 100644
index 00000000..22fc43b6
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public class PipeTransferTabletBinaryReqV2 extends PipeTransferTabletBinaryReq {
+
+ private transient String dataBaseName;
+
+ protected PipeTransferTabletBinaryReqV2() {
+ // Do nothing
+ }
+
+ public String getDataBaseName() {
+ return dataBaseName;
+ }
+
+ // public InsertBaseStatement constructStatement() {
+ // final InsertNode insertNode = parseByteBuffer();
+ //
+ // if (!(insertNode instanceof InsertRowNode
+ // || insertNode instanceof InsertTabletNode
+ // || insertNode instanceof InsertRowsNode)) {
+ // throw new UnsupportedOperationException(
+ // String.format(
+ // "Unknown InsertNode type %s when constructing statement from insert node.",
+ // insertNode));
+ // }
+ //
+ // final InsertBaseStatement statement =
+ // (InsertBaseStatement)
+ // IoTDBDataNodeReceiver.PLAN_TO_STATEMENT_VISITOR.process(insertNode, null);
+ //
+ // // Tree model
+ // if (Objects.isNull(dataBaseName)) {
+ // return statement;
+ // }
+ //
+ // // Table model
+ // statement.setWriteToTable(true);
+ // if (statement instanceof InsertRowsStatement) {
+ // List rowStatements =
+ // ((InsertRowsStatement) statement).getInsertRowStatementList();
+ // if (rowStatements != null && !rowStatements.isEmpty()) {
+ // for (InsertRowStatement insertRowStatement : rowStatements) {
+ // insertRowStatement.setWriteToTable(true);
+ // insertRowStatement.setDatabaseName(dataBaseName);
+ // }
+ // }
+ // }
+ // statement.setDatabaseName(dataBaseName);
+ // return statement;
+ // }
+
+ /////////////////////////////// Batch ///////////////////////////////
+
+ public static PipeTransferTabletBinaryReqV2 toTPipeTransferBinaryReq(
+ final ByteBuffer byteBuffer, final String dataBaseName) {
+ final PipeTransferTabletBinaryReqV2 req = new PipeTransferTabletBinaryReqV2();
+
+ req.byteBuffer = byteBuffer;
+ req.dataBaseName = dataBaseName;
+ req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ req.type = PipeRequestType.TRANSFER_TABLET_BINARY_V2.getType();
+
+ return req;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletBinaryReqV2 toTPipeTransferReq(
+ final ByteBuffer byteBuffer, final String dataBaseName) throws IOException {
+ final PipeTransferTabletBinaryReqV2 req = new PipeTransferTabletBinaryReqV2();
+ req.byteBuffer = byteBuffer;
+ req.dataBaseName = dataBaseName;
+
+ req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ req.type = PipeRequestType.TRANSFER_TABLET_BINARY_V2.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(byteBuffer.limit(), outputStream);
+ outputStream.write(byteBuffer.array(), 0, byteBuffer.limit());
+ ReadWriteIOUtils.write(dataBaseName, outputStream);
+ req.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return req;
+ }
+
+ public static PipeTransferTabletBinaryReqV2 fromTPipeTransferReq(
+ final org.apache.iotdb.service.rpc.thrift.TPipeTransferReq transferReq) {
+ final PipeTransferTabletBinaryReqV2 binaryReq = new PipeTransferTabletBinaryReqV2();
+
+ final int length = ReadWriteIOUtils.readInt(transferReq.body);
+ final byte[] body = new byte[length];
+ transferReq.body.get(body);
+ binaryReq.byteBuffer = ByteBuffer.wrap(body);
+ binaryReq.dataBaseName = ReadWriteIOUtils.readString(transferReq.body);
+
+ binaryReq.version = transferReq.version;
+ binaryReq.type = transferReq.type;
+ binaryReq.body = transferReq.body;
+
+ return binaryReq;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(final ByteBuffer byteBuffer, final String dataBaseName)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_BINARY_V2.getType(), outputStream);
+ ReadWriteIOUtils.write(byteBuffer.limit(), outputStream);
+ outputStream.write(byteBuffer.array(), 0, byteBuffer.limit());
+ ReadWriteIOUtils.write(dataBaseName, outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ final PipeTransferTabletBinaryReqV2 that = (PipeTransferTabletBinaryReqV2) o;
+ return Objects.equals(dataBaseName, that.dataBaseName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), dataBaseName);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReq.java
new file mode 100644
index 00000000..226e28b7
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReq.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.apache.tsfile.write.record.Tablet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PipeTransferTabletRawReq extends TPipeTransferReq {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTransferTabletRawReq.class);
+
+ protected transient Tablet tablet;
+ protected transient boolean isAligned;
+
+ public Tablet getTablet() {
+ return tablet;
+ }
+
+ public boolean getIsAligned() {
+ return isAligned;
+ }
+
+ // public InsertTabletStatement constructStatement() {
+ // new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary();
+ //
+ // try {
+ // if (isTabletEmpty(tablet)) {
+ // // Empty statement, will be filtered after construction
+ // return new InsertTabletStatement();
+ // }
+ //
+ // final TSInsertTabletReq request = new TSInsertTabletReq();
+ //
+ // for (final IMeasurementSchema measurementSchema : tablet.getSchemas()) {
+ // request.addToMeasurements(measurementSchema.getMeasurementName());
+ // request.addToTypes(measurementSchema.getType().ordinal());
+ // }
+ //
+ // request.setPrefixPath(tablet.getDeviceId());
+ // request.setIsAligned(isAligned);
+ // request.setTimestamps(SessionUtils.getTimeBuffer(tablet));
+ // request.setValues(SessionUtils.getValueBuffer(tablet));
+ // request.setSize(tablet.getRowSize());
+ // request.setMeasurements(
+ // PathUtils.checkIsLegalSingleMeasurementsAndUpdate(request.getMeasurements()));
+ //
+ // return StatementGenerator.createStatement(request);
+ // } catch (final MetadataException e) {
+ // LOGGER.warn("Generate Statement from tablet {} error.", tablet, e);
+ // return null;
+ // }
+ // }
+
+ /////////////////////////////// WriteBack & Batch ///////////////////////////////
+
+ public static PipeTransferTabletRawReq toTPipeTransferRawReq(
+ final Tablet tablet, final boolean isAligned) {
+ final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq();
+
+ tabletReq.tablet = tablet;
+ tabletReq.isAligned = isAligned;
+
+ return tabletReq;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletRawReq toTPipeTransferReq(
+ final Tablet tablet, final boolean isAligned) throws IOException {
+ final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq();
+
+ tabletReq.tablet = tablet;
+ tabletReq.isAligned = isAligned;
+
+ tabletReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ tabletReq.type = PipeRequestType.TRANSFER_TABLET_RAW.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ tablet.serialize(outputStream);
+ ReadWriteIOUtils.write(isAligned, outputStream);
+ tabletReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return tabletReq;
+ }
+
+ public static PipeTransferTabletRawReq fromTPipeTransferReq(final TPipeTransferReq transferReq) {
+ final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq();
+
+ tabletReq.tablet = Tablet.deserialize(transferReq.body);
+ tabletReq.isAligned = ReadWriteIOUtils.readBool(transferReq.body);
+
+ tabletReq.version = transferReq.version;
+ tabletReq.type = transferReq.type;
+ tabletReq.body = transferReq.body;
+
+ return tabletReq;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(final Tablet tablet, final boolean isAligned)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), outputStream);
+ tablet.serialize(outputStream);
+ ReadWriteIOUtils.write(isAligned, outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeTransferTabletRawReq that = (PipeTransferTabletRawReq) obj;
+ return Objects.equals(tablet, that.tablet)
+ && isAligned == that.isAligned
+ && version == that.version
+ && type == that.type
+ && Objects.equals(body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tablet, isAligned, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java
new file mode 100644
index 00000000..7e5eb180
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.apache.tsfile.write.record.Tablet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PipeTransferTabletRawReqV2 extends PipeTransferTabletRawReq {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTransferTabletRawReqV2.class);
+
+ protected transient String dataBaseName;
+
+ public String getDataBaseName() {
+ return dataBaseName;
+ }
+
+ // @Override
+ // public InsertTabletStatement constructStatement() {
+ // if (Objects.isNull(dataBaseName)) {
+ // new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary();
+ // } else {
+ // new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByTimestampIfNecessary();
+ // }
+ //
+ // try {
+ // if (isTabletEmpty(tablet)) {
+ // // Empty statement, will be filtered after construction
+ // return new InsertTabletStatement();
+ // }
+ //
+ // final TSInsertTabletReq request = new TSInsertTabletReq();
+ //
+ // for (final IMeasurementSchema measurementSchema : tablet.getSchemas()) {
+ // request.addToMeasurements(measurementSchema.getMeasurementName());
+ // request.addToTypes(measurementSchema.getType().ordinal());
+ // }
+ //
+ // request.setPrefixPath(tablet.getDeviceId());
+ // request.setIsAligned(isAligned);
+ // request.setTimestamps(SessionUtils.getTimeBuffer(tablet));
+ // request.setValues(SessionUtils.getValueBuffer(tablet));
+ // request.setSize(tablet.getRowSize());
+ //
+ // // Tree model
+ // if (Objects.isNull(dataBaseName)) {
+ // request.setMeasurements(
+ // PathUtils.checkIsLegalSingleMeasurementsAndUpdate(request.getMeasurements()));
+ // return StatementGenerator.createStatement(request);
+ // }
+ //
+ // // Table model
+ // request.setWriteToTable(true);
+ // request.columnCategories =
+ // tablet.getColumnTypes().stream()
+ // .map(t -> (byte) t.ordinal())
+ // .collect(Collectors.toList());
+ // final InsertTabletStatement statement = StatementGenerator.createStatement(request);
+ // statement.setDatabaseName(dataBaseName);
+ // return statement;
+ // } catch (final MetadataException e) {
+ // LOGGER.warn("Generate Statement from tablet {} error.", tablet, e);
+ // return null;
+ // }
+ // }
+
+ /////////////////////////////// WriteBack & Batch ///////////////////////////////
+
+ public static PipeTransferTabletRawReqV2 toTPipeTransferRawReq(
+ final Tablet tablet, final boolean isAligned, final String dataBaseName) {
+ final PipeTransferTabletRawReqV2 tabletReq = new PipeTransferTabletRawReqV2();
+
+ tabletReq.tablet = tablet;
+ tabletReq.isAligned = isAligned;
+ tabletReq.dataBaseName = dataBaseName;
+ tabletReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ tabletReq.type = PipeRequestType.TRANSFER_TABLET_RAW_V2.getType();
+
+ return tabletReq;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletRawReqV2 toTPipeTransferReq(
+ final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException {
+ final PipeTransferTabletRawReqV2 tabletReq = new PipeTransferTabletRawReqV2();
+
+ tabletReq.tablet = tablet;
+ tabletReq.isAligned = isAligned;
+ tabletReq.dataBaseName = dataBaseName;
+
+ tabletReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ tabletReq.type = PipeRequestType.TRANSFER_TABLET_RAW_V2.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ tablet.serialize(outputStream);
+ ReadWriteIOUtils.write(isAligned, outputStream);
+ ReadWriteIOUtils.write(dataBaseName, outputStream);
+ tabletReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return tabletReq;
+ }
+
+ public static PipeTransferTabletRawReqV2 fromTPipeTransferReq(
+ final TPipeTransferReq transferReq) {
+ final PipeTransferTabletRawReqV2 tabletReq = new PipeTransferTabletRawReqV2();
+
+ tabletReq.tablet = Tablet.deserialize(transferReq.body);
+ tabletReq.isAligned = ReadWriteIOUtils.readBool(transferReq.body);
+ tabletReq.dataBaseName = ReadWriteIOUtils.readString(transferReq.body);
+
+ tabletReq.version = transferReq.version;
+ tabletReq.type = transferReq.type;
+ tabletReq.body = transferReq.body;
+
+ return tabletReq;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), outputStream);
+ tablet.serialize(outputStream);
+ ReadWriteIOUtils.write(isAligned, outputStream);
+ ReadWriteIOUtils.write(dataBaseName, outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ final PipeTransferTabletRawReqV2 that = (PipeTransferTabletRawReqV2) o;
+ return Objects.equals(dataBaseName, that.dataBaseName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), dataBaseName);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceReq.java
new file mode 100644
index 00000000..6cf4eac6
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceReq.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.IOException;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFilePieceReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+public class PipeTransferTsFilePieceReq extends PipeTransferFilePieceReq {
+
+ private PipeTransferTsFilePieceReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_TS_FILE_PIECE;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTsFilePieceReq toTPipeTransferReq(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return (PipeTransferTsFilePieceReq)
+ new PipeTransferTsFilePieceReq()
+ .convertToTPipeTransferReq(fileName, startWritingOffset, filePiece);
+ }
+
+ public static PipeTransferTsFilePieceReq fromTPipeTransferReq(TPipeTransferReq transferReq) {
+ return (PipeTransferTsFilePieceReq)
+ new PipeTransferTsFilePieceReq().translateFromTPipeTransferReq(transferReq);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return new PipeTransferTsFilePieceReq()
+ .convertToTPipeTransferBytes(fileName, startWritingOffset, filePiece);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeTransferTsFilePieceReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceWithModReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceWithModReq.java
new file mode 100644
index 00000000..16e87e91
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceWithModReq.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.IOException;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFilePieceReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+public class PipeTransferTsFilePieceWithModReq extends PipeTransferFilePieceReq {
+
+ private PipeTransferTsFilePieceWithModReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_TS_FILE_PIECE_WITH_MOD;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTsFilePieceWithModReq toTPipeTransferReq(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return (PipeTransferTsFilePieceWithModReq)
+ new PipeTransferTsFilePieceWithModReq()
+ .convertToTPipeTransferReq(fileName, startWritingOffset, filePiece);
+ }
+
+ public static PipeTransferTsFilePieceWithModReq fromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+ return (PipeTransferTsFilePieceWithModReq)
+ new PipeTransferTsFilePieceWithModReq().translateFromTPipeTransferReq(transferReq);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return new PipeTransferTsFilePieceWithModReq()
+ .convertToTPipeTransferBytes(fileName, startWritingOffset, filePiece);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeTransferTsFilePieceWithModReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealReq.java
new file mode 100644
index 00000000..a53039e9
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealReq.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.IOException;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFileSealReqV1;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+public class PipeTransferTsFileSealReq extends PipeTransferFileSealReqV1 {
+
+ private PipeTransferTsFileSealReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_TS_FILE_SEAL;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTsFileSealReq toTPipeTransferReq(String fileName, long fileLength)
+ throws IOException {
+ return (PipeTransferTsFileSealReq)
+ new PipeTransferTsFileSealReq().convertToTPipeTransferReq(fileName, fileLength);
+ }
+
+ public static PipeTransferTsFileSealReq fromTPipeTransferReq(TPipeTransferReq req) {
+ return (PipeTransferTsFileSealReq)
+ new PipeTransferTsFileSealReq().translateFromTPipeTransferReq(req);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(String fileName, long fileLength) throws IOException {
+ return new PipeTransferTsFileSealReq()
+ .convertToTPipeTransferSnapshotSealBytes(fileName, fileLength);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeTransferTsFileSealReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealWithModReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealWithModReq.java
new file mode 100644
index 00000000..3bda972a
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealWithModReq.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFileSealReqV2;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+public class PipeTransferTsFileSealWithModReq extends PipeTransferFileSealReqV2 {
+
+ private PipeTransferTsFileSealWithModReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_TS_FILE_SEAL_WITH_MOD;
+ }
+
+ protected static final String DATABASE_NAME_KEY_PREFIX = "DATABASE_NAME_";
+
+ public String getDatabaseNameByTsFileName() {
+ return parameters == null
+ ? null
+ : parameters.get(generateDatabaseNameWithFileNameKey(fileNames.get(fileNames.size() - 1)));
+ }
+
+ protected static String generateDatabaseNameWithFileNameKey(final String fileName) {
+ return DATABASE_NAME_KEY_PREFIX + fileName;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTsFileSealWithModReq toTPipeTransferReq(
+ final String modFileName,
+ final long modFileLength,
+ final String tsFileName,
+ final long tsFileLength)
+ throws IOException {
+ return toTPipeTransferReq(modFileName, modFileLength, tsFileName, tsFileLength, null);
+ }
+
+ public static PipeTransferTsFileSealWithModReq toTPipeTransferReq(
+ final String modFileName,
+ final long modFileLength,
+ final String tsFileName,
+ final long tsFileLength,
+ final String dataBaseName)
+ throws IOException {
+ return (PipeTransferTsFileSealWithModReq)
+ new PipeTransferTsFileSealWithModReq()
+ .convertToTPipeTransferReq(
+ Arrays.asList(modFileName, tsFileName),
+ Arrays.asList(modFileLength, tsFileLength),
+ Collections.singletonMap(
+ generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName));
+ }
+
+ public static PipeTransferTsFileSealWithModReq toTPipeTransferReq(
+ final String tsFileName, final long tsFileLength, final String dataBaseName)
+ throws IOException {
+ return (PipeTransferTsFileSealWithModReq)
+ new PipeTransferTsFileSealWithModReq()
+ .convertToTPipeTransferReq(
+ Collections.singletonList(tsFileName),
+ Collections.singletonList(tsFileLength),
+ Collections.singletonMap(
+ generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName));
+ }
+
+ public static PipeTransferTsFileSealWithModReq fromTPipeTransferReq(final TPipeTransferReq req) {
+ return (PipeTransferTsFileSealWithModReq)
+ new PipeTransferTsFileSealWithModReq().translateFromTPipeTransferReq(req);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ final String modFileName,
+ final long modFileLength,
+ final String tsFileName,
+ final long tsFileLength)
+ throws IOException {
+ return new PipeTransferTsFileSealWithModReq()
+ .convertToTPipeTransferSnapshotSealBytes(
+ Arrays.asList(modFileName, tsFileName),
+ Arrays.asList(modFileLength, tsFileLength),
+ new HashMap<>());
+ }
+
+ public static byte[] toTPipeTransferBytes(
+ final String modFileName,
+ final long modFileLength,
+ final String tsFileName,
+ final long tsFileLength,
+ final String dataBaseName)
+ throws IOException {
+ return new PipeTransferTsFileSealWithModReq()
+ .convertToTPipeTransferSnapshotSealBytes(
+ Arrays.asList(modFileName, tsFileName),
+ Arrays.asList(modFileLength, tsFileLength),
+ Collections.singletonMap(
+ generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName));
+ }
+
+ public static byte[] toTPipeTransferBytes(
+ final String tsFileName, final long tsFileLength, final String dataBaseName)
+ throws IOException {
+ return new PipeTransferTsFileSealWithModReq()
+ .convertToTPipeTransferSnapshotSealBytes(
+ Collections.singletonList(tsFileName),
+ Collections.singletonList(tsFileLength),
+ Collections.singletonMap(
+ generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName));
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeTransferTsFileSealWithModReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferHandshakeConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferHandshakeConstant.java
new file mode 100644
index 00000000..f2a7b3ab
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferHandshakeConstant.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.common;
+
+public class PipeTransferHandshakeConstant {
+
+ public static final String HANDSHAKE_KEY_TIME_PRECISION = "timestampPrecision";
+ public static final String HANDSHAKE_KEY_CLUSTER_ID = "clusterID";
+ public static final String HANDSHAKE_KEY_CONVERT_ON_TYPE_MISMATCH = "convertOnTypeMismatch";
+ public static final String HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY = "loadTsFileStrategy";
+ public static final String HANDSHAKE_KEY_USERNAME = "username";
+ public static final String HANDSHAKE_KEY_PASSWORD = "password";
+ public static final String HANDSHAKE_KEY_VALIDATE_TSFILE = "validateTsFile";
+ public static final String HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST = "markAsPipeRequest";
+
+ private PipeTransferHandshakeConstant() {
+ // Utility class
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferSliceReqHandler.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferSliceReqHandler.java
new file mode 100644
index 00000000..4491a99d
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferSliceReqHandler.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.common;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferSliceReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PipeTransferSliceReqHandler {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTransferSliceReqHandler.class);
+
+ private int orderId = -1;
+
+ private short originReqType = -1;
+ private int originBodySize = -1;
+
+ private int sliceCount = -1;
+ private final List sliceBodies = new ArrayList<>();
+
+ public boolean receiveSlice(final PipeTransferSliceReq req) {
+ if (orderId == -1
+ || originReqType == -1
+ || originBodySize == -1
+ || sliceCount == -1
+ || sliceBodies.isEmpty()) {
+ if (orderId == -1
+ && originReqType == -1
+ && originBodySize == -1
+ && sliceCount == -1
+ && sliceBodies.isEmpty()) {
+ orderId = req.getOrderId();
+ originReqType = req.getOriginReqType();
+ originBodySize = req.getOriginBodySize();
+ sliceCount = req.getSliceCount();
+ } else {
+ LOGGER.warn(
+ "Invalid state: orderId={}, originReqType={}, originBodySize={}, sliceCount={}, sliceBodies.size={}",
+ orderId,
+ originReqType,
+ originBodySize,
+ sliceCount,
+ sliceBodies.size());
+ clear();
+ return false;
+ }
+ }
+
+ if (orderId != req.getOrderId()) {
+ LOGGER.warn("Order ID mismatch: expected {}, actual {}", orderId, req.getOrderId());
+ clear();
+ return false;
+ }
+ if (originReqType != req.getOriginReqType()) {
+ LOGGER.warn(
+ "Origin request type mismatch: expected {}, actual {}",
+ originReqType,
+ req.getOriginReqType());
+ clear();
+ return false;
+ }
+ if (originBodySize != req.getOriginBodySize()) {
+ LOGGER.warn(
+ "Origin body size mismatch: expected {}, actual {}",
+ originBodySize,
+ req.getOriginBodySize());
+ clear();
+ return false;
+ }
+ if (sliceCount != req.getSliceCount()) {
+ LOGGER.warn("Slice count mismatch: expected {}, actual {}", sliceCount, req.getSliceCount());
+ clear();
+ return false;
+ }
+ if (sliceBodies.size() != req.getSliceIndex()) {
+ LOGGER.warn(
+ "Invalid slice index: expected {}, actual {}", sliceBodies.size(), req.getSliceIndex());
+ clear();
+ return false;
+ }
+
+ sliceBodies.add(req.getSliceBody());
+ return true;
+ }
+
+ public Optional makeReqIfComplete() {
+ if (sliceBodies.size() != sliceCount) {
+ return Optional.empty();
+ }
+
+ final TPipeTransferReq req = new TPipeTransferReq();
+ req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ req.type = originReqType;
+
+ final ByteBuffer body = ByteBuffer.allocate(originBodySize);
+ sliceBodies.forEach(body::put);
+ body.flip();
+ req.body = body;
+
+ return Optional.of(req);
+ }
+
+ public void clear() {
+ orderId = -1;
+ originReqType = -1;
+ originBodySize = -1;
+ sliceCount = -1;
+ sliceBodies.clear();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/IoTDBConnectorRequestVersion.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/IoTDBConnectorRequestVersion.java
new file mode 100644
index 00000000..54f10f23
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/IoTDBConnectorRequestVersion.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+public enum IoTDBConnectorRequestVersion {
+ VERSION_1((byte) 1),
+ VERSION_2((byte) 2),
+ ;
+
+ private final byte version;
+
+ IoTDBConnectorRequestVersion(byte type) {
+ this.version = type;
+ }
+
+ public byte getVersion() {
+ return version;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeRequestType.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeRequestType.java
new file mode 100644
index 00000000..aab7926a
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeRequestType.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public enum PipeRequestType {
+
+ // Handshake
+ HANDSHAKE_CONFIGNODE_V1((short) 0),
+ HANDSHAKE_DATANODE_V1((short) 1),
+ HANDSHAKE_CONFIGNODE_V2((short) 50),
+ HANDSHAKE_DATANODE_V2((short) 51),
+
+ // Data region
+ TRANSFER_TABLET_INSERT_NODE((short) 2),
+ TRANSFER_TABLET_RAW((short) 3),
+ TRANSFER_TS_FILE_PIECE((short) 4),
+ TRANSFER_TS_FILE_SEAL((short) 5),
+ TRANSFER_TABLET_BATCH((short) 6),
+ TRANSFER_TABLET_BINARY((short) 7),
+ TRANSFER_TS_FILE_PIECE_WITH_MOD((short) 8),
+ TRANSFER_TS_FILE_SEAL_WITH_MOD((short) 9),
+
+ TRANSFER_TABLET_INSERT_NODE_V2((short) 10),
+ TRANSFER_TABLET_RAW_V2((short) 11),
+ TRANSFER_TABLET_BINARY_V2((short) 12),
+ TRANSFER_TABLET_BATCH_V2((short) 13),
+
+ // Schema region / Delete Data
+ TRANSFER_PLAN_NODE((short) 100),
+ TRANSFER_SCHEMA_SNAPSHOT_PIECE((short) 101),
+ TRANSFER_SCHEMA_SNAPSHOT_SEAL((short) 102),
+
+ // Config region
+ TRANSFER_CONFIG_PLAN((short) 200),
+ TRANSFER_CONFIG_SNAPSHOT_PIECE((short) 201),
+ TRANSFER_CONFIG_SNAPSHOT_SEAL((short) 202),
+
+ // RPC Compression
+ TRANSFER_COMPRESSED((short) 300),
+
+ // Fallback Handling
+ TRANSFER_SLICE((short) 400),
+ ;
+
+ private final short type;
+
+ PipeRequestType(short type) {
+ this.type = type;
+ }
+
+ public short getType() {
+ return type;
+ }
+
+ private static final Map TYPE_MAP =
+ Arrays.stream(PipeRequestType.values())
+ .collect(
+ HashMap::new,
+ (typeMap, pipeRequestType) -> typeMap.put(pipeRequestType.getType(), pipeRequestType),
+ HashMap::putAll);
+
+ public static boolean isValidatedRequestType(short type) {
+ return TYPE_MAP.containsKey(type);
+ }
+
+ public static PipeRequestType valueOf(short type) {
+ return TYPE_MAP.get(type);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferCompressedReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferCompressedReq.java
new file mode 100644
index 00000000..975e2f7f
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferCompressedReq.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressor;
+import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressorFactory;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.BytesUtils;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public class PipeTransferCompressedReq extends TPipeTransferReq {
+
+ /** Generate a compressed req with provided compressors. */
+ public static TPipeTransferReq toTPipeTransferReq(
+ final TPipeTransferReq originalReq, final List compressors)
+ throws IOException {
+ // The generated PipeTransferCompressedReq consists of:
+ // version
+ // type: TRANSFER_COMPRESSED
+ // body:
+ // (byte) count of compressors (n)
+ // (n*3 bytes) for each compressor:
+ // (byte) compressor type
+ // (int) length of uncompressed bytes
+ // compressed req:
+ // (byte) version
+ // (2 bytes) type
+ // (bytes) body
+ final PipeTransferCompressedReq compressedReq = new PipeTransferCompressedReq();
+ compressedReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ compressedReq.type = PipeRequestType.TRANSFER_COMPRESSED.getType();
+
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ byte[] body =
+ BytesUtils.concatByteArrayList(
+ Arrays.asList(
+ new byte[] {originalReq.version},
+ BytesUtils.shortToBytes(originalReq.type),
+ originalReq.getBody()));
+
+ ReadWriteIOUtils.write((byte) compressors.size(), outputStream);
+ for (final PipeCompressor compressor : compressors) {
+ ReadWriteIOUtils.write(compressor.serialize(), outputStream);
+ ReadWriteIOUtils.write(body.length, outputStream);
+ body = compressor.compress(body);
+ }
+ outputStream.write(body);
+
+ compressedReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+ return compressedReq;
+ }
+
+ /** Get the original req from a compressed req. */
+ public static TPipeTransferReq fromTPipeTransferReq(final TPipeTransferReq transferReq)
+ throws IOException {
+ final ByteBuffer compressedBuffer = transferReq.body;
+
+ final List compressors = new ArrayList<>();
+ final List uncompressedLengths = new ArrayList<>();
+ final int compressorsSize = ReadWriteIOUtils.readByte(compressedBuffer);
+ for (int i = 0; i < compressorsSize; ++i) {
+ compressors.add(
+ PipeCompressorFactory.getCompressor(ReadWriteIOUtils.readByte(compressedBuffer)));
+ uncompressedLengths.add(ReadWriteIOUtils.readInt(compressedBuffer));
+ }
+
+ byte[] body = new byte[compressedBuffer.remaining()];
+ compressedBuffer.get(body);
+
+ for (int i = compressors.size() - 1; i >= 0; --i) {
+ body = compressors.get(i).decompress(body, uncompressedLengths.get(i));
+ }
+
+ final ByteBuffer decompressedBuffer = ByteBuffer.wrap(body);
+
+ final TPipeTransferReq decompressedReq = new TPipeTransferReq();
+ decompressedReq.version = ReadWriteIOUtils.readByte(decompressedBuffer);
+ decompressedReq.type = ReadWriteIOUtils.readShort(decompressedBuffer);
+ decompressedReq.body = decompressedBuffer.slice();
+
+ return decompressedReq;
+ }
+
+ /**
+ * For air-gap connectors. Generate the bytes of a compressed req from the bytes of original req.
+ */
+ public static byte[] toTPipeTransferReqBytes(
+ final byte[] rawReqInBytes, final List compressors) throws IOException {
+ // The generated bytes consists of:
+ // (byte) version
+ // (2 bytes) type: TRANSFER_COMPRESSED
+ // (byte) count of compressors (n)
+ // (n*3 bytes) for each compressor:
+ // (byte) compressor type
+ // (int) length of uncompressed bytes
+ // compressed req:
+ // (byte) version
+ // (2 bytes) type
+ // (bytes) body
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ byte[] body = rawReqInBytes;
+
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(PipeRequestType.TRANSFER_COMPRESSED.getType(), outputStream);
+ ReadWriteIOUtils.write((byte) compressors.size(), outputStream);
+ for (final PipeCompressor compressor : compressors) {
+ ReadWriteIOUtils.write(compressor.serialize(), outputStream);
+ ReadWriteIOUtils.write(body.length, outputStream);
+ body = compressor.compress(body);
+ }
+ outputStream.write(body);
+
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ private PipeTransferCompressedReq() {
+ // Empty constructor
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFilePieceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFilePieceReq.java
new file mode 100644
index 00000000..9b8cb161
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFilePieceReq.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Objects;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public abstract class PipeTransferFilePieceReq extends TPipeTransferReq {
+
+ private transient String fileName;
+ private transient long startWritingOffset;
+ private transient byte[] filePiece;
+
+ public final String getFileName() {
+ return fileName;
+ }
+
+ public final long getStartWritingOffset() {
+ return startWritingOffset;
+ }
+
+ public final byte[] getFilePiece() {
+ return filePiece;
+ }
+
+ protected abstract PipeRequestType getPlanType();
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ protected final PipeTransferFilePieceReq convertToTPipeTransferReq(
+ String snapshotName, long startWritingOffset, byte[] snapshotPiece) throws IOException {
+
+ this.fileName = snapshotName;
+ this.startWritingOffset = startWritingOffset;
+ this.filePiece = snapshotPiece;
+
+ this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ this.type = getPlanType().getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(snapshotName, outputStream);
+ ReadWriteIOUtils.write(startWritingOffset, outputStream);
+ ReadWriteIOUtils.write(new Binary(snapshotPiece), outputStream);
+ body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return this;
+ }
+
+ protected final PipeTransferFilePieceReq translateFromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+
+ fileName = ReadWriteIOUtils.readString(transferReq.body);
+ startWritingOffset = ReadWriteIOUtils.readLong(transferReq.body);
+ filePiece = ReadWriteIOUtils.readBinary(transferReq.body).getValues();
+
+ version = transferReq.version;
+ type = transferReq.type;
+ body = transferReq.body;
+
+ return this;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ protected final byte[] convertToTPipeTransferBytes(
+ String snapshotName, long startWritingOffset, byte[] snapshotPiece) throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(getPlanType().getType(), outputStream);
+ ReadWriteIOUtils.write(snapshotName, outputStream);
+ ReadWriteIOUtils.write(startWritingOffset, outputStream);
+ ReadWriteIOUtils.write(new Binary(snapshotPiece), outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferFilePieceReq that = (PipeTransferFilePieceReq) obj;
+ return fileName.equals(that.fileName)
+ && startWritingOffset == that.startWritingOffset
+ && Arrays.equals(filePiece, that.filePiece)
+ && version == that.version
+ && type == that.type
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ fileName, startWritingOffset, Arrays.hashCode(filePiece), version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV1.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV1.java
new file mode 100644
index 00000000..549c3868
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV1.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public abstract class PipeTransferFileSealReqV1 extends TPipeTransferReq {
+
+ private transient String fileName;
+ private transient long fileLength;
+
+ public final String getFileName() {
+ return fileName;
+ }
+
+ public final long getFileLength() {
+ return fileLength;
+ }
+
+ protected abstract PipeRequestType getPlanType();
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ protected PipeTransferFileSealReqV1 convertToTPipeTransferReq(String fileName, long fileLength)
+ throws IOException {
+
+ this.fileName = fileName;
+ this.fileLength = fileLength;
+
+ this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ this.type = getPlanType().getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(fileName, outputStream);
+ ReadWriteIOUtils.write(fileLength, outputStream);
+ this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return this;
+ }
+
+ public PipeTransferFileSealReqV1 translateFromTPipeTransferReq(TPipeTransferReq req) {
+
+ fileName = ReadWriteIOUtils.readString(req.body);
+ fileLength = ReadWriteIOUtils.readLong(req.body);
+
+ version = req.version;
+ type = req.type;
+ body = req.body;
+
+ return this;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public byte[] convertToTPipeTransferSnapshotSealBytes(String fileName, long fileLength)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(getPlanType().getType(), outputStream);
+ ReadWriteIOUtils.write(fileName, outputStream);
+ ReadWriteIOUtils.write(fileLength, outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferFileSealReqV1 that = (PipeTransferFileSealReqV1) obj;
+ return fileName.equals(that.fileName)
+ && fileLength == that.fileLength
+ && version == that.version
+ && type == that.type
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(fileName, fileLength, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV2.java
new file mode 100644
index 00000000..08c5d973
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV2.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public abstract class PipeTransferFileSealReqV2 extends TPipeTransferReq {
+
+ public static final String DATABASE_PATTERN = "database_pattern";
+ public static final String TREE = "tree";
+ public static final String TABLE = "table";
+ protected transient List fileNames;
+ protected transient List fileLengths;
+ protected transient Map parameters;
+
+ public final List getFileNames() {
+ return fileNames;
+ }
+
+ public final List getFileLengths() {
+ return fileLengths;
+ }
+
+ public final Map getParameters() {
+ return parameters;
+ }
+
+ protected abstract PipeRequestType getPlanType();
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ protected PipeTransferFileSealReqV2 convertToTPipeTransferReq(
+ final List fileNames,
+ final List fileLengths,
+ final Map parameters)
+ throws IOException {
+
+ this.fileNames = fileNames;
+ this.fileLengths = fileLengths;
+ this.parameters = parameters;
+
+ this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ this.type = getPlanType().getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(fileNames.size(), outputStream);
+ for (final String fileName : fileNames) {
+ ReadWriteIOUtils.write(fileName, outputStream);
+ }
+ ReadWriteIOUtils.write(fileLengths.size(), outputStream);
+ for (final Long fileLength : fileLengths) {
+ ReadWriteIOUtils.write(fileLength, outputStream);
+ }
+ ReadWriteIOUtils.write(parameters.size(), outputStream);
+ for (final Map.Entry entry : parameters.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+ this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return this;
+ }
+
+ public PipeTransferFileSealReqV2 translateFromTPipeTransferReq(final TPipeTransferReq req) {
+ fileNames = new ArrayList<>();
+ int size = ReadWriteIOUtils.readInt(req.body);
+ for (int i = 0; i < size; ++i) {
+ fileNames.add(ReadWriteIOUtils.readString(req.body));
+ }
+
+ fileLengths = new ArrayList<>();
+ size = ReadWriteIOUtils.readInt(req.body);
+ for (int i = 0; i < size; ++i) {
+ fileLengths.add(ReadWriteIOUtils.readLong(req.body));
+ }
+
+ parameters = new HashMap<>();
+ size = ReadWriteIOUtils.readInt(req.body);
+ for (int i = 0; i < size; ++i) {
+ final String key = ReadWriteIOUtils.readString(req.body);
+ final String value = ReadWriteIOUtils.readString(req.body);
+ parameters.put(key, value);
+ }
+
+ version = req.version;
+ type = req.type;
+ body = req.body;
+
+ return this;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public byte[] convertToTPipeTransferSnapshotSealBytes(
+ List fileNames, List fileLengths, Map parameters)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(getPlanType().getType(), outputStream);
+ ReadWriteIOUtils.write(fileNames.size(), outputStream);
+ for (String fileName : fileNames) {
+ ReadWriteIOUtils.write(fileName, outputStream);
+ }
+ ReadWriteIOUtils.write(fileLengths.size(), outputStream);
+ for (Long fileLength : fileLengths) {
+ ReadWriteIOUtils.write(fileLength, outputStream);
+ }
+ ReadWriteIOUtils.write(parameters.size(), outputStream);
+ for (final Map.Entry entry : parameters.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferFileSealReqV2 that = (PipeTransferFileSealReqV2) obj;
+ return Objects.equals(fileNames, that.fileNames)
+ && Objects.equals(fileLengths, that.fileLengths)
+ && Objects.equals(parameters, that.parameters)
+ && Objects.equals(version, that.version)
+ && Objects.equals(type, that.type)
+ && Objects.equals(body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(fileNames, fileLengths, parameters, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV1Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV1Req.java
new file mode 100644
index 00000000..5ed7eebb
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV1Req.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public abstract class PipeTransferHandshakeV1Req extends TPipeTransferReq {
+
+ private transient String timestampPrecision;
+
+ public final String getTimestampPrecision() {
+ return timestampPrecision;
+ }
+
+ protected abstract PipeRequestType getPlanType();
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public final PipeTransferHandshakeV1Req convertToTPipeTransferReq(String timestampPrecision)
+ throws IOException {
+ this.timestampPrecision = timestampPrecision;
+
+ this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ this.type = getPlanType().getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(timestampPrecision, outputStream);
+ this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return this;
+ }
+
+ protected final PipeTransferHandshakeV1Req translateFromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+ timestampPrecision = ReadWriteIOUtils.readString(transferReq.body);
+
+ version = transferReq.version;
+ type = transferReq.type;
+ body = transferReq.body;
+
+ return this;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ protected final byte[] convertToTransferHandshakeBytes(String timestampPrecision)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(getPlanType().getType(), outputStream);
+ ReadWriteIOUtils.write(timestampPrecision, outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferHandshakeV1Req that = (PipeTransferHandshakeV1Req) obj;
+ return timestampPrecision.equals(that.timestampPrecision)
+ && version == that.version
+ && type == that.type
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(timestampPrecision, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java
new file mode 100644
index 00000000..58fe726a
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public abstract class PipeTransferHandshakeV2Req extends TPipeTransferReq {
+
+ private transient Map params;
+
+ public Map getParams() {
+ return params;
+ }
+
+ protected abstract PipeRequestType getPlanType();
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map params)
+ throws IOException {
+ this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ this.type = getPlanType().getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(params.size(), outputStream);
+ for (final Map.Entry entry : params.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+ this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ this.params = params;
+ return this;
+ }
+
+ protected final PipeTransferHandshakeV2Req translateFromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+ Map params = new HashMap<>();
+ final int size = ReadWriteIOUtils.readInt(transferReq.body);
+ for (int i = 0; i < size; ++i) {
+ final String key = ReadWriteIOUtils.readString(transferReq.body);
+ final String value = ReadWriteIOUtils.readString(transferReq.body);
+ params.put(key, value);
+ }
+ this.params = params;
+
+ version = transferReq.version;
+ type = transferReq.type;
+ body = transferReq.body;
+
+ return this;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public final byte[] convertToTransferHandshakeBytes(Map params)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(getPlanType().getType(), outputStream);
+ ReadWriteIOUtils.write(params.size(), outputStream);
+ for (final Map.Entry entry : params.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferHandshakeV2Req that = (PipeTransferHandshakeV2Req) obj;
+ return Objects.equals(params, that.params)
+ && version == that.version
+ && type == that.type
+ && Objects.equals(body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(params, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSliceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSliceReq.java
new file mode 100644
index 00000000..4541b84a
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSliceReq.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Objects;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public class PipeTransferSliceReq extends TPipeTransferReq {
+
+ private transient int orderId;
+
+ private transient short originReqType;
+ private transient int originBodySize;
+
+ private transient byte[] sliceBody;
+
+ private transient int sliceIndex;
+ private transient int sliceCount;
+
+ public int getOrderId() {
+ return orderId;
+ }
+
+ public short getOriginReqType() {
+ return originReqType;
+ }
+
+ public int getOriginBodySize() {
+ return originBodySize;
+ }
+
+ public byte[] getSliceBody() {
+ return sliceBody;
+ }
+
+ public int getSliceIndex() {
+ return sliceIndex;
+ }
+
+ public int getSliceCount() {
+ return sliceCount;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferSliceReq toTPipeTransferReq(
+ final int orderId,
+ final short originReqType,
+ final int sliceIndex,
+ final int sliceCount,
+ final ByteBuffer duplicatedOriginBody,
+ final int startIndexInBody,
+ final int endIndexInBody)
+ throws IOException {
+ final PipeTransferSliceReq sliceReq = new PipeTransferSliceReq();
+
+ sliceReq.orderId = orderId;
+
+ sliceReq.originReqType = originReqType;
+ sliceReq.originBodySize = duplicatedOriginBody.limit();
+
+ sliceReq.sliceBody = new byte[endIndexInBody - startIndexInBody];
+ duplicatedOriginBody.position(startIndexInBody);
+ duplicatedOriginBody.get(sliceReq.sliceBody);
+
+ sliceReq.sliceIndex = sliceIndex;
+ sliceReq.sliceCount = sliceCount;
+
+ sliceReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ sliceReq.type = PipeRequestType.TRANSFER_SLICE.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(sliceReq.orderId, outputStream);
+
+ ReadWriteIOUtils.write(sliceReq.originReqType, outputStream);
+ ReadWriteIOUtils.write(sliceReq.originBodySize, outputStream);
+
+ ReadWriteIOUtils.write(new Binary(sliceReq.sliceBody), outputStream);
+
+ ReadWriteIOUtils.write(sliceReq.sliceIndex, outputStream);
+ ReadWriteIOUtils.write(sliceReq.sliceCount, outputStream);
+
+ sliceReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return sliceReq;
+ }
+
+ public static PipeTransferSliceReq fromTPipeTransferReq(final TPipeTransferReq transferReq) {
+ final PipeTransferSliceReq sliceReq = new PipeTransferSliceReq();
+
+ sliceReq.orderId = ReadWriteIOUtils.readInt(transferReq.body);
+
+ sliceReq.originReqType = ReadWriteIOUtils.readShort(transferReq.body);
+ sliceReq.originBodySize = ReadWriteIOUtils.readInt(transferReq.body);
+
+ sliceReq.sliceBody = ReadWriteIOUtils.readBinary(transferReq.body).getValues();
+
+ sliceReq.sliceIndex = ReadWriteIOUtils.readInt(transferReq.body);
+ sliceReq.sliceCount = ReadWriteIOUtils.readInt(transferReq.body);
+
+ sliceReq.version = transferReq.version;
+ sliceReq.type = transferReq.type;
+ sliceReq.body = transferReq.body;
+
+ return sliceReq;
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeTransferSliceReq that = (PipeTransferSliceReq) obj;
+ return Objects.equals(orderId, that.orderId)
+ && Objects.equals(originReqType, that.originReqType)
+ && Objects.equals(originBodySize, that.originBodySize)
+ && Arrays.equals(sliceBody, that.sliceBody)
+ && Objects.equals(sliceIndex, that.sliceIndex)
+ && Objects.equals(sliceCount, that.sliceCount)
+ && Objects.equals(version, that.version)
+ && Objects.equals(type, that.type)
+ && Objects.equals(body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ orderId,
+ originReqType,
+ originBodySize,
+ Arrays.hashCode(sliceBody),
+ sliceIndex,
+ sliceCount,
+ version,
+ type,
+ body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/response/PipeTransferFilePieceResp.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/response/PipeTransferFilePieceResp.java
new file mode 100644
index 00000000..180ea293
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/response/PipeTransferFilePieceResp.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.response;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+public class PipeTransferFilePieceResp extends TPipeTransferResp {
+
+ public static final long ERROR_END_OFFSET = -1;
+
+ private long endWritingOffset;
+
+ private PipeTransferFilePieceResp() {
+ // Empty constructor
+ }
+
+ public long getEndWritingOffset() {
+ return endWritingOffset;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferFilePieceResp toTPipeTransferResp(
+ TSStatus status, long endWritingOffset) throws IOException {
+ final PipeTransferFilePieceResp filePieceResp = new PipeTransferFilePieceResp();
+
+ filePieceResp.status = status;
+
+ filePieceResp.endWritingOffset = endWritingOffset;
+ try (PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(endWritingOffset, outputStream);
+ filePieceResp.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return filePieceResp;
+ }
+
+ public static PipeTransferFilePieceResp toTPipeTransferResp(TSStatus status) {
+ final PipeTransferFilePieceResp filePieceResp = new PipeTransferFilePieceResp();
+
+ filePieceResp.status = status;
+
+ return filePieceResp;
+ }
+
+ public static PipeTransferFilePieceResp fromTPipeTransferResp(TPipeTransferResp transferResp) {
+ final PipeTransferFilePieceResp filePieceResp = new PipeTransferFilePieceResp();
+
+ filePieceResp.status = transferResp.status;
+
+ if (transferResp.isSetBody()) {
+ filePieceResp.endWritingOffset = ReadWriteIOUtils.readLong(transferResp.body);
+ filePieceResp.body = transferResp.body;
+ }
+
+ return filePieceResp;
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferFilePieceResp that = (PipeTransferFilePieceResp) obj;
+ return endWritingOffset == that.endWritingOffset
+ && status.equals(that.status)
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(endWritingOffset, status, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/ClientPoolProperty.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/ClientPoolProperty.java
new file mode 100644
index 00000000..5026093c
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/ClientPoolProperty.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.property;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
+
+public class ClientPoolProperty {
+
+ private final GenericKeyedObjectPoolConfig config;
+
+ private ClientPoolProperty(GenericKeyedObjectPoolConfig config) {
+ this.config = config;
+ }
+
+ public GenericKeyedObjectPoolConfig getConfig() {
+ return config;
+ }
+
+ public static class Builder {
+
+ /**
+ * when the number of the client to a single node exceeds maxClientNumForEachNode, the thread
+ * for applying for a client will be blocked for waitClientTimeoutMs, then ClientManager will
+ * throw ClientManagerException if there are no clients after the block time.
+ */
+ private long waitClientTimeoutMs = DefaultProperty.WAIT_CLIENT_TIMEOUT_MS;
+
+ /**
+ * the maximum number of clients that can be allocated for a node. When some clients are idle
+ * for more than {@code maxIdleTimeForClient}, they will be cleaned up.
+ */
+ private int maxClientNumForEachNode = DefaultProperty.MAX_CLIENT_NUM_FOR_EACH_NODE;
+
+ /**
+ * the minimum amount of time a client may sit idle in the pool before it is eligible for
+ * eviction by the idle object evictor.
+ */
+ private long minIdleTimeForClient = DefaultProperty.MIN_IDLE_TIME_FOR_CLIENT_MS;
+
+ /**
+ * the duration to sleep between runs of the idle object evictor thread. When non-positive, no
+ * idle object evictor thread will be run, which means clients that are idle for more than
+ * {@code minIdleTimeForClient} will never be cleaned up.
+ */
+ private long timeBetweenEvictionRuns = DefaultProperty.TIME_BETWEEN_EVICTION_RUNS_MS;
+
+ public Builder setWaitClientTimeoutMs(long waitClientTimeoutMs) {
+ this.waitClientTimeoutMs = waitClientTimeoutMs;
+ return this;
+ }
+
+ public Builder setMaxClientNumForEachNode(int maxClientNumForEachNode) {
+ this.maxClientNumForEachNode = maxClientNumForEachNode;
+ return this;
+ }
+
+ public Builder setMinIdleTimeForClient(long minIdleTimeForClient) {
+ this.minIdleTimeForClient = minIdleTimeForClient;
+ return this;
+ }
+
+ public Builder setTimeBetweenEvictionRuns(long timeBetweenEvictionRuns) {
+ this.timeBetweenEvictionRuns = timeBetweenEvictionRuns;
+ return this;
+ }
+
+ public ClientPoolProperty build() {
+ GenericKeyedObjectPoolConfig poolConfig = new GenericKeyedObjectPoolConfig<>();
+ poolConfig.setMaxTotalPerKey(maxClientNumForEachNode);
+ poolConfig.setMaxIdlePerKey(maxClientNumForEachNode);
+ poolConfig.setTimeBetweenEvictionRuns(Duration.ofMillis(timeBetweenEvictionRuns));
+ poolConfig.setMinEvictableIdleTime(Duration.ofMillis(minIdleTimeForClient));
+ poolConfig.setMaxWait(Duration.ofMillis(waitClientTimeoutMs));
+ poolConfig.setTestOnReturn(true);
+ poolConfig.setTestOnBorrow(true);
+ return new ClientPoolProperty<>(poolConfig);
+ }
+ }
+
+ public static class DefaultProperty {
+
+ private DefaultProperty() {}
+
+ public static final long WAIT_CLIENT_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(30);
+ public static final long MIN_IDLE_TIME_FOR_CLIENT_MS = TimeUnit.MINUTES.toMillis(1);
+ public static final long TIME_BETWEEN_EVICTION_RUNS_MS = TimeUnit.MINUTES.toMillis(1);
+ public static final int MAX_CLIENT_NUM_FOR_EACH_NODE = 1000;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/PipeConsensusClientProperty.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/PipeConsensusClientProperty.java
new file mode 100644
index 00000000..219e3477
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/PipeConsensusClientProperty.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.property;
+
+/** This class defines the configurations used by the PipeConsensus Client. */
+public class PipeConsensusClientProperty {
+ private final boolean isRpcThriftCompressionEnabled;
+ private final int selectorNumOfClientManager;
+ private final boolean printLogWhenThriftClientEncounterException;
+ private final int maxClientNumForEachNode;
+
+ public PipeConsensusClientProperty(
+ boolean isRpcThriftCompressionEnabled,
+ int selectorNumOfClientManager,
+ boolean printLogWhenThriftClientEncounterException,
+ int maxClientNumForEachNode) {
+ this.isRpcThriftCompressionEnabled = isRpcThriftCompressionEnabled;
+ this.selectorNumOfClientManager = selectorNumOfClientManager;
+ this.printLogWhenThriftClientEncounterException = printLogWhenThriftClientEncounterException;
+ this.maxClientNumForEachNode = maxClientNumForEachNode;
+ }
+
+ public boolean isRpcThriftCompressionEnabled() {
+ return isRpcThriftCompressionEnabled;
+ }
+
+ public int getSelectorNumOfClientManager() {
+ return selectorNumOfClientManager;
+ }
+
+ public boolean isPrintLogWhenThriftClientEncounterException() {
+ return printLogWhenThriftClientEncounterException;
+ }
+
+ public int getMaxClientNumForEachNode() {
+ return maxClientNumForEachNode;
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private boolean isRpcThriftCompressionEnabled = false;
+ private int selectorNumOfClientManager = 1;
+ private boolean printLogWhenThriftClientEncounterException = true;
+ private int maxClientNumForEachNode =
+ ClientPoolProperty.DefaultProperty.MAX_CLIENT_NUM_FOR_EACH_NODE;
+
+ public Builder setIsRpcThriftCompressionEnabled(
+ boolean isRpcThriftCompressionEnabled) {
+ this.isRpcThriftCompressionEnabled = isRpcThriftCompressionEnabled;
+ return this;
+ }
+
+ public Builder setSelectorNumOfClientManager(
+ int selectorNumOfClientManager) {
+ this.selectorNumOfClientManager = selectorNumOfClientManager;
+ return this;
+ }
+
+ public Builder setPrintLogWhenThriftClientEncounterException(
+ boolean printLogWhenThriftClientEncounterException) {
+ this.printLogWhenThriftClientEncounterException = printLogWhenThriftClientEncounterException;
+ return this;
+ }
+
+ public Builder setMaxClientNumForEachNode(
+ int maxClientNumForEachNode) {
+ this.maxClientNumForEachNode = maxClientNumForEachNode;
+ return this;
+ }
+
+ public PipeConsensusClientProperty build() {
+ return new PipeConsensusClientProperty(
+ isRpcThriftCompressionEnabled,
+ selectorNumOfClientManager,
+ printLogWhenThriftClientEncounterException,
+ maxClientNumForEachNode);
+ }
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/ThriftClient.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/ThriftClient.java
new file mode 100644
index 00000000..4ed09dfe
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/ThriftClient.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.property;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.util.Optional;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class defines the failed interfaces that thrift client needs to support so that the Thrift
+ * Client can clean up the clientManager when it receives the corresponding exception.
+ */
+public interface ThriftClient {
+
+ Logger logger = LoggerFactory.getLogger(ThriftClient.class);
+
+ /** Close this connection. */
+ void invalidate();
+
+ /** Removing all pooled instances corresponding to current instance's endpoint. */
+ void invalidateAll();
+
+ /**
+ * Whether to print logs when exceptions are encountered.
+ *
+ * @return result
+ */
+ boolean printLogWhenEncounterException();
+
+ /**
+ * Perform corresponding operations on ThriftClient o based on the Throwable t.
+ *
+ * @param t Throwable
+ * @param o ThriftClient
+ */
+ static void resolveException(Throwable t, ThriftClient o) {
+ Throwable origin = t;
+ if (t instanceof InvocationTargetException) {
+ origin = ((InvocationTargetException) t).getTargetException();
+ }
+ Throwable cur = origin;
+ if (cur instanceof TException) {
+ int level = 0;
+ while (cur != null) {
+ logger.debug(
+ "level-{} Exception class {}, message {}",
+ level,
+ cur.getClass().getName(),
+ cur.getMessage());
+ cur = cur.getCause();
+ level++;
+ }
+ o.invalidate();
+ }
+
+ Throwable rootCause = ExceptionUtils.getRootCause(origin);
+ if (rootCause != null) {
+ // if the exception is SocketException and its error message is Broken pipe, it means that
+ // the remote node may restart and all the connection we cached before should be cleared.
+ logger.debug(
+ "root cause message {}, LocalizedMessage {}, ",
+ rootCause.getMessage(),
+ rootCause.getLocalizedMessage(),
+ rootCause);
+ if (isConnectionBroken(rootCause)) {
+ if (o.printLogWhenEncounterException()) {
+ logger.info(
+ "Broken pipe error happened in sending RPC,"
+ + " we need to clear all previous cached connection, error msg is {}",
+ rootCause.toString());
+ }
+ o.invalidateAll();
+ }
+ }
+ }
+
+ /**
+ * Determine whether the target node has gone offline once based on the cause.
+ *
+ * @param cause Throwable
+ * @return true/false
+ */
+ static boolean isConnectionBroken(Throwable cause) {
+ return (cause instanceof SocketException && cause.getMessage().contains("Broken pipe"))
+ || (cause instanceof TTransportException
+ && (hasExpectedMessage(cause, "Socket is closed by peer")
+ || hasExpectedMessage(cause, "Read call frame size failed")))
+ || (cause instanceof IOException
+ && (hasExpectedMessage(cause, "Connection reset by peer")
+ || hasExpectedMessage(cause, "Broken pipe")))
+ || (cause instanceof ConnectException && hasExpectedMessage(cause, "Connection refused"));
+ }
+
+ static boolean hasExpectedMessage(Throwable cause, String expectedMessage) {
+ return Optional.ofNullable(cause.getMessage())
+ .map(m -> m.contains(expectedMessage))
+ .orElse(false);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/ThriftClientProperty.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/ThriftClientProperty.java
new file mode 100644
index 00000000..1d4e7777
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/property/ThriftClientProperty.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.property;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.protocol.TProtocolFactory;
+
+/** This class defines the configurations commonly used by the Thrift Client. */
+public class ThriftClientProperty {
+
+ private final TProtocolFactory protocolFactory;
+ private final int connectionTimeoutMs;
+ private final int selectorNumOfAsyncClientPool;
+ private final boolean printLogWhenEncounterException;
+
+ private ThriftClientProperty(
+ TProtocolFactory protocolFactory,
+ int connectionTimeoutMs,
+ int selectorNumOfAsyncClientPool,
+ boolean printLogWhenEncounterException) {
+ this.protocolFactory = protocolFactory;
+ this.connectionTimeoutMs = connectionTimeoutMs;
+ this.selectorNumOfAsyncClientPool = selectorNumOfAsyncClientPool;
+ this.printLogWhenEncounterException = printLogWhenEncounterException;
+ }
+
+ public TProtocolFactory getProtocolFactory() {
+ return protocolFactory;
+ }
+
+ public int getConnectionTimeoutMs() {
+ return connectionTimeoutMs;
+ }
+
+ public int getSelectorNumOfAsyncClientPool() {
+ return selectorNumOfAsyncClientPool;
+ }
+
+ public boolean isPrintLogWhenEncounterException() {
+ return printLogWhenEncounterException;
+ }
+
+ public static class Builder {
+
+ /** whether to use thrift compression. */
+ private boolean rpcThriftCompressionEnabled = DefaultProperty.RPC_THRIFT_COMPRESSED_ENABLED;
+
+ /** socket timeout for thrift client. */
+ private int connectionTimeoutMs = DefaultProperty.CONNECTION_TIMEOUT_MS;
+
+ /** number of selector threads for asynchronous thrift client in a clientManager. */
+ private int selectorNumOfAsyncClientManager =
+ DefaultProperty.SELECTOR_NUM_OF_ASYNC_CLIENT_MANAGER;
+
+ /**
+ * Whether to print logs when the client encounters exceptions. For example, logs are not
+ * printed in the heartbeat client.
+ */
+ private boolean printLogWhenEncounterException =
+ DefaultProperty.PRINT_LOG_WHEN_ENCOUNTER_EXCEPTION;
+
+ public Builder setRpcThriftCompressionEnabled(boolean rpcThriftCompressionEnabled) {
+ this.rpcThriftCompressionEnabled = rpcThriftCompressionEnabled;
+ return this;
+ }
+
+ public Builder setConnectionTimeoutMs(int connectionTimeoutMs) {
+ this.connectionTimeoutMs = connectionTimeoutMs;
+ return this;
+ }
+
+ public Builder setSelectorNumOfAsyncClientManager(int selectorNumOfAsyncClientManager) {
+ this.selectorNumOfAsyncClientManager = selectorNumOfAsyncClientManager;
+ return this;
+ }
+
+ public Builder setPrintLogWhenEncounterException(boolean printLogWhenEncounterException) {
+ this.printLogWhenEncounterException = printLogWhenEncounterException;
+ return this;
+ }
+
+ public ThriftClientProperty build() {
+ return new ThriftClientProperty(
+ rpcThriftCompressionEnabled
+ ? new TCompactProtocol.Factory()
+ : new TBinaryProtocol.Factory(),
+ connectionTimeoutMs,
+ selectorNumOfAsyncClientManager,
+ printLogWhenEncounterException);
+ }
+ }
+
+ public static class DefaultProperty {
+
+ private DefaultProperty() {}
+
+ public static final boolean RPC_THRIFT_COMPRESSED_ENABLED = false;
+ public static final int CONNECTION_TIMEOUT_MS = (int) TimeUnit.SECONDS.toMillis(20);
+ public static final int CONNECTION_NEVER_TIMEOUT_MS = 0;
+ public static final int SELECTOR_NUM_OF_ASYNC_CLIENT_MANAGER = 1;
+ public static final boolean PRINT_LOG_WHEN_ENCOUNTER_EXCEPTION = true;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/airgap/IoTDBAirGapConnector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/airgap/IoTDBAirGapConnector.java
new file mode 100644
index 00000000..d50ffed8
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/airgap/IoTDBAirGapConnector.java
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.protocol.airgap;
+
+import org.apache.iotdb.collector.config.PipeOptions;
+import org.apache.iotdb.collector.plugin.builtin.annotation.TableModel;
+import org.apache.iotdb.collector.plugin.builtin.annotation.TreeModel;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.airgap.AirGapELanguageConstant;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.airgap.AirGapOneByteResponse;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.exception.PipeConnectionException;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.tsfile.utils.BytesUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.CRC32;
+
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_AIR_GAP_E_LANGUAGE_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_AIR_GAP_E_LANGUAGE_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_AIR_GAP_HANDSHAKE_TIMEOUT_MS_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_AIR_GAP_HANDSHAKE_TIMEOUT_MS_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_AIR_GAP_E_LANGUAGE_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_AIR_GAP_HANDSHAKE_TIMEOUT_MS_KEY;
+import static org.apache.tsfile.utils.ReadWriteIOUtils.LONG_LEN;
+
+@TreeModel
+@TableModel
+public abstract class IoTDBAirGapConnector extends IoTDBConnector {
+
+ protected static class AirGapSocket extends Socket {
+
+ private final TEndPoint endPoint;
+
+ public AirGapSocket(final String ip, final int port) {
+ this.endPoint = new TEndPoint(ip, port);
+ }
+
+ public TEndPoint getEndPoint() {
+ return endPoint;
+ }
+
+ @Override
+ public String toString() {
+ return "AirGapSocket{" + "endPoint=" + endPoint + "} (" + super.toString() + ")";
+ }
+ }
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBAirGapConnector.class);
+
+ protected final List sockets = new ArrayList<>();
+ protected final List isSocketAlive = new ArrayList<>();
+
+ private LoadBalancer loadBalancer;
+ private long currentClientIndex = 0;
+
+ private int handshakeTimeoutMs;
+
+ private boolean eLanguageEnable;
+
+ // The air gap connector does not use clientManager thus we put handshake type here
+ protected boolean supportModsIfIsDataNodeReceiver = true;
+
+ private final Map failLogTimes = new HashMap<>();
+
+ @Override
+ public void customize(
+ final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration)
+ throws Exception {
+ super.customize(parameters, configuration);
+
+ if (isTabletBatchModeEnabled) {
+ LOGGER.warn(
+ "Batch mode is enabled by the given parameters. "
+ + "IoTDBAirGapConnector does not support batch mode. "
+ + "Disable batch mode.");
+ }
+
+ for (int i = 0; i < nodeUrls.size(); i++) {
+ isSocketAlive.add(false);
+ sockets.add(null);
+ }
+
+ switch (loadBalanceStrategy) {
+ case CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY:
+ loadBalancer = new RoundRobinLoadBalancer();
+ break;
+ case CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY:
+ loadBalancer = new RandomLoadBalancer();
+ break;
+ case CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY:
+ loadBalancer = new PriorityLoadBalancer();
+ break;
+ default:
+ LOGGER.warn(
+ "Unknown load balance strategy: {}, use round-robin strategy instead.",
+ loadBalanceStrategy);
+ loadBalancer = new RoundRobinLoadBalancer();
+ }
+
+ handshakeTimeoutMs =
+ parameters.getIntOrDefault(
+ Arrays.asList(
+ CONNECTOR_AIR_GAP_HANDSHAKE_TIMEOUT_MS_KEY, SINK_AIR_GAP_HANDSHAKE_TIMEOUT_MS_KEY),
+ CONNECTOR_AIR_GAP_HANDSHAKE_TIMEOUT_MS_DEFAULT_VALUE);
+ LOGGER.info(
+ "IoTDBAirGapConnector is customized with handshakeTimeoutMs: {}.", handshakeTimeoutMs);
+
+ eLanguageEnable =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(
+ CONNECTOR_AIR_GAP_E_LANGUAGE_ENABLE_KEY, SINK_AIR_GAP_E_LANGUAGE_ENABLE_KEY),
+ CONNECTOR_AIR_GAP_E_LANGUAGE_ENABLE_DEFAULT_VALUE);
+ LOGGER.info("IoTDBAirGapConnector is customized with eLanguageEnable: {}.", eLanguageEnable);
+ }
+
+ @Override
+ @SuppressWarnings("java:S2095")
+ public void handshake() throws Exception {
+ for (int i = 0; i < sockets.size(); i++) {
+ if (Boolean.TRUE.equals(isSocketAlive.get(i))) {
+ continue;
+ }
+
+ final String ip = nodeUrls.get(i).getIp();
+ final int port = nodeUrls.get(i).getPort();
+
+ // Close the socket if necessary
+ if (sockets.get(i) != null) {
+ try {
+ sockets.set(i, null).close();
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to close socket with target server ip: {}, port: {}, because: {}. Ignore it.",
+ ip,
+ port,
+ e.getMessage());
+ }
+ }
+
+ final AirGapSocket socket = new AirGapSocket(ip, port);
+
+ try {
+ socket.connect(new InetSocketAddress(ip, port), handshakeTimeoutMs);
+ socket.setKeepAlive(true);
+ sockets.set(i, socket);
+ LOGGER.info("Successfully connected to target server ip: {}, port: {}.", ip, port);
+ failLogTimes.remove(nodeUrls.get(i));
+ } catch (final Exception e) {
+ final TEndPoint endPoint = nodeUrls.get(i);
+ final long currentTimeMillis = System.currentTimeMillis();
+ final Long lastFailLogTime = failLogTimes.get(endPoint);
+ if (lastFailLogTime == null || currentTimeMillis - lastFailLogTime > 60000) {
+ failLogTimes.put(endPoint, currentTimeMillis);
+ LOGGER.warn(
+ "Failed to connect to target server ip: {}, port: {}, because: {}. Ignore it.",
+ ip,
+ port,
+ e.getMessage());
+ }
+ continue;
+ }
+
+ try {
+ sendHandshakeReq(socket);
+ isSocketAlive.set(i, true);
+ } catch (Exception e) {
+ LOGGER.warn(
+ "Handshake error occurs. It may be caused by an error on the receiving end. Ignore it.",
+ e);
+ }
+ }
+
+ for (int i = 0; i < sockets.size(); i++) {
+ if (Boolean.TRUE.equals(isSocketAlive.get(i))) {
+ return;
+ }
+ }
+ throw new PipeConnectionException(
+ String.format("All target servers %s are not available.", nodeUrls));
+ }
+
+ protected void sendHandshakeReq(final AirGapSocket socket) throws IOException {
+ socket.setSoTimeout(handshakeTimeoutMs);
+ // Try to handshake by PipeTransferHandshakeV2Req. If failed, retry to handshake by
+ // PipeTransferHandshakeV1Req. If failed again, throw PipeConnectionException.
+ if (!send(socket, generateHandShakeV2Payload())) {
+ supportModsIfIsDataNodeReceiver = false;
+ if (!send(socket, generateHandShakeV1Payload())) {
+ throw new PipeConnectionException("Handshake error with target server, socket: " + socket);
+ }
+ } else {
+ supportModsIfIsDataNodeReceiver = true;
+ }
+ socket.setSoTimeout(PipeOptions.PIPE_CONNECTOR_TRANSFER_TIMEOUT_MS.value());
+ LOGGER.info("Handshake success. Socket: {}", socket);
+ }
+
+ protected abstract byte[] generateHandShakeV1Payload() throws IOException;
+
+ protected abstract byte[] generateHandShakeV2Payload() throws IOException;
+
+ @Override
+ public void heartbeat() {
+ try {
+ handshake();
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to reconnect to target server, because: {}. Try to reconnect later.",
+ e.getMessage(),
+ e);
+ }
+ }
+
+ protected void transferFilePieces(
+ final String pipeName,
+ final long creationTime,
+ final File file,
+ final AirGapSocket socket,
+ final boolean isMultiFile)
+ throws PipeException, IOException {
+ final int readFileBufferSize = PipeOptions.PIPE_CONNECTOR_READ_FILE_BUFFER_SIZE.value();
+ final byte[] readBuffer = new byte[readFileBufferSize];
+ long position = 0;
+ try (final RandomAccessFile reader = new RandomAccessFile(file, "r")) {
+ while (true) {
+ final int readLength = reader.read(readBuffer);
+ if (readLength == -1) {
+ break;
+ }
+
+ final byte[] payload =
+ readLength == readFileBufferSize
+ ? readBuffer
+ : Arrays.copyOfRange(readBuffer, 0, readLength);
+ if (!send(
+ pipeName,
+ creationTime,
+ socket,
+ isMultiFile
+ ? getTransferMultiFilePieceBytes(file.getName(), position, payload)
+ : getTransferSingleFilePieceBytes(file.getName(), position, payload))) {
+ final String errorMessage =
+ String.format("Transfer file %s error. Socket %s.", file, socket);
+ if (mayNeedHandshakeWhenFail()) {
+ // Send handshake because we don't know whether the receiver side configNode
+ // has set up a new one
+ sendHandshakeReq(socket);
+ }
+ receiverStatusHandler.handle(
+ new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode())
+ .setMessage(errorMessage),
+ errorMessage,
+ file.toString());
+ } else {
+ position += readLength;
+ }
+ }
+ }
+ }
+
+ protected abstract boolean mayNeedHandshakeWhenFail();
+
+ protected abstract byte[] getTransferSingleFilePieceBytes(
+ final String fileName, final long position, final byte[] payLoad) throws IOException;
+
+ protected abstract byte[] getTransferMultiFilePieceBytes(
+ final String fileName, final long position, final byte[] payLoad) throws IOException;
+
+ protected int nextSocketIndex() {
+ return loadBalancer.nextSocketIndex();
+ }
+
+ protected boolean send(
+ final String pipeName, final long creationTime, final AirGapSocket socket, byte[] bytes)
+ throws IOException {
+ if (!socket.isConnected()) {
+ throw new SocketException(
+ String.format("Socket %s is closed, will try to handshake", socket));
+ }
+
+ bytes = compressIfNeeded(bytes);
+
+ rateLimitIfNeeded(pipeName, creationTime, socket.getEndPoint(), bytes.length);
+
+ final BufferedOutputStream outputStream = new BufferedOutputStream(socket.getOutputStream());
+ bytes = enrichWithLengthAndChecksum(bytes);
+ outputStream.write(eLanguageEnable ? enrichWithELanguage(bytes) : bytes);
+ outputStream.flush();
+
+ final byte[] response = new byte[1];
+ final int size = socket.getInputStream().read(response);
+ return size > 0 && Arrays.equals(AirGapOneByteResponse.OK, response);
+ }
+
+ protected boolean send(final AirGapSocket socket, final byte[] bytes) throws IOException {
+ return send(null, 0, socket, bytes);
+ }
+
+ private byte[] enrichWithLengthAndChecksum(final byte[] bytes) {
+ // Length of checksum and bytes payload
+ final byte[] length = BytesUtils.intToBytes(bytes.length + LONG_LEN);
+
+ final CRC32 crc32 = new CRC32();
+ crc32.update(bytes, 0, bytes.length);
+
+ // Double length as simple checksum
+ return BytesUtils.concatByteArrayList(
+ Arrays.asList(length, length, BytesUtils.longToBytes(crc32.getValue()), bytes));
+ }
+
+ private byte[] enrichWithELanguage(final byte[] bytes) {
+ return BytesUtils.concatByteArrayList(
+ Arrays.asList(
+ AirGapELanguageConstant.E_LANGUAGE_PREFIX,
+ bytes,
+ AirGapELanguageConstant.E_LANGUAGE_SUFFIX));
+ }
+
+ @Override
+ public void close() {
+ for (int i = 0; i < sockets.size(); ++i) {
+ try {
+ if (sockets.get(i) != null) {
+ sockets.set(i, null).close();
+ }
+ } catch (final Exception e) {
+ LOGGER.warn("Failed to close client {}.", i, e);
+ } finally {
+ isSocketAlive.set(i, false);
+ }
+ }
+
+ super.close();
+ }
+
+ /////////////////////// Strategies for load balance //////////////////////////
+
+ private interface LoadBalancer {
+ int nextSocketIndex();
+ }
+
+ private class RoundRobinLoadBalancer implements LoadBalancer {
+ @Override
+ public int nextSocketIndex() {
+ final int socketSize = sockets.size();
+ // Round-robin, find the next alive client
+ for (int tryCount = 0; tryCount < socketSize; ++tryCount) {
+ final int clientIndex = (int) (currentClientIndex++ % socketSize);
+ if (Boolean.TRUE.equals(isSocketAlive.get(clientIndex))) {
+ return clientIndex;
+ }
+ }
+
+ throw new PipeConnectionException(
+ "All sockets are dead, please check the connection to the receiver.");
+ }
+ }
+
+ private class RandomLoadBalancer implements LoadBalancer {
+ @Override
+ public int nextSocketIndex() {
+ final int socketSize = sockets.size();
+ final int clientIndex = (int) (Math.random() * socketSize);
+ if (Boolean.TRUE.equals(isSocketAlive.get(clientIndex))) {
+ return clientIndex;
+ }
+
+ // Random, find the next alive client
+ for (int tryCount = 0; tryCount < socketSize - 1; ++tryCount) {
+ final int nextClientIndex = (clientIndex + tryCount + 1) % socketSize;
+ if (Boolean.TRUE.equals(isSocketAlive.get(nextClientIndex))) {
+ return nextClientIndex;
+ }
+ }
+
+ throw new PipeConnectionException(
+ "All sockets are dead, please check the connection to the receiver.");
+ }
+ }
+
+ private class PriorityLoadBalancer implements LoadBalancer {
+ @Override
+ public int nextSocketIndex() {
+ // Priority, find the first alive client
+ final int socketSize = sockets.size();
+ for (int i = 0; i < socketSize; ++i) {
+ if (Boolean.TRUE.equals(isSocketAlive.get(i))) {
+ return i;
+ }
+ }
+
+ throw new PipeConnectionException(
+ "All sockets are dead, please check the connection to the receiver.");
+ }
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/airgap/IoTDBConnector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/airgap/IoTDBConnector.java
new file mode 100644
index 00000000..f1c4da29
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/airgap/IoTDBConnector.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.protocol.airgap;
+
+import org.apache.iotdb.collector.plugin.builtin.annotation.TableModel;
+import org.apache.iotdb.collector.plugin.builtin.annotation.TreeModel;
+import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressor;
+import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressorConfig;
+import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressorFactory;
+import org.apache.iotdb.collector.plugin.builtin.sink.limiter.GlobalRateLimiter;
+import org.apache.iotdb.collector.plugin.builtin.sink.limiter.PipeEndPointRateLimiter;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferCompressedReq;
+import org.apache.iotdb.collector.plugin.builtin.sink.receiver.PipeReceiverStatusHandler;
+import org.apache.iotdb.collector.plugin.builtin.sink.utils.NodeUrlUtils;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.pipe.api.PipeConnector;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_SET;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_TABLET_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_SIZE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_HOST_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_IP_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_NODE_URLS_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PASSWORD_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PORT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_USERNAME_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_USER_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_USER_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_STRATEGY_SET;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_STRATEGY_SET;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_VALIDATION_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_VALIDATION_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_MARK_AS_PIPE_REQUEST_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_MARK_AS_PIPE_REQUEST_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_RATE_LIMIT_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_RATE_LIMIT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_COMPRESSOR_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_COMPRESSOR_ZSTD_LEVEL_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_FORMAT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_MODE_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_SIZE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_HOST_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_IP_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_NODE_URLS_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_PASSWORD_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_PORT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_USERNAME_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_USER_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LOAD_BALANCE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LOAD_TSFILE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LOAD_TSFILE_VALIDATION_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_MARK_AS_PIPE_REQUEST_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_RATE_LIMIT_KEY;
+
+@TreeModel
+@TableModel
+public abstract class IoTDBConnector implements PipeConnector {
+
+ private static final String PARSE_URL_ERROR_FORMATTER =
+ "Exception occurred while parsing node urls from target servers: {}";
+ private static final String PARSE_URL_ERROR_MESSAGE =
+ "Error occurred while parsing node urls from target servers, please check the specified 'host':'port' or 'node-urls'";
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConnector.class);
+
+ protected final List nodeUrls = new ArrayList<>();
+
+ protected String username = CONNECTOR_IOTDB_USER_DEFAULT_VALUE;
+ protected String password = CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE;
+
+ protected String loadBalanceStrategy;
+
+ protected String loadTsFileStrategy;
+ protected boolean loadTsFileValidation;
+
+ protected boolean shouldMarkAsPipeRequest;
+
+ private boolean isRpcCompressionEnabled;
+ private final List compressors = new ArrayList<>();
+
+ private static final Map, PipeEndPointRateLimiter>
+ PIPE_END_POINT_RATE_LIMITER_MAP = new ConcurrentHashMap<>();
+ private double endPointRateLimitBytesPerSecond = -1;
+ private static final GlobalRateLimiter GLOBAL_RATE_LIMITER = new GlobalRateLimiter();
+
+ protected boolean isTabletBatchModeEnabled = true;
+
+ protected PipeReceiverStatusHandler receiverStatusHandler;
+ protected boolean shouldReceiverConvertOnTypeMismatch =
+ CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE;
+
+ @Override
+ public void validate(final PipeParameterValidator validator) throws Exception {
+ final PipeParameters parameters = validator.getParameters();
+
+ validator.validate(
+ args ->
+ (boolean) args[0]
+ || (((boolean) args[1] || (boolean) args[2]) && (boolean) args[3])
+ || (boolean) args[4]
+ || (((boolean) args[5] || (boolean) args[6]) && (boolean) args[7]),
+ String.format(
+ "One of %s, %s:%s, %s, %s:%s must be specified",
+ CONNECTOR_IOTDB_NODE_URLS_KEY,
+ CONNECTOR_IOTDB_HOST_KEY,
+ CONNECTOR_IOTDB_PORT_KEY,
+ SINK_IOTDB_NODE_URLS_KEY,
+ SINK_IOTDB_HOST_KEY,
+ SINK_IOTDB_PORT_KEY),
+ parameters.hasAttribute(CONNECTOR_IOTDB_NODE_URLS_KEY),
+ parameters.hasAttribute(CONNECTOR_IOTDB_IP_KEY),
+ parameters.hasAttribute(CONNECTOR_IOTDB_HOST_KEY),
+ parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY),
+ parameters.hasAttribute(SINK_IOTDB_NODE_URLS_KEY),
+ parameters.hasAttribute(SINK_IOTDB_IP_KEY),
+ parameters.hasAttribute(SINK_IOTDB_HOST_KEY),
+ parameters.hasAttribute(SINK_IOTDB_PORT_KEY));
+
+ validator.validate(
+ requestMaxBatchSizeInBytes -> (long) requestMaxBatchSizeInBytes > 0,
+ String.format(
+ "%s must be > 0, but got %s",
+ SINK_IOTDB_BATCH_SIZE_KEY,
+ parameters.getLongOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY),
+ CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE)),
+ parameters.getLongOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY),
+ CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE));
+
+ // Check coexistence of user and username
+ validator.validateSynonymAttributes(
+ Arrays.asList(CONNECTOR_IOTDB_USER_KEY, SINK_IOTDB_USER_KEY),
+ Arrays.asList(CONNECTOR_IOTDB_USERNAME_KEY, SINK_IOTDB_USERNAME_KEY),
+ false);
+
+ username =
+ parameters.getStringOrDefault(
+ Arrays.asList(
+ CONNECTOR_IOTDB_USER_KEY,
+ SINK_IOTDB_USER_KEY,
+ CONNECTOR_IOTDB_USERNAME_KEY,
+ SINK_IOTDB_USERNAME_KEY),
+ CONNECTOR_IOTDB_USER_DEFAULT_VALUE);
+ password =
+ parameters.getStringOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_PASSWORD_KEY, SINK_IOTDB_PASSWORD_KEY),
+ CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE);
+
+ loadBalanceStrategy =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(CONNECTOR_LOAD_BALANCE_STRATEGY_KEY, SINK_LOAD_BALANCE_STRATEGY_KEY),
+ CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY)
+ .trim()
+ .toLowerCase();
+ validator.validate(
+ arg -> CONNECTOR_LOAD_BALANCE_STRATEGY_SET.contains(loadBalanceStrategy),
+ String.format(
+ "Load balance strategy should be one of %s, but got %s.",
+ CONNECTOR_LOAD_BALANCE_STRATEGY_SET, loadBalanceStrategy),
+ loadBalanceStrategy);
+
+ loadTsFileStrategy =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(CONNECTOR_LOAD_TSFILE_STRATEGY_KEY, SINK_LOAD_TSFILE_STRATEGY_KEY),
+ CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE)
+ .trim()
+ .toLowerCase();
+ validator.validate(
+ arg -> CONNECTOR_LOAD_TSFILE_STRATEGY_SET.contains(loadTsFileStrategy),
+ String.format(
+ "Load tsfile strategy should be one of %s, but got %s.",
+ CONNECTOR_LOAD_TSFILE_STRATEGY_SET, loadTsFileStrategy),
+ loadTsFileStrategy);
+ loadTsFileValidation =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(CONNECTOR_LOAD_TSFILE_VALIDATION_KEY, SINK_LOAD_TSFILE_VALIDATION_KEY),
+ CONNECTOR_LOAD_TSFILE_VALIDATION_DEFAULT_VALUE);
+
+ final int zstdCompressionLevel =
+ parameters.getIntOrDefault(
+ Arrays.asList(CONNECTOR_COMPRESSOR_ZSTD_LEVEL_KEY, SINK_COMPRESSOR_ZSTD_LEVEL_KEY),
+ CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE);
+ validator.validate(
+ arg ->
+ (int) arg >= CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE
+ && (int) arg <= CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE,
+ String.format(
+ "Zstd compression level should be in the range [%d, %d], but got %d.",
+ CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE,
+ CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE,
+ zstdCompressionLevel),
+ zstdCompressionLevel);
+
+ final String compressionTypes =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(CONNECTOR_COMPRESSOR_KEY, SINK_COMPRESSOR_KEY),
+ CONNECTOR_COMPRESSOR_DEFAULT_VALUE)
+ .toLowerCase();
+ if (!compressionTypes.isEmpty()) {
+ for (final String compressionType : compressionTypes.split(",")) {
+ final String trimmedCompressionType = compressionType.trim();
+ if (trimmedCompressionType.isEmpty()) {
+ continue;
+ }
+
+ validator.validate(
+ arg -> CONNECTOR_COMPRESSOR_SET.contains(trimmedCompressionType),
+ String.format(
+ "Compressor should be one of %s, but got %s.",
+ CONNECTOR_COMPRESSOR_SET, trimmedCompressionType),
+ trimmedCompressionType);
+ compressors.add(
+ PipeCompressorFactory.getCompressor(
+ new PipeCompressorConfig(trimmedCompressionType, zstdCompressionLevel)));
+ }
+ }
+ validator.validate(
+ arg -> compressors.size() <= Byte.MAX_VALUE,
+ String.format(
+ "The number of compressors should be less than or equal to %d, but got %d.",
+ Byte.MAX_VALUE, compressors.size()),
+ compressors.size());
+ isRpcCompressionEnabled = !compressors.isEmpty();
+
+ endPointRateLimitBytesPerSecond =
+ parameters.getDoubleOrDefault(
+ Arrays.asList(CONNECTOR_RATE_LIMIT_KEY, SINK_RATE_LIMIT_KEY),
+ CONNECTOR_RATE_LIMIT_DEFAULT_VALUE);
+ validator.validate(
+ arg -> endPointRateLimitBytesPerSecond <= Double.MAX_VALUE,
+ String.format(
+ "Rate limit should be in the range (0, %f], but got %f.",
+ Double.MAX_VALUE, endPointRateLimitBytesPerSecond),
+ endPointRateLimitBytesPerSecond);
+
+ validator.validate(
+ arg -> arg.equals("retry") || arg.equals("ignore"),
+ String.format(
+ "The value of key %s or %s must be either 'retry' or 'ignore'.",
+ CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY,
+ SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY),
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY,
+ SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY),
+ CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE)
+ .trim()
+ .toLowerCase());
+
+ validator.validateAttributeValueRange(
+ validator.getParameters().hasAttribute(CONNECTOR_FORMAT_KEY)
+ ? CONNECTOR_FORMAT_KEY
+ : SINK_FORMAT_KEY,
+ true,
+ CONNECTOR_FORMAT_TABLET_VALUE,
+ CONNECTOR_FORMAT_HYBRID_VALUE,
+ CONNECTOR_FORMAT_TS_FILE_VALUE);
+ }
+
+ @Override
+ public void customize(
+ final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration)
+ throws Exception {
+ nodeUrls.clear();
+ nodeUrls.addAll(parseNodeUrls(parameters));
+ LOGGER.info("IoTDBConnector nodeUrls: {}", nodeUrls);
+
+ isTabletBatchModeEnabled =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(
+ CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY, SINK_IOTDB_BATCH_MODE_ENABLE_KEY),
+ CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE)
+ || parameters
+ .getStringOrDefault(
+ Arrays.asList(CONNECTOR_FORMAT_KEY, SINK_FORMAT_KEY),
+ CONNECTOR_FORMAT_HYBRID_VALUE)
+ .equals(CONNECTOR_FORMAT_TS_FILE_VALUE);
+ LOGGER.info("IoTDBConnector isTabletBatchModeEnabled: {}", isTabletBatchModeEnabled);
+
+ shouldMarkAsPipeRequest =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(CONNECTOR_MARK_AS_PIPE_REQUEST_KEY, SINK_MARK_AS_PIPE_REQUEST_KEY),
+ CONNECTOR_MARK_AS_PIPE_REQUEST_DEFAULT_VALUE);
+ LOGGER.info("IoTDBConnector shouldMarkAsPipeRequest: {}", shouldMarkAsPipeRequest);
+
+ shouldReceiverConvertOnTypeMismatch =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(
+ CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY,
+ SINK_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY),
+ CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE);
+ LOGGER.info(
+ "IoTDBConnector {} = {}",
+ CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY,
+ shouldReceiverConvertOnTypeMismatch);
+ }
+
+ protected LinkedHashSet parseNodeUrls(final PipeParameters parameters)
+ throws PipeParameterNotValidException {
+ final LinkedHashSet givenNodeUrls = new LinkedHashSet<>(nodeUrls);
+
+ try {
+ if (parameters.hasAttribute(CONNECTOR_IOTDB_IP_KEY)
+ && parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY)) {
+ givenNodeUrls.add(
+ new TEndPoint(
+ parameters.getStringByKeys(CONNECTOR_IOTDB_IP_KEY),
+ parameters.getIntByKeys(CONNECTOR_IOTDB_PORT_KEY)));
+ }
+
+ if (parameters.hasAttribute(SINK_IOTDB_IP_KEY)
+ && parameters.hasAttribute(SINK_IOTDB_PORT_KEY)) {
+ givenNodeUrls.add(
+ new TEndPoint(
+ parameters.getStringByKeys(SINK_IOTDB_IP_KEY),
+ parameters.getIntByKeys(SINK_IOTDB_PORT_KEY)));
+ }
+
+ if (parameters.hasAttribute(CONNECTOR_IOTDB_HOST_KEY)
+ && parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY)) {
+ givenNodeUrls.add(
+ new TEndPoint(
+ parameters.getStringByKeys(CONNECTOR_IOTDB_HOST_KEY),
+ parameters.getIntByKeys(CONNECTOR_IOTDB_PORT_KEY)));
+ }
+
+ if (parameters.hasAttribute(SINK_IOTDB_HOST_KEY)
+ && parameters.hasAttribute(SINK_IOTDB_PORT_KEY)) {
+ givenNodeUrls.add(
+ new TEndPoint(
+ parameters.getStringByKeys(SINK_IOTDB_HOST_KEY),
+ parameters.getIntByKeys(SINK_IOTDB_PORT_KEY)));
+ }
+
+ if (parameters.hasAttribute(CONNECTOR_IOTDB_NODE_URLS_KEY)) {
+ givenNodeUrls.addAll(
+ NodeUrlUtils.parseTEndPointUrls(
+ Arrays.asList(
+ parameters
+ .getStringByKeys(CONNECTOR_IOTDB_NODE_URLS_KEY)
+ .replace(" ", "")
+ .split(","))));
+ }
+
+ if (parameters.hasAttribute(SINK_IOTDB_NODE_URLS_KEY)) {
+ givenNodeUrls.addAll(
+ NodeUrlUtils.parseTEndPointUrls(
+ Arrays.asList(
+ parameters
+ .getStringByKeys(SINK_IOTDB_NODE_URLS_KEY)
+ .replace(" ", "")
+ .split(","))));
+ }
+ } catch (final Exception e) {
+ LOGGER.warn(PARSE_URL_ERROR_FORMATTER, e.toString());
+ throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE);
+ }
+
+ checkNodeUrls(givenNodeUrls);
+
+ return givenNodeUrls;
+ }
+
+ private void checkNodeUrls(final Set nodeUrls) throws PipeParameterNotValidException {
+ for (final TEndPoint nodeUrl : nodeUrls) {
+ if (Objects.isNull(nodeUrl.ip) || nodeUrl.ip.isEmpty()) {
+ LOGGER.warn(PARSE_URL_ERROR_FORMATTER, "host cannot be empty");
+ throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE);
+ }
+ if (nodeUrl.port == 0) {
+ LOGGER.warn(PARSE_URL_ERROR_FORMATTER, "port cannot be empty");
+ throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE);
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ // TODO: Not all the limiters should be closed here, but it's fine for now.
+ PIPE_END_POINT_RATE_LIMITER_MAP.clear();
+ }
+
+ protected TPipeTransferReq compressIfNeeded(final TPipeTransferReq req) throws IOException {
+ return isRpcCompressionEnabled
+ ? PipeTransferCompressedReq.toTPipeTransferReq(req, compressors)
+ : req;
+ }
+
+ protected byte[] compressIfNeeded(final byte[] reqInBytes) throws IOException {
+ return isRpcCompressionEnabled
+ ? PipeTransferCompressedReq.toTPipeTransferReqBytes(reqInBytes, compressors)
+ : reqInBytes;
+ }
+
+ public boolean isRpcCompressionEnabled() {
+ return isRpcCompressionEnabled;
+ }
+
+ public List