diff --git a/iotdb-collector/collector-core/pom.xml b/iotdb-collector/collector-core/pom.xml index 7b68e680..f6619948 100644 --- a/iotdb-collector/collector-core/pom.xml +++ b/iotdb-collector/collector-core/pom.xml @@ -48,20 +48,15 @@ - - org.apache.iotdb - node-commons - 1.3.2 - org.apache.iotdb service-rpc - 1.3.2 + 2.0.1-beta org.apache.iotdb pipe-api - 1.3.2 + 2.0.1-beta jakarta.servlet @@ -104,6 +99,101 @@ disruptor + org.java-websocket + Java-WebSocket + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + org.apache.commons + commons-pool2 + + + org.apache.tsfile + tsfile + + + + + + + + org.apache.iotdb + iotdb-thrift-commons + 2.0.1-beta + + + org.apache.iotdb + iotdb-thrift + 2.0.1-beta + + + org.apache.iotdb + iotdb-session + 2.0.1-beta + + + net.java.dev.jna + jna + + + net.java.dev.jna + jna-platform + + + com.google.code.findbugs + jsr305 + + + org.apache.thrift + libthrift + + + com.google.guava + guava + + + org.eclipse.milo + stack-core + + + org.eclipse.milo + sdk-core + + + org.eclipse.milo + stack-server + + + org.eclipse.milo + sdk-server + + + commons-io + commons-io + + + com.github.luben + zstd-jni + + + org.apache.tsfile + common + + + com.github.ben-manes.caffeine + caffeine + + + io.moquette + moquette-broker + com.google.code.findbugs jsr305 @@ -121,7 +211,7 @@ org.glassfish.jersey.inject:jersey-hk2 - + org.apache.maven.pluginsmaven-compiler-plugin88 diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/IoTDBCommonOptions.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/IoTDBCommonOptions.java new file mode 100644 index 00000000..ac2304f6 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/IoTDBCommonOptions.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.config; + +public class IoTDBCommonOptions extends Options {} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java index e0670863..97d4d8da 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java @@ -36,6 +36,7 @@ public class Options { try { Class.forName(ApiServiceOptions.class.getName()); Class.forName(TaskRuntimeOptions.class.getName()); + Class.forName(PipeOptions.class.getName()); } catch (final ClassNotFoundException e) { throw new RuntimeException("Failed to load options", e); } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/PipeOptions.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/PipeOptions.java new file mode 100644 index 00000000..d1e6ad46 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/PipeOptions.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.config; + +import org.apache.iotdb.rpc.RpcUtils; + +import java.io.File; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +public class PipeOptions extends Options { + + public static volatile Option PIPE_CHECK_MEMORY_ENOUGH_INTERVAL_MS = + new Option("pipe_check_memory_enough_interval_ms", 10L) { + @Override + public void setValue(final String valueString) { + value = Long.parseLong(valueString); + } + }; + + public static final Option PIPE_CONNECTOR_READ_FILE_BUFFER_MEMORY_CONTROL = + new Option("pipe_connector_read_file_buffer_memory_control", false) { + @Override + public void setValue(final String valueString) { + value = Boolean.parseBoolean(valueString); + } + }; + + public static final Option PIPE_CONNECTOR_TRANSFER_TIMEOUT_MS = + new Option("pipe_connector_transfer_timeout_ms", 15 * 60 * 1000) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option PIPE_SUBTASK_EXECUTOR_MAX_THREAD_NUM = + new Option( + "pipe_subtask_executor_max_thread_num", + Math.max(5, Runtime.getRuntime().availableProcessors() / 2)) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option PIPE_CONNECTOR_READ_FILE_BUFFER_SIZE = + new Option("pipe_connector_read_file_buffer_size", 8388608) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option PIPE_EVENT_REFERENCE_TRACKING_ENABLED = + new Option("pipe_event_reference_tracking_enabled", true) { + @Override + public void setValue(final String valueString) { + value = Boolean.parseBoolean(valueString); + } + }; + + public static final Option RATE_LIMITER_HOT_RELOAD_CHECK_INTERVAL_MS = + new Option("rate_limiter_hot_reload_check_interval_ms", 1000) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option PIPE_ALL_SINK_RATE_LIMIT_BYTES_PER_SECOND = + new Option("pipe_all_sink_rate_limit_bytes_per_second", -1d) { + @Override + public void setValue(final String valueString) { + value = Double.parseDouble(valueString); + } + }; + + public static final Option PIPE_DATA_STRUCTURE_TABLET_SIZE_IN_BYTES = + new Option("pipe_data_structure_tablet_size_in_bytes", 2097152) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option PIPE_DATA_STRUCTURE_TABLET_ROW_SIZE = + new Option("pipe_data_structure_tablet_row_size", 2048) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option PIPE_AIR_GAP_RECEIVER_ENABLED = + new Option("pipe_air_gap_receiver_enabled", true) { + + @Override + public void setValue(final String valueString) { + value = Boolean.parseBoolean(valueString); + } + }; + + public static final Option PIPE_AIR_GAP_RECEIVER_PORT = + new Option("pipe_air_gap_receiver_port", 9780) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option RPC_ADDRESS = + new Option("rpc_address", "0.0.0.0") { + @Override + public void setValue(final String valueString) { + value = valueString; + } + }; + + public static final Option RPC_PORT = + new Option("rpc_port", 6667) { + @Override + public void setValue(String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option TIMESTAMP_PRECISION = + new Option("timestamp_precision", "ms") { + @Override + public void setValue(final String valueString) { + value = valueString; + } + }; + + public static final Option PIPE_SUBTASK_EXECUTOR_PENDING_QUEUE_MAX_BLOCKING_TIME_MS = + new Option("pipe_subtask_executor_pending_queue_max_blocking_time_ms", 1000L) { + @Override + public void setValue(final String valueString) { + value = Long.parseLong(valueString); + } + }; + + public static final Option TIMESTAMP_PRECISION_CHECK_ENABLED = + new Option("timestamp_precision_check_enabled", true) { + @Override + public void setValue(final String valueString) { + value = Boolean.parseBoolean(valueString); + } + }; + + public static final Option DN_CONNECTION_TIMEOUT_IN_MS = + new Option("dn_connection_timeout_in_ms", (int) TimeUnit.SECONDS.toMillis(60)) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option IS_RPC_THRIFT_COMPRESSION_ENABLED = + new Option("is_rpc_thrift_compression_enabled", false) { + @Override + public void setValue(final String valueString) { + value = Boolean.parseBoolean(valueString); + } + }; + + public static final Option PIPE_CONNECTOR_REQUEST_SLICE_THRESHOLD_BYTES = + new Option( + "pipe_connector_request_slice_threshold_bytes", + (int) (RpcUtils.THRIFT_FRAME_MAX_SIZE * 0.8)) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option CLUSTER_ID = + new Option("cluster_id", "") { + @Override + public void setValue(final String valueString) { + value = valueString; + } + }; + + public static final Option PIPE_CONNECTOR_HANDSHAKE_TIMEOUT_MS = + new Option("pipe_connector_handshake_timeout_ms", 10 * 1000) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option PIPE_CONNECTOR_RPC_THRIFT_COMPRESSION_ENABLED = + new Option("pipe_connector_rpc_thrift_compression_enabled", false) { + @Override + public void setValue(final String valueString) { + value = Boolean.parseBoolean(valueString); + } + }; + + public static final Option PIPE_ASYNC_CONNECTOR_SELECTOR_NUMBER = + new Option( + "pipe_async_connector_selector_number", + Math.max(4, Runtime.getRuntime().availableProcessors() / 2)) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option PIPE_ASYNC_CONNECTOR_MAX_CLIENT_NUMBER = + new Option( + "pipe_async_connector_max_client_number", + Math.max(16, Runtime.getRuntime().availableProcessors() / 2)) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option SYSTEM_DIR = + new Option( + "system_dir", + IoTDBConstant.DN_DEFAULT_DATA_DIR + File.separator + IoTDBConstant.SYSTEM_FOLDER_NAME) { + @Override + public void setValue(final String valueString) { + value = valueString; + } + }; + + public static final Option PIPE_RECEIVER_FILE_DIRS = + new Option("pipe_receiver_file_dirs", new String[0]) { + @Override + public void setValue(final String valueString) { + value = + (Objects.isNull(valueString) || valueString.split(",").length == 0) + ? new String[] { + SYSTEM_DIR.value() + File.separator + "pipe" + File.separator + "receiver" + } + : valueString.split(","); + } + }; + + public static volatile Option DATA_NODE_ID = + new Option("data_node_id", -1) { + @Override + public void setValue(final String valueString) { + value = Integer.parseInt(valueString); + } + }; + + public static final Option PIPE_MEMORY_ALLOCATE_FOR_TS_FILE_SEQUENCE_READER_IN_BYTES = + new Option( + "pipe_memory_allocate_for_ts_file_sequence_reader_in_bytes", 2 * 1024 * 1024L) { + @Override + public void setValue(final String valueString) { + value = Long.parseLong(valueString); + } + }; +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java index 8ace81c5..07c60a20 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java @@ -49,6 +49,11 @@ public long getCreationTime() { return creationTime; } + @Override + public int getRegionId() { + return getInstanceIndex(); + } + public int getParallelism() { return parallelism; } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/BuiltinPlugin.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/BuiltinPlugin.java index 203570c7..af408668 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/BuiltinPlugin.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/BuiltinPlugin.java @@ -20,7 +20,7 @@ package org.apache.iotdb.collector.plugin.builtin; import org.apache.iotdb.collector.plugin.builtin.processor.DoNothingProcessor; -import org.apache.iotdb.collector.plugin.builtin.sink.DemoSink; +import org.apache.iotdb.collector.plugin.builtin.sink.protocol.DemoSink; import org.apache.iotdb.collector.plugin.builtin.source.HttpPullSource; import org.apache.iotdb.collector.plugin.builtin.source.HttpPushSource; diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBClientManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBClientManager.java new file mode 100644 index 00000000..2bfd5c67 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBClientManager.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.client; + +import org.apache.iotdb.collector.config.PipeOptions; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.SocketTimeoutException; +import java.util.List; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +public abstract class IoTDBClientManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBClientManager.class); + + protected final List endPointList; + protected long currentClientIndex = 0; + + protected final boolean useLeaderCache; + + protected final String username; + protected final String password; + + protected final boolean validateTsFile; + + protected final boolean shouldReceiverConvertOnTypeMismatch; + protected final String loadTsFileStrategy; + + protected final boolean shouldMarkAsPipeRequest; + + // This flag indicates whether the receiver supports mods transferring if + // it is a DataNode receiver. The flag is useless for configNode receiver. + protected boolean supportModsIfIsDataNodeReceiver = true; + + private static final int MAX_CONNECTION_TIMEOUT_MS = 24 * 60 * 60 * 1000; // 1 day + private static final int FIRST_ADJUSTMENT_TIMEOUT_MS = 6 * 60 * 60 * 1000; // 6 hours + protected static final AtomicInteger CONNECTION_TIMEOUT_MS = + new AtomicInteger(PipeOptions.PIPE_CONNECTOR_TRANSFER_TIMEOUT_MS.value()); + + protected IoTDBClientManager( + final List endPointList, + /* The following parameters are used locally. */ + final boolean useLeaderCache, + /* The following parameters are used to handshake with the receiver. */ + final String username, + final String password, + final boolean shouldReceiverConvertOnTypeMismatch, + final String loadTsFileStrategy, + final boolean validateTsFile, + final boolean shouldMarkAsPipeRequest) { + this.endPointList = endPointList; + + this.useLeaderCache = useLeaderCache; + + this.username = username; + this.password = password; + this.shouldReceiverConvertOnTypeMismatch = shouldReceiverConvertOnTypeMismatch; + this.loadTsFileStrategy = loadTsFileStrategy; + this.validateTsFile = validateTsFile; + this.shouldMarkAsPipeRequest = shouldMarkAsPipeRequest; + } + + public boolean supportModsIfIsDataNodeReceiver() { + return supportModsIfIsDataNodeReceiver; + } + + public void adjustTimeoutIfNecessary(Throwable e) { + do { + if (e instanceof SocketTimeoutException || e instanceof TimeoutException) { + int newConnectionTimeout; + try { + newConnectionTimeout = + Math.min( + Math.max( + FIRST_ADJUSTMENT_TIMEOUT_MS, + Math.toIntExact(CONNECTION_TIMEOUT_MS.get() * 2L)), + MAX_CONNECTION_TIMEOUT_MS); + } catch (ArithmeticException arithmeticException) { + newConnectionTimeout = MAX_CONNECTION_TIMEOUT_MS; + } + + if (newConnectionTimeout != CONNECTION_TIMEOUT_MS.get()) { + CONNECTION_TIMEOUT_MS.set(newConnectionTimeout); + LOGGER.info( + "Pipe connection timeout is adjusted to {} ms ({} mins)", + newConnectionTimeout, + newConnectionTimeout / 60000.0); + } + return; + } + } while ((e = e.getCause()) != null); + } + + public int getConnectionTimeout() { + return CONNECTION_TIMEOUT_MS.get(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeCacheLeaderClientManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeCacheLeaderClientManager.java new file mode 100644 index 00000000..945bb8f4 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeCacheLeaderClientManager.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.client; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Weigher; +import com.google.common.util.concurrent.AtomicDouble; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ConcurrentHashMap; + +public interface IoTDBDataNodeCacheLeaderClientManager { + + LeaderCacheManager LEADER_CACHE_MANAGER = new LeaderCacheManager(); + + class LeaderCacheManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(LeaderCacheManager.class); + // private static final PipeConfig CONFIG = PipeConfig.getInstance(); + + private final AtomicDouble memoryUsageCheatFactor = new AtomicDouble(1); + + // leader cache built by LRU + private final Cache device2endpoint; + // a hashmap to reuse the created endpoint + private final ConcurrentHashMap endPoints = new ConcurrentHashMap<>(); + + public LeaderCacheManager() { + // final long initMemorySizeInBytes = + // PipeDataNodeResourceManager.memory().getTotalMemorySizeInBytes() / 10; + // final long maxMemorySizeInBytes = + // (long) + // (PipeDataNodeResourceManager.memory().getTotalMemorySizeInBytes() + // * CONFIG.getPipeLeaderCacheMemoryUsagePercentage()); + // + // // properties required by pipe memory control framework + // final PipeMemoryBlock allocatedMemoryBlock = + // PipeDataNodeResourceManager.memory() + // .tryAllocate(initMemorySizeInBytes) + // .setShrinkMethod(oldMemory -> Math.max(oldMemory / 2, 1)) + // .setShrinkCallback( + // (oldMemory, newMemory) -> { + // memoryUsageCheatFactor.updateAndGet( + // factor -> factor * ((double) oldMemory / newMemory)); + // LOGGER.info( + // "LeaderCacheManager.allocatedMemoryBlock has shrunk from {} to {}.", + // oldMemory, + // newMemory); + // }) + // .setExpandMethod( + // oldMemory -> Math.min(Math.max(oldMemory, 1) * 2, maxMemorySizeInBytes)) + // .setExpandCallback( + // (oldMemory, newMemory) -> { + // memoryUsageCheatFactor.updateAndGet( + // factor -> factor / ((double) newMemory / oldMemory)); + // LOGGER.info( + // "LeaderCacheManager.allocatedMemoryBlock has expanded from {} to {}.", + // oldMemory, + // newMemory); + // }); + + device2endpoint = + Caffeine.newBuilder() + // .maximumWeight(allocatedMemoryBlock.getMemoryUsageInBytes()) + .weigher( + (Weigher) + (device, endPoint) -> { + final long weightInLong = + (long) (device.getBytes().length * memoryUsageCheatFactor.get()); + if (weightInLong <= 0) { + return Integer.MAX_VALUE; + } + final int weightInInt = (int) weightInLong; + return weightInInt != weightInLong ? Integer.MAX_VALUE : weightInInt; + }) + .recordStats() + .build(); + } + + public TEndPoint getLeaderEndPoint(final String deviceId) { + return deviceId == null ? null : device2endpoint.getIfPresent(deviceId); + } + + public void updateLeaderEndPoint(final String deviceId, final TEndPoint endPoint) { + if (deviceId == null || endPoint == null) { + return; + } + + final TEndPoint endPointFromMap = endPoints.putIfAbsent(endPoint, endPoint); + if (endPointFromMap != null) { + device2endpoint.put(deviceId, endPointFromMap); + } else { + device2endpoint.put(deviceId, endPoint); + } + } + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeSyncClientManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeSyncClientManager.java new file mode 100644 index 00000000..93b7e601 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeSyncClientManager.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.client; + +import org.apache.iotdb.collector.config.PipeOptions; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV1Req; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferHandshakeV2Req; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; + +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class IoTDBDataNodeSyncClientManager extends IoTDBSyncClientManager + implements IoTDBDataNodeCacheLeaderClientManager { + + private static final Logger LOGGER = + LoggerFactory.getLogger(IoTDBDataNodeSyncClientManager.class); + + public IoTDBDataNodeSyncClientManager( + final List endPoints, + final boolean useSSL, + final String trustStorePath, + final String trustStorePwd, + /* The following parameters are used locally. */ + final boolean useLeaderCache, + final String loadBalanceStrategy, + /* The following parameters are used to handshake with the receiver. */ + final String username, + final String password, + final boolean shouldReceiverConvertOnTypeMismatch, + final String loadTsFileStrategy, + final boolean validateTsFile, + final boolean shouldMarkAsPipeRequest) { + super( + endPoints, + useSSL, + trustStorePath, + trustStorePwd, + useLeaderCache, + loadBalanceStrategy, + username, + password, + shouldReceiverConvertOnTypeMismatch, + loadTsFileStrategy, + validateTsFile, + shouldMarkAsPipeRequest); + } + + @Override + protected PipeTransferDataNodeHandshakeV1Req buildHandshakeV1Req() throws IOException { + return PipeTransferDataNodeHandshakeV1Req.toTPipeTransferReq( + PipeOptions.TIMESTAMP_PRECISION.value()); + } + + @Override + protected PipeTransferHandshakeV2Req buildHandshakeV2Req(final Map params) + throws IOException { + return PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params); + } + + @Override + protected String getClusterId() { + return PipeOptions.CLUSTER_ID.value(); + } + + public Pair getClient(final String deviceId) { + final TEndPoint endPoint = LEADER_CACHE_MANAGER.getLeaderEndPoint(deviceId); + return useLeaderCache + && endPoint != null + && endPoint2ClientAndStatus.containsKey(endPoint) + && Boolean.TRUE.equals(endPoint2ClientAndStatus.get(endPoint).getRight()) + ? endPoint2ClientAndStatus.get(endPoint) + : getClient(); + } + + public Pair getClient(final TEndPoint endPoint) { + return useLeaderCache + && endPoint != null + && endPoint2ClientAndStatus.containsKey(endPoint) + && Boolean.TRUE.equals(endPoint2ClientAndStatus.get(endPoint).getRight()) + ? endPoint2ClientAndStatus.get(endPoint) + : getClient(); + } + + public void updateLeaderCache(final String deviceId, final TEndPoint endPoint) { + if (!useLeaderCache || deviceId == null || endPoint == null) { + return; + } + + try { + if (!endPoint2ClientAndStatus.containsKey(endPoint)) { + endPointList.add(endPoint); + endPoint2ClientAndStatus.put(endPoint, new Pair<>(null, false)); + reconstructClient(endPoint); + } + + LEADER_CACHE_MANAGER.updateLeaderEndPoint(deviceId, endPoint); + } catch (final Exception e) { + LOGGER.warn( + "Failed to update leader cache for device {} with endpoint {}:{}.", + deviceId, + endPoint.getIp(), + endPoint.getPort(), + e); + } + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClient.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClient.java new file mode 100644 index 00000000..2b1b7b94 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClient.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.client; + +import org.apache.iotdb.collector.config.PipeOptions; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferSliceReq; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.pipe.api.exception.PipeConnectionException; +import org.apache.iotdb.rpc.DeepCopyRpcTransportFactory; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.TimeoutChangeableTransport; +import org.apache.iotdb.service.rpc.thrift.IClientRPCService; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; + +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.atomic.AtomicInteger; + +public class IoTDBSyncClient extends IClientRPCService.Client + implements ThriftClient, AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSyncClient.class); + + private static final AtomicInteger SLICE_ORDER_ID_GENERATOR = new AtomicInteger(0); + + private final String ipAddress; + private final int port; + private final TEndPoint endPoint; + + public IoTDBSyncClient( + ThriftClientProperty property, + String ipAddress, + int port, + boolean useSSL, + String trustStore, + String trustStorePwd) + throws TTransportException { + super( + property + .getProtocolFactory() + .getProtocol( + useSSL + ? DeepCopyRpcTransportFactory.INSTANCE.getTransport( + ipAddress, + port, + property.getConnectionTimeoutMs(), + trustStore, + trustStorePwd) + : DeepCopyRpcTransportFactory.INSTANCE.getTransport( + ipAddress, port, property.getConnectionTimeoutMs()))); + this.ipAddress = ipAddress; + this.port = port; + this.endPoint = new TEndPoint(ipAddress, port); + final TTransport transport = getInputProtocol().getTransport(); + if (!transport.isOpen()) { + transport.open(); + } + } + + public String getIpAddress() { + return ipAddress; + } + + public int getPort() { + return port; + } + + public TEndPoint getEndPoint() { + return endPoint; + } + + public void setTimeout(int timeout) { + ((TimeoutChangeableTransport) (getInputProtocol().getTransport())).setTimeout(timeout); + } + + @Override + public TPipeTransferResp pipeTransfer(final TPipeTransferReq req) throws TException { + final int bodySizeLimit = PipeOptions.PIPE_CONNECTOR_REQUEST_SLICE_THRESHOLD_BYTES.value(); + if (req.getVersion() != IoTDBConnectorRequestVersion.VERSION_1.getVersion() + || req.body.limit() < bodySizeLimit) { + return super.pipeTransfer(req); + } + + LOGGER.warn( + "The body size of the request is too large. The request will be sliced. Origin req: {}-{}. " + + "Request body size: {}, threshold: {}", + req.getVersion(), + req.getType(), + req.body.limit(), + bodySizeLimit); + + try { + final int sliceOrderId = SLICE_ORDER_ID_GENERATOR.getAndIncrement(); + // Slice the buffer to avoid the buffer being too large + final int sliceCount = + req.body.limit() / bodySizeLimit + (req.body.limit() % bodySizeLimit == 0 ? 0 : 1); + for (int i = 0; i < sliceCount; ++i) { + final int startIndexInBody = i * bodySizeLimit; + final int endIndexInBody = Math.min((i + 1) * bodySizeLimit, req.body.limit()); + final TPipeTransferResp sliceResp = + super.pipeTransfer( + PipeTransferSliceReq.toTPipeTransferReq( + sliceOrderId, + req.getType(), + i, + sliceCount, + req.body.duplicate(), + startIndexInBody, + endIndexInBody)); + + if (i == sliceCount - 1) { + return sliceResp; + } + + if (sliceResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new PipeConnectionException( + String.format( + "Failed to transfer slice. Origin req: %s-%s, slice index: %d, slice count: %d. Reason: %s", + req.getVersion(), req.getType(), i, sliceCount, sliceResp.getStatus())); + } + } + + // Should not reach here + return super.pipeTransfer(req); + } catch (final Exception e) { + LOGGER.warn( + "Failed to transfer slice. Origin req: {}-{}. Retry the whole transfer.", + req.getVersion(), + req.getType(), + e); + // Fall back to the original behavior + return super.pipeTransfer(req); + } + } + + @Override + public void close() throws Exception { + invalidate(); + } + + @Override + public void invalidate() { + if (getInputProtocol().getTransport().isOpen()) { + getInputProtocol().getTransport().close(); + } + } + + @Override + public void invalidateAll() { + invalidate(); + } + + @Override + public boolean printLogWhenEncounterException() { + return true; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClientManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClientManager.java new file mode 100644 index 00000000..74db57d4 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClientManager.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.client; + +import org.apache.iotdb.collector.config.PipeOptions; +import org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeTransferHandshakeConstant; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferHandshakeV1Req; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferHandshakeV2Req; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.pipe.api.exception.PipeConnectionException; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; + +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY; + +public abstract class IoTDBSyncClientManager extends IoTDBClientManager implements Closeable { + + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSyncClientManager.class); + + private final boolean useSSL; + private final String trustStorePath; + private final String trustStorePwd; + + protected final Map> endPoint2ClientAndStatus = + new ConcurrentHashMap<>(); + private final Map endPoint2HandshakeErrorMessage = new ConcurrentHashMap<>(); + + private final LoadBalancer loadBalancer; + + protected IoTDBSyncClientManager( + List endPoints, + boolean useSSL, + String trustStorePath, + String trustStorePwd, + /* The following parameters are used locally. */ + boolean useLeaderCache, + String loadBalanceStrategy, + /* The following parameters are used to handshake with the receiver. */ + String username, + String password, + boolean shouldReceiverConvertOnTypeMismatch, + String loadTsFileStrategy, + boolean validateTsFile, + boolean shouldMarkAsPipeRequest) { + super( + endPoints, + useLeaderCache, + username, + password, + shouldReceiverConvertOnTypeMismatch, + loadTsFileStrategy, + validateTsFile, + shouldMarkAsPipeRequest); + + this.useSSL = useSSL; + this.trustStorePath = trustStorePath; + this.trustStorePwd = trustStorePwd; + + for (final TEndPoint endPoint : endPoints) { + endPoint2ClientAndStatus.put(endPoint, new Pair<>(null, false)); + } + + switch (loadBalanceStrategy) { + case CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY: + loadBalancer = new RoundRobinLoadBalancer(); + break; + case CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY: + loadBalancer = new RandomLoadBalancer(); + break; + case CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY: + loadBalancer = new PriorityLoadBalancer(); + break; + default: + LOGGER.warn( + "Unknown load balance strategy: {}, use round-robin strategy instead.", + loadBalanceStrategy); + loadBalancer = new RoundRobinLoadBalancer(); + } + } + + public void checkClientStatusAndTryReconstructIfNecessary() { + // Reconstruct all dead clients + for (final Map.Entry> entry : + endPoint2ClientAndStatus.entrySet()) { + if (Boolean.TRUE.equals(entry.getValue().getRight())) { + continue; + } + + reconstructClient(entry.getKey()); + } + + // Check whether any clients are available + for (final Pair clientAndStatus : endPoint2ClientAndStatus.values()) { + if (Boolean.TRUE.equals(clientAndStatus.getRight())) { + return; + } + } + + // If all clients are not available, throw an exception + final StringBuilder errorMessage = + new StringBuilder( + String.format( + "All target servers %s are not available.", endPoint2ClientAndStatus.keySet())); + for (final Map.Entry entry : endPoint2HandshakeErrorMessage.entrySet()) { + errorMessage + .append(" (") + .append("host: ") + .append(entry.getKey().getIp()) + .append(", port: ") + .append(entry.getKey().getPort()) + .append(", because: ") + .append(entry.getValue()) + .append(")"); + } + throw new PipeConnectionException(errorMessage.toString()); + } + + protected void reconstructClient(TEndPoint endPoint) { + endPoint2HandshakeErrorMessage.remove(endPoint); + + final Pair clientAndStatus = endPoint2ClientAndStatus.get(endPoint); + + if (clientAndStatus.getLeft() != null) { + try { + clientAndStatus.getLeft().close(); + } catch (Exception e) { + LOGGER.warn( + "Failed to close client with target server ip: {}, port: {}, because: {}. Ignore it.", + endPoint.getIp(), + endPoint.getPort(), + e.getMessage()); + } + } + + // It is necessary to ensure that the client is initialized successfully and not null. If false + // is returned, it means that the initialization is not successful and the handshake operation + // is not performed. + if (initClientAndStatus(clientAndStatus, endPoint)) { + sendHandshakeReq(clientAndStatus); + } + } + + private boolean initClientAndStatus( + final Pair clientAndStatus, final TEndPoint endPoint) { + try { + clientAndStatus.setLeft( + new IoTDBSyncClient( + new ThriftClientProperty.Builder() + .setConnectionTimeoutMs(PipeOptions.PIPE_CONNECTOR_HANDSHAKE_TIMEOUT_MS.value()) + .setRpcThriftCompressionEnabled( + PipeOptions.PIPE_CONNECTOR_RPC_THRIFT_COMPRESSION_ENABLED.value()) + .build(), + endPoint.getIp(), + endPoint.getPort(), + useSSL, + trustStorePath, + trustStorePwd)); + return true; + } catch (Exception e) { + endPoint2HandshakeErrorMessage.put(endPoint, e.getMessage()); + LOGGER.warn( + "Failed to initialize client with target server ip: {}, port: {}, because {}", + endPoint.getIp(), + endPoint.getPort(), + e.getMessage(), + e); + return false; + } + } + + public void sendHandshakeReq(final Pair clientAndStatus) { + final IoTDBSyncClient client = clientAndStatus.getLeft(); + try { + final HashMap params = new HashMap<>(); + params.put( + PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION, + PipeOptions.TIMESTAMP_PRECISION.value()); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId()); + params.put( + PipeTransferHandshakeConstant.HANDSHAKE_KEY_CONVERT_ON_TYPE_MISMATCH, + Boolean.toString(shouldReceiverConvertOnTypeMismatch)); + params.put( + PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, loadTsFileStrategy); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, username); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password); + params.put( + PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE, + Boolean.toString(validateTsFile)); + params.put( + PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST, + Boolean.toString(shouldMarkAsPipeRequest)); + + // Try to handshake by PipeTransferHandshakeV2Req. + TPipeTransferResp resp = client.pipeTransfer(buildHandshakeV2Req(params)); + // Receiver may be an old version, so we need to retry to handshake by + // PipeTransferHandshakeV1Req. + if (resp.getStatus().getCode() == TSStatusCode.PIPE_TYPE_ERROR.getStatusCode()) { + LOGGER.warn( + "Handshake error with target server ip: {}, port: {}, because: {}. " + + "Retry to handshake by PipeTransferHandshakeV1Req.", + client.getIpAddress(), + client.getPort(), + resp.getStatus()); + supportModsIfIsDataNodeReceiver = false; + resp = client.pipeTransfer(buildHandshakeV1Req()); + } + + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.warn( + "Handshake error with target server ip: {}, port: {}, because: {}.", + client.getIpAddress(), + client.getPort(), + resp.getStatus()); + endPoint2HandshakeErrorMessage.put(client.getEndPoint(), resp.getStatus().getMessage()); + } else { + clientAndStatus.setRight(true); + client.setTimeout(CONNECTION_TIMEOUT_MS.get()); + LOGGER.info( + "Handshake success. Target server ip: {}, port: {}", + client.getIpAddress(), + client.getPort()); + } + } catch (Exception e) { + LOGGER.warn( + "Handshake error with target server ip: {}, port: {}, because: {}.", + client.getIpAddress(), + client.getPort(), + e.getMessage(), + e); + endPoint2HandshakeErrorMessage.put(client.getEndPoint(), e.getMessage()); + } + } + + protected abstract PipeTransferHandshakeV1Req buildHandshakeV1Req() throws IOException; + + protected abstract PipeTransferHandshakeV2Req buildHandshakeV2Req(Map params) + throws IOException; + + protected abstract String getClusterId(); + + public Pair getClient() { + return loadBalancer.getClient(); + } + + @Override + public void close() { + for (final Map.Entry> entry : + endPoint2ClientAndStatus.entrySet()) { + final TEndPoint endPoint = entry.getKey(); + final Pair clientAndStatus = entry.getValue(); + + if (clientAndStatus == null) { + continue; + } + + try { + if (clientAndStatus.getLeft() != null) { + clientAndStatus.getLeft().close(); + clientAndStatus.setLeft(null); + } + LOGGER.info("Client {}:{} closed.", endPoint.getIp(), endPoint.getPort()); + } catch (Exception e) { + LOGGER.warn( + "Failed to close client {}:{}, because: {}.", + endPoint.getIp(), + endPoint.getPort(), + e.getMessage(), + e); + } finally { + clientAndStatus.setRight(false); + } + } + } + + /////////////////////// Strategies for load balance ////////////////////////// + + private interface LoadBalancer { + Pair getClient(); + } + + private class RoundRobinLoadBalancer implements LoadBalancer { + @Override + public Pair getClient() { + final int clientSize = endPointList.size(); + // Round-robin, find the next alive client + for (int tryCount = 0; tryCount < clientSize; ++tryCount) { + final int clientIndex = (int) (currentClientIndex++ % clientSize); + final Pair clientAndStatus = + endPoint2ClientAndStatus.get(endPointList.get(clientIndex)); + if (Boolean.TRUE.equals(clientAndStatus.getRight())) { + return clientAndStatus; + } + } + + throw new PipeConnectionException( + "All clients are dead, please check the connection to the receiver."); + } + } + + private class RandomLoadBalancer implements LoadBalancer { + @Override + public Pair getClient() { + final int clientSize = endPointList.size(); + final int clientIndex = (int) (Math.random() * clientSize); + final Pair clientAndStatus = + endPoint2ClientAndStatus.get(endPointList.get(clientIndex)); + if (Boolean.TRUE.equals(clientAndStatus.getRight())) { + return clientAndStatus; + } + + // Random, find the next alive client + for (int tryCount = 0; tryCount < clientSize - 1; ++tryCount) { + final int nextClientIndex = (clientIndex + tryCount + 1) % clientSize; + final Pair nextClientAndStatus = + endPoint2ClientAndStatus.get(endPointList.get(nextClientIndex)); + if (Boolean.TRUE.equals(nextClientAndStatus.getRight())) { + return nextClientAndStatus; + } + } + + throw new PipeConnectionException( + "All clients are dead, please check the connection to the receiver."); + } + } + + private class PriorityLoadBalancer implements LoadBalancer { + @Override + public Pair getClient() { + // Priority, find the first alive client + for (final TEndPoint endPoint : endPointList) { + final Pair clientAndStatus = + endPoint2ClientAndStatus.get(endPoint); + if (Boolean.TRUE.equals(clientAndStatus.getRight())) { + return clientAndStatus; + } + } + + throw new PipeConnectionException( + "All clients are dead, please check the connection to the receiver."); + } + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClient.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClient.java new file mode 100644 index 00000000..20061efa --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClient.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.client; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.ConnectException; +import java.net.SocketException; +import java.util.Optional; + +/** + * This class defines the failed interfaces that thrift client needs to support so that the Thrift + * Client can clean up the clientManager when it receives the corresponding exception. + */ +public interface ThriftClient { + + Logger logger = LoggerFactory.getLogger(ThriftClient.class); + + /** Close this connection. */ + void invalidate(); + + /** Removing all pooled instances corresponding to current instance's endpoint. */ + void invalidateAll(); + + /** + * Whether to print logs when exceptions are encountered. + * + * @return result + */ + boolean printLogWhenEncounterException(); + + /** + * Perform corresponding operations on ThriftClient o based on the Throwable t. + * + * @param t Throwable + * @param o ThriftClient + */ + static void resolveException(Throwable t, ThriftClient o) { + Throwable origin = t; + if (t instanceof InvocationTargetException) { + origin = ((InvocationTargetException) t).getTargetException(); + } + Throwable cur = origin; + if (cur instanceof TException) { + int level = 0; + while (cur != null) { + logger.debug( + "level-{} Exception class {}, message {}", + level, + cur.getClass().getName(), + cur.getMessage()); + cur = cur.getCause(); + level++; + } + o.invalidate(); + } + + Throwable rootCause = ExceptionUtils.getRootCause(origin); + if (rootCause != null) { + // if the exception is SocketException and its error message is Broken pipe, it means that + // the remote node may restart and all the connection we cached before should be cleared. + logger.debug( + "root cause message {}, LocalizedMessage {}, ", + rootCause.getMessage(), + rootCause.getLocalizedMessage(), + rootCause); + if (isConnectionBroken(rootCause)) { + if (o.printLogWhenEncounterException()) { + logger.info( + "Broken pipe error happened in sending RPC," + + " we need to clear all previous cached connection, error msg is {}", + rootCause.toString()); + } + o.invalidateAll(); + } + } + } + + /** + * Determine whether the target node has gone offline once based on the cause. + * + * @param cause Throwable + * @return true/false + */ + static boolean isConnectionBroken(Throwable cause) { + return (cause instanceof SocketException && cause.getMessage().contains("Broken pipe")) + || (cause instanceof TTransportException + && (hasExpectedMessage(cause, "Socket is closed by peer") + || hasExpectedMessage(cause, "Read call frame size failed"))) + || (cause instanceof IOException + && (hasExpectedMessage(cause, "Connection reset by peer") + || hasExpectedMessage(cause, "Broken pipe"))) + || (cause instanceof ConnectException && hasExpectedMessage(cause, "Connection refused")); + } + + static boolean hasExpectedMessage(Throwable cause, String expectedMessage) { + return Optional.ofNullable(cause.getMessage()) + .map(m -> m.contains(expectedMessage)) + .orElse(false); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClientProperty.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClientProperty.java new file mode 100644 index 00000000..b85c2739 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClientProperty.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.client; + +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TProtocolFactory; + +import java.util.concurrent.TimeUnit; + +/** This class defines the configurations commonly used by the Thrift Client. */ +public class ThriftClientProperty { + + private final TProtocolFactory protocolFactory; + private final int connectionTimeoutMs; + private final int selectorNumOfAsyncClientPool; + private final boolean printLogWhenEncounterException; + + private ThriftClientProperty( + TProtocolFactory protocolFactory, + int connectionTimeoutMs, + int selectorNumOfAsyncClientPool, + boolean printLogWhenEncounterException) { + this.protocolFactory = protocolFactory; + this.connectionTimeoutMs = connectionTimeoutMs; + this.selectorNumOfAsyncClientPool = selectorNumOfAsyncClientPool; + this.printLogWhenEncounterException = printLogWhenEncounterException; + } + + public TProtocolFactory getProtocolFactory() { + return protocolFactory; + } + + public int getConnectionTimeoutMs() { + return connectionTimeoutMs; + } + + public int getSelectorNumOfAsyncClientPool() { + return selectorNumOfAsyncClientPool; + } + + public boolean isPrintLogWhenEncounterException() { + return printLogWhenEncounterException; + } + + public static class Builder { + + /** whether to use thrift compression. */ + private boolean rpcThriftCompressionEnabled = DefaultProperty.RPC_THRIFT_COMPRESSED_ENABLED; + + /** socket timeout for thrift client. */ + private int connectionTimeoutMs = DefaultProperty.CONNECTION_TIMEOUT_MS; + + /** number of selector threads for asynchronous thrift client in a clientManager. */ + private int selectorNumOfAsyncClientManager = + DefaultProperty.SELECTOR_NUM_OF_ASYNC_CLIENT_MANAGER; + + /** + * Whether to print logs when the client encounters exceptions. For example, logs are not + * printed in the heartbeat client. + */ + private boolean printLogWhenEncounterException = + DefaultProperty.PRINT_LOG_WHEN_ENCOUNTER_EXCEPTION; + + public Builder setRpcThriftCompressionEnabled(boolean rpcThriftCompressionEnabled) { + this.rpcThriftCompressionEnabled = rpcThriftCompressionEnabled; + return this; + } + + public Builder setConnectionTimeoutMs(int connectionTimeoutMs) { + this.connectionTimeoutMs = connectionTimeoutMs; + return this; + } + + public Builder setSelectorNumOfAsyncClientManager(int selectorNumOfAsyncClientManager) { + this.selectorNumOfAsyncClientManager = selectorNumOfAsyncClientManager; + return this; + } + + public Builder setPrintLogWhenEncounterException(boolean printLogWhenEncounterException) { + this.printLogWhenEncounterException = printLogWhenEncounterException; + return this; + } + + public ThriftClientProperty build() { + return new ThriftClientProperty( + rpcThriftCompressionEnabled + ? new TCompactProtocol.Factory() + : new TBinaryProtocol.Factory(), + connectionTimeoutMs, + selectorNumOfAsyncClientManager, + printLogWhenEncounterException); + } + } + + public static class DefaultProperty { + + private DefaultProperty() {} + + public static final boolean RPC_THRIFT_COMPRESSED_ENABLED = false; + public static final int CONNECTION_TIMEOUT_MS = (int) TimeUnit.SECONDS.toMillis(20); + public static final int CONNECTION_NEVER_TIMEOUT_MS = 0; + public static final int SELECTOR_NUM_OF_ASYNC_CLIENT_MANAGER = 1; + public static final boolean PRINT_LOG_WHEN_ENCOUNTER_EXCEPTION = true; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressor.java new file mode 100644 index 00000000..dc3f1e79 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressor.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.compressor; + +import java.io.IOException; + +public abstract class PipeCompressor { + + public enum PipeCompressionType { + SNAPPY((byte) 0), + GZIP((byte) 1), + LZ4((byte) 2), + ZSTD((byte) 3), + LZMA2((byte) 4); + + final byte index; + + PipeCompressionType(byte index) { + this.index = index; + } + + public byte getIndex() { + return index; + } + } + + private final PipeCompressionType compressionType; + + protected PipeCompressor(PipeCompressionType compressionType) { + this.compressionType = compressionType; + } + + public abstract byte[] compress(byte[] data) throws IOException; + + /** + * Decompress the byte array to a byte array. NOTE: the length of the decompressed byte array is + * not provided in this method, and some decompressors (LZ4) may construct large byte arrays, + * leading to potential OOM. + * + * @param byteArray the byte array to be decompressed + * @return the decompressed byte array + * @throws IOException + */ + public abstract byte[] decompress(byte[] byteArray) throws IOException; + + /** + * Decompress the byte array to a byte array with a known length. + * + * @param byteArray the byte array to be decompressed + * @param decompressedLength the length of the decompressed byte array + * @return the decompressed byte array + * @throws IOException + */ + public abstract byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException; + + public byte serialize() { + return compressionType.getIndex(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorConfig.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorConfig.java new file mode 100644 index 00000000..fc54f3af --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorConfig.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.compressor; + +public class PipeCompressorConfig { + + private final String name; + private final int zstdCompressionLevel; + + public PipeCompressorConfig(String name, int zstdCompressionLevel) { + this.name = name; + this.zstdCompressionLevel = zstdCompressionLevel; + } + + public String getName() { + return name; + } + + public int getZstdCompressionLevel() { + return zstdCompressionLevel; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorFactory.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorFactory.java new file mode 100644 index 00000000..e2da42f1 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorFactory.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.compressor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_GZIP; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_LZ4; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_LZMA2; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_SNAPPY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE; + +public class PipeCompressorFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeCompressorFactory.class); + + private static final Map COMPRESSOR_NAME_TO_INSTANCE = + new ConcurrentHashMap<>(); + + static { + COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_SNAPPY, new PipeSnappyCompressor()); + COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_GZIP, new PipeGZIPCompressor()); + COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_LZ4, new PipeLZ4Compressor()); + COMPRESSOR_NAME_TO_INSTANCE.put( + CONNECTOR_COMPRESSOR_ZSTD, + new PipeZSTDCompressor(CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE)); + COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_LZMA2, new PipeLZMA2Compressor()); + } + + public static PipeCompressor getCompressor(PipeCompressorConfig config) { + if (config == null) { + throw new IllegalArgumentException("PipeCompressorConfig is null"); + } + if (config.getName() == null) { + throw new IllegalArgumentException("PipeCompressorConfig.getName() is null"); + } + + final String compressorName = config.getName(); + + // For ZSTD compressor, we need to consider the compression level + if (compressorName.equals(CONNECTOR_COMPRESSOR_ZSTD)) { + final int zstdCompressionLevel = config.getZstdCompressionLevel(); + return COMPRESSOR_NAME_TO_INSTANCE.computeIfAbsent( + CONNECTOR_COMPRESSOR_ZSTD + "_" + zstdCompressionLevel, + key -> { + LOGGER.info("Create new PipeZSTDCompressor with level: {}", zstdCompressionLevel); + return new PipeZSTDCompressor(zstdCompressionLevel); + }); + } + + // For other compressors, we can directly get the instance by name + final PipeCompressor compressor = COMPRESSOR_NAME_TO_INSTANCE.get(compressorName); + if (compressor != null) { + return compressor; + } + + throw new UnsupportedOperationException("PipeCompressor not found for name: " + compressorName); + } + + private static Map COMPRESSOR_INDEX_TO_INSTANCE = new HashMap<>(); + + static { + COMPRESSOR_INDEX_TO_INSTANCE.put( + PipeCompressor.PipeCompressionType.SNAPPY.getIndex(), + COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_SNAPPY)); + COMPRESSOR_INDEX_TO_INSTANCE.put( + PipeCompressor.PipeCompressionType.GZIP.getIndex(), + COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_GZIP)); + COMPRESSOR_INDEX_TO_INSTANCE.put( + PipeCompressor.PipeCompressionType.LZ4.getIndex(), + COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_LZ4)); + COMPRESSOR_INDEX_TO_INSTANCE.put( + PipeCompressor.PipeCompressionType.ZSTD.getIndex(), + COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_ZSTD)); + COMPRESSOR_INDEX_TO_INSTANCE.put( + PipeCompressor.PipeCompressionType.LZMA2.getIndex(), + COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_LZMA2)); + COMPRESSOR_INDEX_TO_INSTANCE = Collections.unmodifiableMap(COMPRESSOR_INDEX_TO_INSTANCE); + } + + public static PipeCompressor getCompressor(byte index) { + final PipeCompressor compressor = COMPRESSOR_INDEX_TO_INSTANCE.get(index); + if (compressor == null) { + throw new UnsupportedOperationException("PipeCompressor not found for index: " + index); + } + return compressor; + } + + private PipeCompressorFactory() { + // Empty constructor + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeGZIPCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeGZIPCompressor.java new file mode 100644 index 00000000..38c844e0 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeGZIPCompressor.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.compressor; + +import org.apache.tsfile.compress.ICompressor; +import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.file.metadata.enums.CompressionType; + +import java.io.IOException; + +public class PipeGZIPCompressor extends PipeCompressor { + + private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.GZIP); + private static final IUnCompressor DECOMPRESSOR = + IUnCompressor.getUnCompressor(CompressionType.GZIP); + + public PipeGZIPCompressor() { + super(PipeCompressionType.GZIP); + } + + @Override + public byte[] compress(byte[] data) throws IOException { + return COMPRESSOR.compress(data); + } + + @Override + public byte[] decompress(byte[] byteArray) throws IOException { + return DECOMPRESSOR.uncompress(byteArray); + } + + @Override + public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException { + byte[] uncompressed = new byte[decompressedLength]; + DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0); + return uncompressed; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZ4Compressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZ4Compressor.java new file mode 100644 index 00000000..9f441d8a --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZ4Compressor.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.compressor; + +import org.apache.tsfile.compress.ICompressor; +import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.file.metadata.enums.CompressionType; + +import java.io.IOException; + +public class PipeLZ4Compressor extends PipeCompressor { + + private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.LZ4); + private static final IUnCompressor DECOMPRESSOR = + IUnCompressor.getUnCompressor(CompressionType.LZ4); + + public PipeLZ4Compressor() { + super(PipeCompressionType.LZ4); + } + + @Override + public byte[] compress(byte[] data) throws IOException { + return COMPRESSOR.compress(data); + } + + @Override + public byte[] decompress(byte[] byteArray) throws IOException { + return DECOMPRESSOR.uncompress(byteArray); + } + + @Override + public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException { + byte[] uncompressed = new byte[decompressedLength]; + DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0); + return uncompressed; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZMA2Compressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZMA2Compressor.java new file mode 100644 index 00000000..69105796 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZMA2Compressor.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.compressor; + +import org.apache.tsfile.compress.ICompressor; +import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.file.metadata.enums.CompressionType; + +import java.io.IOException; + +public class PipeLZMA2Compressor extends PipeCompressor { + + private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.LZMA2); + private static final IUnCompressor DECOMPRESSOR = + IUnCompressor.getUnCompressor(CompressionType.LZMA2); + + public PipeLZMA2Compressor() { + super(PipeCompressionType.LZMA2); + } + + @Override + public byte[] compress(byte[] data) throws IOException { + return COMPRESSOR.compress(data); + } + + @Override + public byte[] decompress(byte[] byteArray) throws IOException { + return DECOMPRESSOR.uncompress(byteArray); + } + + @Override + public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException { + byte[] uncompressed = new byte[decompressedLength]; + DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0); + return uncompressed; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeSnappyCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeSnappyCompressor.java new file mode 100644 index 00000000..996f95dd --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeSnappyCompressor.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.compressor; + +import org.apache.tsfile.compress.ICompressor; +import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.file.metadata.enums.CompressionType; + +import java.io.IOException; + +public class PipeSnappyCompressor extends PipeCompressor { + + private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.SNAPPY); + private static final IUnCompressor DECOMPRESSOR = + IUnCompressor.getUnCompressor(CompressionType.SNAPPY); + + public PipeSnappyCompressor() { + super(PipeCompressionType.SNAPPY); + } + + @Override + public byte[] compress(byte[] data) throws IOException { + return COMPRESSOR.compress(data); + } + + @Override + public byte[] decompress(byte[] byteArray) throws IOException { + return DECOMPRESSOR.uncompress(byteArray); + } + + @Override + public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException { + byte[] uncompressed = new byte[decompressedLength]; + DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0); + return uncompressed; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeZSTDCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeZSTDCompressor.java new file mode 100644 index 00000000..67e40ac1 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeZSTDCompressor.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.compressor; + +import com.github.luben.zstd.Zstd; + +import java.io.IOException; + +public class PipeZSTDCompressor extends PipeCompressor { + + private final int compressionLevel; + + public PipeZSTDCompressor(int compressionLevel) { + super(PipeCompressionType.ZSTD); + this.compressionLevel = compressionLevel; + } + + @Override + public byte[] compress(byte[] data) throws IOException { + return Zstd.compress(data, compressionLevel); + } + + @Override + public byte[] decompress(byte[] byteArray) { + return Zstd.decompress(byteArray, (int) Zstd.decompressedSize(byteArray, 0, byteArray.length)); + } + + @Override + public byte[] decompress(byte[] byteArray, int decompressedLength) { + return Zstd.decompress(byteArray, decompressedLength); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/ColumnHeaderConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/ColumnHeaderConstant.java new file mode 100644 index 00000000..5d4db675 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/ColumnHeaderConstant.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.constant; + +public class ColumnHeaderConstant { + + private ColumnHeaderConstant() { + // forbidding instantiation + } + + public static final String DATABASE = "Database"; + + public static final String PATH_PATTERN = "PathPattern"; + + public static final String TYPE = "Type"; + + public static final String TABLE_NAME = "TableName"; +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeConnectorConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeConnectorConstant.java new file mode 100644 index 00000000..314d167f --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeConnectorConstant.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.constant; + +import org.apache.iotdb.collector.config.PipeOptions; + +import com.github.luben.zstd.Zstd; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static io.moquette.BrokerConstants.MB; + +public class PipeConnectorConstant { + + public static final String CONNECTOR_KEY = "connector"; + public static final String SINK_KEY = "sink"; + + public static final String CONNECTOR_IOTDB_IP_KEY = "connector.ip"; + public static final String SINK_IOTDB_IP_KEY = "sink.ip"; + public static final String CONNECTOR_IOTDB_HOST_KEY = "connector.host"; + public static final String SINK_IOTDB_HOST_KEY = "sink.host"; + public static final String CONNECTOR_IOTDB_PORT_KEY = "connector.port"; + public static final String SINK_IOTDB_PORT_KEY = "sink.port"; + public static final String CONNECTOR_IOTDB_NODE_URLS_KEY = "connector.node-urls"; + public static final String SINK_IOTDB_NODE_URLS_KEY = "sink.node-urls"; + + public static final String SINK_IOTDB_SSL_ENABLE_KEY = "sink.ssl.enable"; + public static final String SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY = "sink.ssl.trust-store-path"; + public static final String SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY = "sink.ssl.trust-store-pwd"; + + public static final String CONNECTOR_IOTDB_PARALLEL_TASKS_KEY = "connector.parallel.tasks"; + public static final String SINK_IOTDB_PARALLEL_TASKS_KEY = "sink.parallel.tasks"; + public static final int CONNECTOR_IOTDB_PARALLEL_TASKS_DEFAULT_VALUE = + PipeOptions.PIPE_SUBTASK_EXECUTOR_MAX_THREAD_NUM.value(); + + public static final String CONNECTOR_REALTIME_FIRST_KEY = "connector.realtime-first"; + public static final String SINK_REALTIME_FIRST_KEY = "sink.realtime-first"; + public static final boolean CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE = true; + + public static final String CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY = "connector.batch.enable"; + public static final String SINK_IOTDB_BATCH_MODE_ENABLE_KEY = "sink.batch.enable"; + public static final boolean CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE = true; + + public static final String CONNECTOR_IOTDB_BATCH_DELAY_KEY = "connector.batch.max-delay-seconds"; + public static final String SINK_IOTDB_BATCH_DELAY_KEY = "sink.batch.max-delay-seconds"; + public static final int CONNECTOR_IOTDB_PLAIN_BATCH_DELAY_DEFAULT_VALUE = 1; + public static final int CONNECTOR_IOTDB_TS_FILE_BATCH_DELAY_DEFAULT_VALUE = 5; + + public static final String CONNECTOR_IOTDB_BATCH_SIZE_KEY = "connector.batch.size-bytes"; + public static final String SINK_IOTDB_BATCH_SIZE_KEY = "sink.batch.size-bytes"; + public static final long CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE = 16 * MB; + public static final long CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE = 80 * MB; + + public static final String CONNECTOR_IOTDB_USER_KEY = "connector.user"; + public static final String SINK_IOTDB_USER_KEY = "sink.user"; + public static final String CONNECTOR_IOTDB_USERNAME_KEY = "connector.username"; + public static final String SINK_IOTDB_USERNAME_KEY = "sink.username"; + public static final String CONNECTOR_IOTDB_USER_DEFAULT_VALUE = "root"; + + public static final String CONNECTOR_IOTDB_PASSWORD_KEY = "connector.password"; + public static final String SINK_IOTDB_PASSWORD_KEY = "sink.password"; + public static final String CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE = "root"; + + public static final String CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY = + "connector.exception.data.convert-on-type-mismatch"; + public static final String SINK_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY = + "sink.exception.data.convert-on-type-mismatch"; + public static final boolean CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE = + true; + + public static final String CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY = + "connector.exception.conflict.resolve-strategy"; + public static final String SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY = + "sink.exception.conflict.resolve-strategy"; + public static final String CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE = "retry"; + + public static final String CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_KEY = + "connector.exception.conflict.retry-max-time-seconds"; + public static final String SINK_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_KEY = + "sink.exception.conflict.retry-max-time-seconds"; + public static final long CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE = 60; + + public static final String CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_KEY = + "connector.exception.conflict.record-ignored-data"; + public static final String SINK_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_KEY = + "sink.exception.conflict.record-ignored-data"; + public static final boolean CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_DEFAULT_VALUE = true; + + public static final String CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_KEY = + "connector.exception.others.retry-max-time-seconds"; + public static final String SINK_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_KEY = + "sink.exception.others.retry-max-time-seconds"; + public static final long CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE = -1; + + public static final String CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_KEY = + "connector.exception.others.record-ignored-data"; + public static final String SINK_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_KEY = + "sink.exception.others.record-ignored-data"; + public static final boolean CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_DEFAULT_VALUE = true; + + public static final String CONNECTOR_AIR_GAP_E_LANGUAGE_ENABLE_KEY = + "connector.air-gap.e-language.enable"; + public static final String SINK_AIR_GAP_E_LANGUAGE_ENABLE_KEY = "sink.air-gap.e-language.enable"; + public static final boolean CONNECTOR_AIR_GAP_E_LANGUAGE_ENABLE_DEFAULT_VALUE = false; + + public static final String CONNECTOR_AIR_GAP_HANDSHAKE_TIMEOUT_MS_KEY = + "connector.air-gap.handshake-timeout-ms"; + public static final String SINK_AIR_GAP_HANDSHAKE_TIMEOUT_MS_KEY = + "sink.air-gap.handshake-timeout-ms"; + public static final int CONNECTOR_AIR_GAP_HANDSHAKE_TIMEOUT_MS_DEFAULT_VALUE = 5000; + + public static final String CONNECTOR_IOTDB_SYNC_CONNECTOR_VERSION_KEY = "connector.version"; + public static final String SINK_IOTDB_SYNC_CONNECTOR_VERSION_KEY = "sink.version"; + public static final String CONNECTOR_IOTDB_SYNC_CONNECTOR_VERSION_DEFAULT_VALUE = "1.1"; + + public static final String CONNECTOR_WEBSOCKET_PORT_KEY = "connector.websocket.port"; + public static final String SINK_WEBSOCKET_PORT_KEY = "sink.websocket.port"; + public static final int CONNECTOR_WEBSOCKET_PORT_DEFAULT_VALUE = 8080; + + public static final String CONNECTOR_OPC_UA_MODEL_KEY = "connector.opcua.model"; + public static final String SINK_OPC_UA_MODEL_KEY = "sink.opcua.model"; + public static final String CONNECTOR_OPC_UA_MODEL_CLIENT_SERVER_VALUE = "client-server"; + public static final String CONNECTOR_OPC_UA_MODEL_PUB_SUB_VALUE = "pub-sub"; + public static final String CONNECTOR_OPC_UA_MODEL_DEFAULT_VALUE = + CONNECTOR_OPC_UA_MODEL_CLIENT_SERVER_VALUE; + + public static final String CONNECTOR_OPC_UA_TCP_BIND_PORT_KEY = "connector.opcua.tcp.port"; + public static final String SINK_OPC_UA_TCP_BIND_PORT_KEY = "sink.opcua.tcp.port"; + public static final int CONNECTOR_OPC_UA_TCP_BIND_PORT_DEFAULT_VALUE = 12686; + + public static final String CONNECTOR_OPC_UA_HTTPS_BIND_PORT_KEY = "connector.opcua.https.port"; + public static final String SINK_OPC_UA_HTTPS_BIND_PORT_KEY = "sink.opcua.https.port"; + public static final int CONNECTOR_OPC_UA_HTTPS_BIND_PORT_DEFAULT_VALUE = 8443; + + public static final String CONNECTOR_OPC_UA_ENABLE_ANONYMOUS_ACCESS_KEY = + "connector.opcua.enable-anonymous-access"; + public static final String SINK_OPC_UA_ENABLE_ANONYMOUS_ACCESS_KEY = + "sink.opcua.enable-anonymous-access"; + public static final boolean CONNECTOR_OPC_UA_ENABLE_ANONYMOUS_ACCESS_DEFAULT_VALUE = true; + + public static final String CONNECTOR_OPC_UA_PLACEHOLDER_KEY = "connector.opcua.placeholder"; + public static final String SINK_OPC_UA_PLACEHOLDER_KEY = "sink.opcua.placeholder"; + public static final String CONNECTOR_OPC_UA_PLACEHOLDER_DEFAULT_VALUE = "null"; + + public static final String CONNECTOR_LEADER_CACHE_ENABLE_KEY = "connector.leader-cache.enable"; + public static final String SINK_LEADER_CACHE_ENABLE_KEY = "sink.leader-cache.enable"; + public static final boolean CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE = true; + + public static final String CONNECTOR_LOAD_BALANCE_STRATEGY_KEY = + "connector.load-balance-strategy"; + public static final String SINK_LOAD_BALANCE_STRATEGY_KEY = "sink.load-balance-strategy"; + public static final String CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY = "round-robin"; + public static final String CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY = "random"; + public static final String CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY = "priority"; + public static final Set CONNECTOR_LOAD_BALANCE_STRATEGY_SET = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY, + CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY, + CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY))); + + public static final String CONNECTOR_COMPRESSOR_KEY = "connector.compressor"; + public static final String SINK_COMPRESSOR_KEY = "sink.compressor"; + public static final String CONNECTOR_COMPRESSOR_DEFAULT_VALUE = ""; + public static final String CONNECTOR_COMPRESSOR_SNAPPY = "snappy"; + public static final String CONNECTOR_COMPRESSOR_GZIP = "gzip"; + public static final String CONNECTOR_COMPRESSOR_LZ4 = "lz4"; + public static final String CONNECTOR_COMPRESSOR_ZSTD = "zstd"; + public static final String CONNECTOR_COMPRESSOR_LZMA2 = "lzma2"; + public static final Set CONNECTOR_COMPRESSOR_SET = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + CONNECTOR_COMPRESSOR_SNAPPY, + CONNECTOR_COMPRESSOR_GZIP, + CONNECTOR_COMPRESSOR_LZ4, + CONNECTOR_COMPRESSOR_ZSTD, + CONNECTOR_COMPRESSOR_LZMA2))); + + public static final String CONNECTOR_COMPRESSOR_ZSTD_LEVEL_KEY = + "connector.compressor.zstd.level"; + public static final String SINK_COMPRESSOR_ZSTD_LEVEL_KEY = "sink.compressor.zstd.level"; + public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE = + Zstd.defaultCompressionLevel(); + public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE = Zstd.minCompressionLevel(); + public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE = Zstd.maxCompressionLevel(); + + public static final String CONNECTOR_RATE_LIMIT_KEY = "connector.rate-limit-bytes-per-second"; + public static final String SINK_RATE_LIMIT_KEY = "sink.rate-limit-bytes-per-second"; + public static final double CONNECTOR_RATE_LIMIT_DEFAULT_VALUE = -1; + + public static final String CONNECTOR_FORMAT_KEY = "connector.format"; + public static final String SINK_FORMAT_KEY = "sink.format"; + public static final String CONNECTOR_FORMAT_TABLET_VALUE = "tablet"; + public static final String CONNECTOR_FORMAT_TS_FILE_VALUE = "tsfile"; + public static final String CONNECTOR_FORMAT_HYBRID_VALUE = "hybrid"; + + public static final String SINK_TOPIC_KEY = "sink.topic"; + public static final String SINK_CONSUMER_GROUP_KEY = "sink.consumer-group"; + + public static final String CONNECTOR_CONSENSUS_GROUP_ID_KEY = "connector.consensus.group-id"; + public static final String CONNECTOR_CONSENSUS_PIPE_NAME = "connector.consensus.pipe-name"; + + public static final String CONNECTOR_LOAD_TSFILE_STRATEGY_KEY = "connector.load-tsfile-strategy"; + public static final String SINK_LOAD_TSFILE_STRATEGY_KEY = "sink.load-tsfile-strategy"; + public static final String CONNECTOR_LOAD_TSFILE_STRATEGY_ASYNC_VALUE = "async"; + public static final String CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE = "sync"; + public static final Set CONNECTOR_LOAD_TSFILE_STRATEGY_SET = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + CONNECTOR_LOAD_TSFILE_STRATEGY_ASYNC_VALUE, + CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE))); + + public static final String CONNECTOR_LOAD_TSFILE_VALIDATION_KEY = + "connector.load-tsfile-validation"; + public static final String SINK_LOAD_TSFILE_VALIDATION_KEY = "sink.load-tsfile-validation"; + public static final boolean CONNECTOR_LOAD_TSFILE_VALIDATION_DEFAULT_VALUE = true; + + public static final String CONNECTOR_MARK_AS_PIPE_REQUEST_KEY = "connector.mark-as-pipe-request"; + public static final String SINK_MARK_AS_PIPE_REQUEST_KEY = "sink.mark-as-pipe-request"; + public static final boolean CONNECTOR_MARK_AS_PIPE_REQUEST_DEFAULT_VALUE = true; + + public static final String CONNECTOR_OPC_DA_CLSID_KEY = "connector.opcda.clsid"; + public static final String SINK_OPC_DA_CLSID_KEY = "sink.opcda.clsid"; + + public static final String CONNECTOR_OPC_DA_PROGID_KEY = "connector.opcda.progid"; + public static final String SINK_OPC_DA_PROGID_KEY = "sink.opcda.progid"; + + private PipeConnectorConstant() { + throw new IllegalStateException("Utility class"); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeTransferHandshakeConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeTransferHandshakeConstant.java new file mode 100644 index 00000000..1adcfabb --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeTransferHandshakeConstant.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.constant; + +public class PipeTransferHandshakeConstant { + + public static final String HANDSHAKE_KEY_TIME_PRECISION = "timestampPrecision"; + public static final String HANDSHAKE_KEY_CLUSTER_ID = "clusterID"; + public static final String HANDSHAKE_KEY_CONVERT_ON_TYPE_MISMATCH = "convertOnTypeMismatch"; + public static final String HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY = "loadTsFileStrategy"; + public static final String HANDSHAKE_KEY_USERNAME = "username"; + public static final String HANDSHAKE_KEY_PASSWORD = "password"; + public static final String HANDSHAKE_KEY_VALIDATE_TSFILE = "validateTsFile"; + public static final String HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST = "markAsPipeRequest"; + + private PipeTransferHandshakeConstant() { + // Utility class + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeInsertionEvent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeInsertionEvent.java new file mode 100644 index 00000000..1027ece7 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeInsertionEvent.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.event; + +public abstract class PipeInsertionEvent { + + protected Boolean isTableModelEvent; + + protected String treeModelDatabaseName; + protected String tableModelDatabaseName; + + public boolean isTableModelEvent() { + return isTableModelEvent; + } + + public String getTreeModelDatabaseName() { + return treeModelDatabaseName; + } + + public String getTableModelDatabaseName() { + return tableModelDatabaseName; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeRawTabletInsertionEvent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeRawTabletInsertionEvent.java new file mode 100644 index 00000000..3ea3452d --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeRawTabletInsertionEvent.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.event; + +import org.apache.iotdb.pipe.api.access.Row; +import org.apache.iotdb.pipe.api.collector.RowCollector; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; + +import org.apache.tsfile.write.record.Tablet; + +import java.util.function.BiConsumer; + +public class PipeRawTabletInsertionEvent extends PipeInsertionEvent + implements TabletInsertionEvent, AutoCloseable { + + private Tablet tablet; + private boolean isAligned; + + @Override + public Iterable processRowByRow( + final BiConsumer consumer) { + throw new UnsupportedOperationException(); + } + + @Override + public Iterable processTablet( + final BiConsumer consumer) { + throw new UnsupportedOperationException(); + } + + public Tablet getTablet() { + return tablet; + } + + public boolean isAligned() { + return isAligned; + } + + public boolean isTableModelEvent() { + return false; + } + + public String getDeviceId() { + return null; + } + + public String getTableModelDatabaseName() { + return null; + } + + @Override + public void close() {} +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeTsFileInsertionEvent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeTsFileInsertionEvent.java new file mode 100644 index 00000000..2fc5e9a1 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeTsFileInsertionEvent.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.event; + +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; +import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; +import org.apache.iotdb.pipe.api.exception.PipeException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +public class PipeTsFileInsertionEvent extends PipeInsertionEvent implements TsFileInsertionEvent { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileInsertionEvent.class); + + private File tsFile; + + @Override + public boolean isTableModelEvent() { + throw new PipeException(""); + } + + public File getTsFile() { + return tsFile; + } + + @Override + public Iterable toTabletInsertionEvents() throws PipeException { + return toTabletInsertionEvents(Long.MAX_VALUE); + } + + public Iterable toTabletInsertionEvents(final long timeoutMs) + throws PipeException { + return null; + } + + @Override + public void close() throws Exception {} +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventBatch.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventBatch.java new file mode 100644 index 00000000..859969e2 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventBatch.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch; + +import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeRawTabletInsertionEvent; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public abstract class PipeTabletEventBatch implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventBatch.class); + + protected final List events = new ArrayList<>(); + + private final int maxDelayInMs; + private long firstEventProcessingTime = Long.MIN_VALUE; + + protected long totalBufferSize = 0; + // private final PipeMemoryBlock allocatedMemoryBlock; + + protected volatile boolean isClosed = false; + + protected PipeTabletEventBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) { + this.maxDelayInMs = maxDelayInMs; + + // limit in buffer size + // this.allocatedMemoryBlock = + // PipeDataNodeResourceManager.memory() + // .tryAllocate(requestMaxBatchSizeInBytes) + // .setShrinkMethod(oldMemory -> Math.max(oldMemory / 2, 0)) + // .setShrinkCallback( + // (oldMemory, newMemory) -> + // LOGGER.info( + // "The batch size limit has shrunk from {} to {}.", oldMemory, newMemory)) + // .setExpandMethod( + // oldMemory -> Math.min(Math.max(oldMemory, 1) * 2, requestMaxBatchSizeInBytes)) + // .setExpandCallback( + // (oldMemory, newMemory) -> + // LOGGER.info( + // "The batch size limit has expanded from {} to {}.", oldMemory, + // newMemory)); + + if (getMaxBatchSizeInBytes() != requestMaxBatchSizeInBytes) { + LOGGER.info( + "PipeTabletEventBatch: the max batch size is adjusted from {} to {} due to the " + + "memory restriction", + requestMaxBatchSizeInBytes, + getMaxBatchSizeInBytes()); + } + } + + /** + * Try offer {@link Event} into batch if the given {@link Event} is not duplicated. + * + * @param event the given {@link Event} + * @return {@code true} if the batch can be transferred + */ + public synchronized boolean onEvent(final TabletInsertionEvent event) throws IOException { + if (isClosed || !(event instanceof PipeRawTabletInsertionEvent)) { + return false; + } + + // The deduplication logic here is to avoid the accumulation of + // the same event in a batch when retrying. + if (events.isEmpty() || !Objects.equals(events.get(events.size() - 1), event)) { + if (constructBatch(event)) { + events.add((PipeRawTabletInsertionEvent) event); + } + + if (firstEventProcessingTime == Long.MIN_VALUE) { + firstEventProcessingTime = System.currentTimeMillis(); + } + } + + return shouldEmit(); + } + + /** + * Added an {@link TabletInsertionEvent} into batch. + * + * @param event the {@link TabletInsertionEvent} in batch + * @return {@code true} if the event is calculated into batch, {@code false} if the event is + * cached and not emitted in this batch. If there are failure encountered, just throw + * exceptions and do not return {@code false} here. + */ + protected abstract boolean constructBatch(final TabletInsertionEvent event) throws IOException; + + public boolean shouldEmit() { + return totalBufferSize >= getMaxBatchSizeInBytes() + || System.currentTimeMillis() - firstEventProcessingTime >= maxDelayInMs; + } + + private long getMaxBatchSizeInBytes() { + // return allocatedMemoryBlock.getMemoryUsageInBytes(); + return 0; + } + + public synchronized void onSuccess() { + events.clear(); + + totalBufferSize = 0; + + firstEventProcessingTime = Long.MIN_VALUE; + } + + @Override + public synchronized void close() { + isClosed = true; + + events.clear(); + } + + public List deepCopyEvents() { + return new ArrayList<>(events); + } + + public boolean isEmpty() { + return events.isEmpty(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java new file mode 100644 index 00000000..8525f571 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch; + +import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeRawTabletInsertionEvent; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request.PipeTransferTabletBatchReqV2; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; + +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PipeTabletEventPlainBatch extends PipeTabletEventBatch { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventPlainBatch.class); + + private final List binaryBuffers = new ArrayList<>(); + private final List insertNodeBuffers = new ArrayList<>(); + private final List tabletBuffers = new ArrayList<>(); + + private static final String TREE_MODEL_DATABASE_PLACEHOLDER = null; + private final List binaryDataBases = new ArrayList<>(); + private final List insertNodeDataBases = new ArrayList<>(); + private final List tabletDataBases = new ArrayList<>(); + + // Used to rate limit when transferring data + private final Map, Long> pipe2BytesAccumulated = new HashMap<>(); + + PipeTabletEventPlainBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) { + super(maxDelayInMs, requestMaxBatchSizeInBytes); + } + + @Override + protected boolean constructBatch(final TabletInsertionEvent event) throws IOException { + final int bufferSize = buildTabletInsertionBuffer(event); + totalBufferSize += bufferSize; + pipe2BytesAccumulated.compute( + new Pair<>( + ((PipeRawTabletInsertionEvent) event).getPipeName(), + ((PipeRawTabletInsertionEvent) event).getCreationTime()), + (pipeName, bytesAccumulated) -> + bytesAccumulated == null ? bufferSize : bytesAccumulated + bufferSize); + return true; + } + + @Override + public synchronized void onSuccess() { + super.onSuccess(); + + binaryBuffers.clear(); + insertNodeBuffers.clear(); + tabletBuffers.clear(); + + binaryDataBases.clear(); + insertNodeDataBases.clear(); + tabletDataBases.clear(); + + pipe2BytesAccumulated.clear(); + } + + public PipeTransferTabletBatchReqV2 toTPipeTransferReq() throws IOException { + return PipeTransferTabletBatchReqV2.toTPipeTransferReq( + binaryBuffers, + insertNodeBuffers, + tabletBuffers, + binaryDataBases, + insertNodeDataBases, + tabletDataBases); + } + + public Map, Long> deepCopyPipeName2BytesAccumulated() { + return new HashMap<>(pipe2BytesAccumulated); + } + + public Map, Long> getPipe2BytesAccumulated() { + return pipe2BytesAccumulated; + } + + private int buildTabletInsertionBuffer(final TabletInsertionEvent event) throws IOException { + int databaseEstimateSize = 0; + final ByteBuffer buffer; + // if (event instanceof PipeInsertNodeTabletInsertionEvent) { + // final PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletInsertionEvent = + // (PipeInsertNodeTabletInsertionEvent) event; + // // Read the bytebuffer from the wal file and transfer it directly without serializing or + // // deserializing if possible + // final InsertNode insertNode = + // pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible(); + // if (Objects.isNull(insertNode)) { + // buffer = pipeInsertNodeTabletInsertionEvent.getByteBuffer(); + // binaryBuffers.add(buffer); + // if (pipeInsertNodeTabletInsertionEvent.isTableModelEvent()) { + // databaseEstimateSize = + // pipeInsertNodeTabletInsertionEvent.getTableModelDatabaseName().length(); + // binaryDataBases.add(pipeInsertNodeTabletInsertionEvent.getTableModelDatabaseName()); + // } else { + // databaseEstimateSize = 4; + // binaryDataBases.add(TREE_MODEL_DATABASE_PLACEHOLDER); + // } + // } else { + // buffer = insertNode.serializeToByteBuffer(); + // insertNodeBuffers.add(buffer); + // if (pipeInsertNodeTabletInsertionEvent.isTableModelEvent()) { + // databaseEstimateSize = + // pipeInsertNodeTabletInsertionEvent.getTableModelDatabaseName().length(); + // + // insertNodeDataBases.add(pipeInsertNodeTabletInsertionEvent.getTableModelDatabaseName()); + // } else { + // databaseEstimateSize = 4; + // insertNodeDataBases.add(TREE_MODEL_DATABASE_PLACEHOLDER); + // } + // } + // } else + { + final PipeRawTabletInsertionEvent pipeRawTabletInsertionEvent = + (PipeRawTabletInsertionEvent) event; + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + pipeRawTabletInsertionEvent.convertToTablet().serialize(outputStream); + ReadWriteIOUtils.write(pipeRawTabletInsertionEvent.isAligned(), outputStream); + buffer = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + tabletBuffers.add(buffer); + if (pipeRawTabletInsertionEvent.isTableModelEvent()) { + databaseEstimateSize = pipeRawTabletInsertionEvent.getTableModelDatabaseName().length(); + tabletDataBases.add(pipeRawTabletInsertionEvent.getTableModelDatabaseName()); + } else { + databaseEstimateSize = 4; + tabletDataBases.add(TREE_MODEL_DATABASE_PLACEHOLDER); + } + } + return buffer.limit() + databaseEstimateSize; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java new file mode 100644 index 00000000..903008b3 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch; + +import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeRawTabletInsertionEvent; +import org.apache.iotdb.collector.plugin.builtin.sink.utils.builder.PipeTableModeTsFileBuilder; +import org.apache.iotdb.collector.plugin.builtin.sink.utils.builder.PipeTreeModelTsFileBuilder; +import org.apache.iotdb.collector.plugin.builtin.sink.utils.builder.PipeTsFileBuilder; +import org.apache.iotdb.collector.plugin.builtin.sink.utils.sorter.PipeTableModelTabletEventSorter; +import org.apache.iotdb.collector.plugin.builtin.sink.utils.sorter.PipeTreeModelTabletEventSorter; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; + +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +public class PipeTabletEventTsFileBatch extends PipeTabletEventBatch { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventTsFileBatch.class); + + private static final AtomicLong BATCH_ID_GENERATOR = new AtomicLong(0); + private final AtomicLong currentBatchId = new AtomicLong(BATCH_ID_GENERATOR.incrementAndGet()); + + private final PipeTsFileBuilder treeModeTsFileBuilder; + private final PipeTsFileBuilder tableModeTsFileBuilder; + + private final Map, Double> pipeName2WeightMap = new HashMap<>(); + + public PipeTabletEventTsFileBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) { + super(maxDelayInMs, requestMaxBatchSizeInBytes); + + final AtomicLong tsFileIdGenerator = new AtomicLong(0); + treeModeTsFileBuilder = new PipeTreeModelTsFileBuilder(currentBatchId, tsFileIdGenerator); + tableModeTsFileBuilder = new PipeTableModeTsFileBuilder(currentBatchId, tsFileIdGenerator); + } + + @Override + protected boolean constructBatch(final TabletInsertionEvent event) { + // if (event instanceof PipeInsertNodeTabletInsertionEvent) { + // final PipeInsertNodeTabletInsertionEvent insertNodeTabletInsertionEvent = + // (PipeInsertNodeTabletInsertionEvent) event; + // final boolean isTableModel = insertNodeTabletInsertionEvent.isTableModelEvent(); + // final List tablets = insertNodeTabletInsertionEvent.convertToTablets(); + // for (int i = 0; i < tablets.size(); ++i) { + // final Tablet tablet = tablets.get(i); + // if (tablet.getRowSize() == 0) { + // continue; + // } + // if (isTableModel) { + // // table Model + // bufferTableModelTablet( + // insertNodeTabletInsertionEvent.getPipeName(), + // insertNodeTabletInsertionEvent.getCreationTime(), + // tablet, + // insertNodeTabletInsertionEvent.getTableModelDatabaseName()); + // } else { + // // tree Model + // bufferTreeModelTablet( + // insertNodeTabletInsertionEvent.getPipeName(), + // insertNodeTabletInsertionEvent.getCreationTime(), + // tablet, + // insertNodeTabletInsertionEvent.isAligned(i)); + // } + // } + // } else + if (event instanceof PipeRawTabletInsertionEvent) { + final PipeRawTabletInsertionEvent rawTabletInsertionEvent = + (PipeRawTabletInsertionEvent) event; + final Tablet tablet = rawTabletInsertionEvent.convertToTablet(); + if (tablet.getRowSize() == 0) { + return true; + } + if (rawTabletInsertionEvent.isTableModelEvent()) { + // table Model + bufferTableModelTablet( + rawTabletInsertionEvent.getPipeName(), + rawTabletInsertionEvent.getCreationTime(), + tablet, + rawTabletInsertionEvent.getTableModelDatabaseName()); + } else { + // tree Model + bufferTreeModelTablet( + rawTabletInsertionEvent.getPipeName(), + rawTabletInsertionEvent.getCreationTime(), + tablet, + rawTabletInsertionEvent.isAligned()); + } + } else { + LOGGER.warn( + "Batch id = {}: Unsupported event {} type {} when constructing tsfile batch", + currentBatchId.get(), + event, + event.getClass()); + } + return true; + } + + private void bufferTreeModelTablet( + final String pipeName, + final long creationTime, + final Tablet tablet, + final boolean isAligned) { + new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + + totalBufferSize += PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet); + + pipeName2WeightMap.compute( + new Pair<>(pipeName, creationTime), + (pipe, weight) -> Objects.nonNull(weight) ? ++weight : 1); + + treeModeTsFileBuilder.bufferTreeModelTablet(tablet, isAligned); + } + + private void bufferTableModelTablet( + final String pipeName, final long creationTime, final Tablet tablet, final String dataBase) { + new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByDevIdTimestamp(); + + totalBufferSize += PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet); + + pipeName2WeightMap.compute( + new Pair<>(pipeName, creationTime), + (pipe, weight) -> Objects.nonNull(weight) ? ++weight : 1); + + tableModeTsFileBuilder.bufferTableModelTablet(dataBase, tablet); + } + + public Map, Double> deepCopyPipe2WeightMap() { + final double sum = pipeName2WeightMap.values().stream().reduce(Double::sum).orElse(0.0); + if (sum == 0.0) { + return Collections.emptyMap(); + } + pipeName2WeightMap.entrySet().forEach(entry -> entry.setValue(entry.getValue() / sum)); + return new HashMap<>(pipeName2WeightMap); + } + + /** + * Converts a Tablet to a TSFile and returns the generated TSFile along with its corresponding + * database name. + * + * @return a list of pairs containing the database name and the generated TSFile + * @throws IOException if an I/O error occurs during the conversion process + * @throws WriteProcessException if an error occurs during the write process + */ + public synchronized List> sealTsFiles() + throws IOException, WriteProcessException { + if (isClosed) { + return Collections.emptyList(); + } + + final List> list = new ArrayList<>(); + if (!treeModeTsFileBuilder.isEmpty()) { + list.addAll(treeModeTsFileBuilder.convertTabletToTsFileWithDBInfo()); + } + if (!tableModeTsFileBuilder.isEmpty()) { + list.addAll(tableModeTsFileBuilder.convertTabletToTsFileWithDBInfo()); + } + return list; + } + + @Override + public synchronized void onSuccess() { + super.onSuccess(); + + pipeName2WeightMap.clear(); + tableModeTsFileBuilder.onSuccess(); + treeModeTsFileBuilder.onSuccess(); + } + + @Override + public synchronized void close() { + super.close(); + + pipeName2WeightMap.clear(); + + tableModeTsFileBuilder.close(); + treeModeTsFileBuilder.close(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java new file mode 100644 index 00000000..4bc87897 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch; + +import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeRawTabletInsertionEvent; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; + +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_DELAY_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_SIZE_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_DELAY_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_TS_FILE_BATCH_DELAY_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_FORMAT_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_DELAY_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_SIZE_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LEADER_CACHE_ENABLE_KEY; + +public class PipeTransferBatchReqBuilder implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTransferBatchReqBuilder.class); + + private final boolean useLeaderCache; + + private final int requestMaxDelayInMs; + private final long requestMaxBatchSizeInBytes; + + // If the leader cache is disabled (or unable to find the endpoint of event in the leader cache), + // the event will be stored in the default batch. + private final PipeTabletEventBatch defaultBatch; + // If the leader cache is enabled, the batch will be divided by the leader endpoint, + // each endpoint has a batch. + // This is only used in plain batch since tsfile does not return redirection info. + private final Map endPointToBatch = new HashMap<>(); + + public PipeTransferBatchReqBuilder(final PipeParameters parameters) { + final boolean usingTsFileBatch = + parameters + .getStringOrDefault( + Arrays.asList(CONNECTOR_FORMAT_KEY, SINK_FORMAT_KEY), CONNECTOR_FORMAT_HYBRID_VALUE) + .equals(CONNECTOR_FORMAT_TS_FILE_VALUE); + + useLeaderCache = + !usingTsFileBatch + && parameters.getBooleanOrDefault( + Arrays.asList(SINK_LEADER_CACHE_ENABLE_KEY, CONNECTOR_LEADER_CACHE_ENABLE_KEY), + CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE); + + final int requestMaxDelayInSeconds; + if (usingTsFileBatch) { + requestMaxDelayInSeconds = + parameters.getIntOrDefault( + Arrays.asList(CONNECTOR_IOTDB_BATCH_DELAY_KEY, SINK_IOTDB_BATCH_DELAY_KEY), + CONNECTOR_IOTDB_TS_FILE_BATCH_DELAY_DEFAULT_VALUE); + requestMaxDelayInMs = + requestMaxDelayInSeconds < 0 ? Integer.MAX_VALUE : requestMaxDelayInSeconds * 1000; + requestMaxBatchSizeInBytes = + parameters.getLongOrDefault( + Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY), + CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE); + this.defaultBatch = + new PipeTabletEventTsFileBatch(requestMaxDelayInMs, requestMaxBatchSizeInBytes); + } else { + requestMaxDelayInSeconds = + parameters.getIntOrDefault( + Arrays.asList(CONNECTOR_IOTDB_BATCH_DELAY_KEY, SINK_IOTDB_BATCH_DELAY_KEY), + CONNECTOR_IOTDB_PLAIN_BATCH_DELAY_DEFAULT_VALUE); + requestMaxDelayInMs = + requestMaxDelayInSeconds < 0 ? Integer.MAX_VALUE : requestMaxDelayInSeconds * 1000; + requestMaxBatchSizeInBytes = + parameters.getLongOrDefault( + Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY), + CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE); + this.defaultBatch = + new PipeTabletEventPlainBatch(requestMaxDelayInMs, requestMaxBatchSizeInBytes); + } + } + + /** + * Try offer {@link Event} into the corresponding batch if the given {@link Event} is not + * duplicated. + * + * @param event the given {@link Event} + * @return {@link Pair}<{@link TEndPoint}, {@link PipeTabletEventPlainBatch}> not null means this + * {@link PipeTabletEventPlainBatch} can be transferred. the first element is the leader + * endpoint to transfer to (might be null), the second element is the batch to be transferred. + */ + public synchronized Pair onEvent( + final TabletInsertionEvent event) throws IOException { + if (!(event instanceof PipeRawTabletInsertionEvent)) { + LOGGER.warn( + "Unsupported event {} type {} when building transfer request", event, event.getClass()); + return null; + } + + if (!useLeaderCache) { + return defaultBatch.onEvent(event) ? new Pair<>(null, defaultBatch) : null; + } + + String deviceId = null; + if (event instanceof PipeRawTabletInsertionEvent) { + deviceId = ((PipeRawTabletInsertionEvent) event).getDeviceId(); + } + + if (Objects.isNull(deviceId)) { + return defaultBatch.onEvent(event) ? new Pair<>(null, defaultBatch) : null; + } + + final TEndPoint endPoint = null; + // IoTDBDataNodeCacheLeaderClientManager.LEADER_CACHE_MANAGER.getLeaderEndPoint(deviceId); + if (Objects.isNull(endPoint)) { + return defaultBatch.onEvent(event) ? new Pair<>(null, defaultBatch) : null; + } + + final PipeTabletEventPlainBatch batch = + endPointToBatch.computeIfAbsent( + endPoint, + k -> new PipeTabletEventPlainBatch(requestMaxDelayInMs, requestMaxBatchSizeInBytes)); + return batch.onEvent(event) ? new Pair<>(endPoint, batch) : null; + } + + /** Get all batches that have at least 1 event. */ + public synchronized List> getAllNonEmptyBatches() { + final List> nonEmptyBatches = new ArrayList<>(); + if (!defaultBatch.isEmpty()) { + nonEmptyBatches.add(new Pair<>(null, defaultBatch)); + } + endPointToBatch.forEach( + (endPoint, batch) -> { + if (!batch.isEmpty()) { + nonEmptyBatches.add(new Pair<>(endPoint, batch)); + } + }); + return nonEmptyBatches; + } + + public boolean isEmpty() { + return defaultBatch.isEmpty() + && endPointToBatch.values().stream().allMatch(PipeTabletEventPlainBatch::isEmpty); + } + + @Override + public synchronized void close() { + defaultBatch.close(); + endPointToBatch.values().forEach(PipeTabletEventPlainBatch::close); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV1Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV1Req.java new file mode 100644 index 00000000..f7e3cb59 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV1Req.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferHandshakeV1Req; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.io.IOException; + +public class PipeTransferDataNodeHandshakeV1Req extends PipeTransferHandshakeV1Req { + + private PipeTransferDataNodeHandshakeV1Req() { + // Empty constructor + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.HANDSHAKE_DATANODE_V1; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferDataNodeHandshakeV1Req toTPipeTransferReq( + final String timestampPrecision) throws IOException { + return (PipeTransferDataNodeHandshakeV1Req) + new PipeTransferDataNodeHandshakeV1Req().convertToTPipeTransferReq(timestampPrecision); + } + + public static PipeTransferDataNodeHandshakeV1Req fromTPipeTransferReq( + final TPipeTransferReq transferReq) { + return (PipeTransferDataNodeHandshakeV1Req) + new PipeTransferDataNodeHandshakeV1Req().translateFromTPipeTransferReq(transferReq); + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes(final String timestampPrecision) throws IOException { + return new PipeTransferDataNodeHandshakeV1Req() + .convertToTransferHandshakeBytes(timestampPrecision); + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(final Object obj) { + return obj instanceof PipeTransferDataNodeHandshakeV1Req && super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.java new file mode 100644 index 00000000..7e334465 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferHandshakeV2Req; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.io.IOException; +import java.util.Map; + +public class PipeTransferDataNodeHandshakeV2Req extends PipeTransferHandshakeV2Req { + + private PipeTransferDataNodeHandshakeV2Req() { + // Empty constructor + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.HANDSHAKE_DATANODE_V2; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferDataNodeHandshakeV2Req toTPipeTransferReq(Map params) + throws IOException { + return (PipeTransferDataNodeHandshakeV2Req) + new PipeTransferDataNodeHandshakeV2Req().convertToTPipeTransferReq(params); + } + + public static PipeTransferDataNodeHandshakeV2Req fromTPipeTransferReq( + TPipeTransferReq transferReq) { + return (PipeTransferDataNodeHandshakeV2Req) + new PipeTransferDataNodeHandshakeV2Req().translateFromTPipeTransferReq(transferReq); + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes(Map params) throws IOException { + return new PipeTransferDataNodeHandshakeV2Req().convertToTransferHandshakeBytes(params); + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object obj) { + return obj instanceof PipeTransferDataNodeHandshakeV2Req && super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotPieceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotPieceReq.java new file mode 100644 index 00000000..6ba01556 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotPieceReq.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFilePieceReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.io.IOException; + +public class PipeTransferSchemaSnapshotPieceReq extends PipeTransferFilePieceReq { + + private PipeTransferSchemaSnapshotPieceReq() { + // Empty constructor + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferSchemaSnapshotPieceReq toTPipeTransferReq( + String fileName, long startWritingOffset, byte[] filePiece) throws IOException { + return (PipeTransferSchemaSnapshotPieceReq) + new PipeTransferSchemaSnapshotPieceReq() + .convertToTPipeTransferReq(fileName, startWritingOffset, filePiece); + } + + public static PipeTransferSchemaSnapshotPieceReq fromTPipeTransferReq( + TPipeTransferReq transferReq) { + return (PipeTransferSchemaSnapshotPieceReq) + new PipeTransferSchemaSnapshotPieceReq().translateFromTPipeTransferReq(transferReq); + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes( + String fileName, long startWritingOffset, byte[] filePiece) throws IOException { + return new PipeTransferSchemaSnapshotPieceReq() + .convertToTPipeTransferBytes(fileName, startWritingOffset, filePiece); + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object obj) { + return obj instanceof PipeTransferSchemaSnapshotPieceReq && super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotSealReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotSealReq.java new file mode 100644 index 00000000..4e961c71 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferSchemaSnapshotSealReq.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.constant.ColumnHeaderConstant; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFileSealReqV2; +import org.apache.iotdb.collector.plugin.builtin.sink.protocol.session.IClientSession; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class PipeTransferSchemaSnapshotSealReq extends PipeTransferFileSealReqV2 { + + private PipeTransferSchemaSnapshotSealReq() { + // Empty constructor + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferSchemaSnapshotSealReq toTPipeTransferReq( + final String treePattern, + final String tablePatternDatabase, + final String tablePatternTable, + final boolean isTreeCaptured, + final boolean isTableCaptured, + final String mTreeSnapshotName, + final long mTreeSnapshotLength, + final String tLogName, + final long tLogLength, + final String attributeSnapshotName, + final long attributeSnapshotLength, + final String databaseName, + final String typeString) + throws IOException { + final Map parameters = new HashMap<>(); + parameters.put(ColumnHeaderConstant.PATH_PATTERN, treePattern); + parameters.put(DATABASE_PATTERN, tablePatternDatabase); + parameters.put(ColumnHeaderConstant.TABLE_NAME, tablePatternTable); + if (isTreeCaptured) { + parameters.put(IClientSession.SqlDialect.TREE.toString(), ""); + } + if (isTableCaptured) { + parameters.put(IClientSession.SqlDialect.TABLE.toString(), ""); + } + parameters.put(ColumnHeaderConstant.DATABASE, databaseName); + parameters.put(ColumnHeaderConstant.TYPE, typeString); + + final List fileNameList; + final List fileLengthList; + + // Tree model sync + if (Objects.isNull(attributeSnapshotName)) { + fileNameList = + Objects.nonNull(tLogName) + ? Arrays.asList(mTreeSnapshotName, tLogName) + : Collections.singletonList(mTreeSnapshotName); + fileLengthList = + Objects.nonNull(tLogName) + ? Arrays.asList(mTreeSnapshotLength, tLogLength) + : Collections.singletonList(mTreeSnapshotLength); + } else { + fileNameList = Arrays.asList(mTreeSnapshotName, tLogName, attributeSnapshotName); + fileLengthList = Arrays.asList(mTreeSnapshotLength, tLogLength, attributeSnapshotLength); + } + + return (PipeTransferSchemaSnapshotSealReq) + new PipeTransferSchemaSnapshotSealReq() + .convertToTPipeTransferReq(fileNameList, fileLengthList, parameters); + } + + public static PipeTransferSchemaSnapshotSealReq fromTPipeTransferReq(final TPipeTransferReq req) { + return (PipeTransferSchemaSnapshotSealReq) + new PipeTransferSchemaSnapshotSealReq().translateFromTPipeTransferReq(req); + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes( + final String treePattern, + final String tablePatternDatabase, + final String tablePatternTable, + final boolean isTreeCaptured, + final boolean isTableCaptured, + final String mTreeSnapshotName, + final long mTreeSnapshotLength, + final String tLogName, + final long tLogLength, + final String attributeSnapshotName, + final long attributeSnapshotLength, + final String databaseName, + final String typeString) + throws IOException { + final Map parameters = new HashMap<>(); + parameters.put(ColumnHeaderConstant.PATH_PATTERN, treePattern); + parameters.put(DATABASE_PATTERN, tablePatternDatabase); + parameters.put(ColumnHeaderConstant.TABLE_NAME, tablePatternTable); + if (isTreeCaptured) { + parameters.put(TREE, ""); + } + if (isTableCaptured) { + parameters.put(TABLE, ""); + } + parameters.put(ColumnHeaderConstant.DATABASE, databaseName); + parameters.put(ColumnHeaderConstant.TYPE, typeString); + + final List fileNameList; + final List fileLengthList; + + // Tree model sync + if (Objects.isNull(attributeSnapshotName)) { + fileNameList = + Objects.nonNull(tLogName) + ? Arrays.asList(mTreeSnapshotName, tLogName) + : Collections.singletonList(mTreeSnapshotName); + fileLengthList = + Objects.nonNull(tLogName) + ? Arrays.asList(mTreeSnapshotLength, tLogLength) + : Collections.singletonList(mTreeSnapshotLength); + } else { + fileNameList = Arrays.asList(mTreeSnapshotName, tLogName, attributeSnapshotName); + fileLengthList = Arrays.asList(mTreeSnapshotLength, tLogLength, attributeSnapshotLength); + } + + return new PipeTransferSchemaSnapshotSealReq() + .convertToTPipeTransferSnapshotSealBytes(fileNameList, fileLengthList, parameters); + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(final Object obj) { + return obj instanceof PipeTransferSchemaSnapshotSealReq && super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReq.java new file mode 100644 index 00000000..7dac01ee --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReq.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class PipeTransferTabletBatchReq extends TPipeTransferReq { + + private final transient List binaryReqs = new ArrayList<>(); + // private final transient List insertNodeReqs = new + // ArrayList<>(); + private final transient List tabletReqs = new ArrayList<>(); + + private PipeTransferTabletBatchReq() { + // Empty constructor + } + + // public Pair constructStatements() { + // final InsertRowsStatement insertRowsStatement = new InsertRowsStatement(); + // final InsertMultiTabletsStatement insertMultiTabletsStatement = + // new InsertMultiTabletsStatement(); + // + // final List insertRowStatementList = new ArrayList<>(); + // final List insertTabletStatementList = new ArrayList<>(); + // + // for (final PipeTransferTabletBinaryReq binaryReq : binaryReqs) { + // final InsertBaseStatement statement = binaryReq.constructStatement(); + // if (statement.isEmpty()) { + // continue; + // } + // if (statement instanceof InsertRowStatement) { + // insertRowStatementList.add((InsertRowStatement) statement); + // } else if (statement instanceof InsertTabletStatement) { + // insertTabletStatementList.add((InsertTabletStatement) statement); + // } else if (statement instanceof InsertRowsStatement) { + // insertRowStatementList.addAll( + // ((InsertRowsStatement) statement).getInsertRowStatementList()); + // } else { + // throw new UnsupportedOperationException( + // String.format( + // "unknown InsertBaseStatement %s constructed from PipeTransferTabletBinaryReq.", + // binaryReq)); + // } + // } + // + // for (final PipeTransferTabletInsertNodeReq insertNodeReq : insertNodeReqs) { + // final InsertBaseStatement statement = insertNodeReq.constructStatement(); + // if (statement.isEmpty()) { + // continue; + // } + // if (statement instanceof InsertRowStatement) { + // insertRowStatementList.add((InsertRowStatement) statement); + // } else if (statement instanceof InsertTabletStatement) { + // insertTabletStatementList.add((InsertTabletStatement) statement); + // } else if (statement instanceof InsertRowsStatement) { + // insertRowStatementList.addAll( + // ((InsertRowsStatement) statement).getInsertRowStatementList()); + // } else { + // throw new UnsupportedOperationException( + // String.format( + // "Unknown InsertBaseStatement %s constructed from + // PipeTransferTabletInsertNodeReq.", + // statement)); + // } + // } + // + // for (final PipeTransferTabletRawReq tabletReq : tabletReqs) { + // final InsertTabletStatement statement = tabletReq.constructStatement(); + // if (statement.isEmpty()) { + // continue; + // } + // insertTabletStatementList.add(statement); + // } + // + // insertRowsStatement.setInsertRowStatementList(insertRowStatementList); + // insertMultiTabletsStatement.setInsertTabletStatementList(insertTabletStatementList); + // return new Pair<>(insertRowsStatement, insertMultiTabletsStatement); + // } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferTabletBatchReq toTPipeTransferReq( + final List binaryBuffers, + final List insertNodeBuffers, + final List tabletBuffers) + throws IOException { + final PipeTransferTabletBatchReq batchReq = new PipeTransferTabletBatchReq(); + + // batchReq.binaryReqs, batchReq.insertNodeReqs, batchReq.tabletReqs are empty + // when this method is called from PipeTransferTabletBatchReqBuilder.toTPipeTransferReq() + + batchReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + batchReq.type = PipeRequestType.TRANSFER_TABLET_BATCH.getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(binaryBuffers.size(), outputStream); + for (final ByteBuffer binaryBuffer : binaryBuffers) { + ReadWriteIOUtils.write(binaryBuffer.limit(), outputStream); + outputStream.write(binaryBuffer.array(), 0, binaryBuffer.limit()); + } + + ReadWriteIOUtils.write(insertNodeBuffers.size(), outputStream); + for (final ByteBuffer insertNodeBuffer : insertNodeBuffers) { + outputStream.write(insertNodeBuffer.array(), 0, insertNodeBuffer.limit()); + } + + ReadWriteIOUtils.write(tabletBuffers.size(), outputStream); + for (final ByteBuffer tabletBuffer : tabletBuffers) { + outputStream.write(tabletBuffer.array(), 0, tabletBuffer.limit()); + } + + batchReq.body = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return batchReq; + } + + // public static PipeTransferTabletBatchReq fromTPipeTransferReq( + // final TPipeTransferReq transferReq) { + // final PipeTransferTabletBatchReq batchReq = new PipeTransferTabletBatchReq(); + // + // int size = ReadWriteIOUtils.readInt(transferReq.body); + // for (int i = 0; i < size; ++i) { + // final int length = ReadWriteIOUtils.readInt(transferReq.body); + // final byte[] body = new byte[length]; + // transferReq.body.get(body); + // batchReq.binaryReqs.add( + // PipeTransferTabletBinaryReq.toTPipeTransferReq(ByteBuffer.wrap(body))); + // } + // + // size = ReadWriteIOUtils.readInt(transferReq.body); + // for (int i = 0; i < size; ++i) { + // batchReq.insertNodeReqs.add( + // PipeTransferTabletInsertNodeReq.toTPipeTransferRawReq( + // (InsertNode) PlanFragment.deserializeHelper(transferReq.body, null))); + // } + // + // size = ReadWriteIOUtils.readInt(transferReq.body); + // for (int i = 0; i < size; ++i) { + // batchReq.tabletReqs.add( + // PipeTransferTabletRawReq.toTPipeTransferRawReq( + // Tablet.deserialize(transferReq.body), + // ReadWriteIOUtils.readBool(transferReq.body))); + // } + // + // batchReq.version = transferReq.version; + // batchReq.type = transferReq.type; + // batchReq.body = transferReq.body; + // + // return batchReq; + // } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final PipeTransferTabletBatchReq that = (PipeTransferTabletBatchReq) obj; + return binaryReqs.equals(that.binaryReqs) + // && insertNodeReqs.equals(that.insertNodeReqs) + && tabletReqs.equals(that.tabletReqs) + && version == that.version + && type == that.type + && body.equals(that.body); + } + + @Override + public int hashCode() { + return Objects.hash(binaryReqs, /*insertNodeReqs,*/ tabletReqs, version, type, body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java new file mode 100644 index 00000000..36dd874a --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class PipeTransferTabletBatchReqV2 extends TPipeTransferReq { + + private final transient List binaryReqs = new ArrayList<>(); + private final transient List insertNodeReqs = + new ArrayList<>(); + private final transient List tabletReqs = new ArrayList<>(); + + private PipeTransferTabletBatchReqV2() { + // Empty constructor + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferTabletBatchReqV2 toTPipeTransferReq( + final List binaryBuffers, + final List insertNodeBuffers, + final List tabletBuffers, + final List binaryDataBases, + final List insertNodeDataBases, + final List tabletDataBases) + throws IOException { + final PipeTransferTabletBatchReqV2 batchReq = new PipeTransferTabletBatchReqV2(); + + batchReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + batchReq.type = PipeRequestType.TRANSFER_TABLET_BATCH_V2.getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(binaryBuffers.size(), outputStream); + for (int i = 0; i < binaryBuffers.size(); i++) { + final ByteBuffer binaryBuffer = binaryBuffers.get(i); + ReadWriteIOUtils.write(binaryBuffer.limit(), outputStream); + outputStream.write(binaryBuffer.array(), 0, binaryBuffer.limit()); + ReadWriteIOUtils.write(binaryDataBases.get(i), outputStream); + } + + ReadWriteIOUtils.write(insertNodeBuffers.size(), outputStream); + for (int i = 0; i < insertNodeBuffers.size(); i++) { + final ByteBuffer insertNodeBuffer = insertNodeBuffers.get(i); + outputStream.write(insertNodeBuffer.array(), 0, insertNodeBuffer.limit()); + ReadWriteIOUtils.write(insertNodeDataBases.get(i), outputStream); + } + + ReadWriteIOUtils.write(tabletBuffers.size(), outputStream); + for (int i = 0; i < tabletBuffers.size(); i++) { + final ByteBuffer tabletBuffer = tabletBuffers.get(i); + outputStream.write(tabletBuffer.array(), 0, tabletBuffer.limit()); + ReadWriteIOUtils.write(tabletDataBases.get(i), outputStream); + } + + batchReq.body = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return batchReq; + } + + // public static PipeTransferTabletBatchReqV2 fromTPipeTransferReq( + // final TPipeTransferReq transferReq) { + // final PipeTransferTabletBatchReqV2 batchReq = new PipeTransferTabletBatchReqV2(); + // + // int size = ReadWriteIOUtils.readInt(transferReq.body); + // for (int i = 0; i < size; ++i) { + // final int length = ReadWriteIOUtils.readInt(transferReq.body); + // final byte[] body = new byte[length]; + // transferReq.body.get(body); + // batchReq.binaryReqs.add( + // PipeTransferTabletBinaryReqV2.toTPipeTransferBinaryReq( + // ByteBuffer.wrap(body), ReadWriteIOUtils.readString(transferReq.body))); + // } + // + // size = ReadWriteIOUtils.readInt(transferReq.body); + // for (int i = 0; i < size; ++i) { + // batchReq.insertNodeReqs.add( + // PipeTransferTabletInsertNodeReqV2.toTabletInsertNodeReq( + // (InsertNode) PlanFragment.deserializeHelper(transferReq.body, null), + // ReadWriteIOUtils.readString(transferReq.body))); + // } + // + // size = ReadWriteIOUtils.readInt(transferReq.body); + // for (int i = 0; i < size; ++i) { + // batchReq.tabletReqs.add( + // PipeTransferTabletRawReqV2.toTPipeTransferRawReq( + // Tablet.deserialize(transferReq.body), + // ReadWriteIOUtils.readBool(transferReq.body), + // ReadWriteIOUtils.readString(transferReq.body))); + // } + // + // batchReq.version = transferReq.version; + // batchReq.type = transferReq.type; + // batchReq.body = transferReq.body; + // + // return batchReq; + // } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final PipeTransferTabletBatchReqV2 that = (PipeTransferTabletBatchReqV2) obj; + return Objects.equals(binaryReqs, that.binaryReqs) + && Objects.equals(tabletReqs, that.tabletReqs) + && version == that.version + && type == that.type + && Objects.equals(body, that.body); + } + + @Override + public int hashCode() { + return Objects.hash(binaryReqs, tabletReqs, version, type, body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java new file mode 100644 index 00000000..55bd26fb --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class PipeTransferTabletBinaryReq extends TPipeTransferReq { + + protected transient ByteBuffer byteBuffer; + + protected PipeTransferTabletBinaryReq() { + // Do nothing + } + + public ByteBuffer getByteBuffer() { + return byteBuffer; + } + + // public InsertBaseStatement constructStatement() { + // final InsertNode insertNode = parseByteBuffer(); + // + // if (!(insertNode instanceof InsertRowNode + // || insertNode instanceof InsertTabletNode + // || insertNode instanceof InsertRowsNode)) { + // throw new UnsupportedOperationException( + // String.format( + // "Unknown InsertNode type %s when constructing statement from insert node.", + // insertNode)); + // } + // + // return (InsertBaseStatement) + // IoTDBDataNodeReceiver.PLAN_TO_STATEMENT_VISITOR.process(insertNode, null); + // } + // + // protected InsertNode parseByteBuffer() { + // final PlanNode node = WALEntry.deserializeForConsensus(byteBuffer); + // return node instanceof InsertNode ? (InsertNode) node : null; + // } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferTabletBinaryReq toTPipeTransferReq(final ByteBuffer byteBuffer) { + final PipeTransferTabletBinaryReq req = new PipeTransferTabletBinaryReq(); + req.byteBuffer = byteBuffer; + + req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + req.type = PipeRequestType.TRANSFER_TABLET_BINARY.getType(); + req.body = byteBuffer; + + return req; + } + + public static PipeTransferTabletBinaryReq fromTPipeTransferReq( + final TPipeTransferReq transferReq) { + final PipeTransferTabletBinaryReq binaryReq = new PipeTransferTabletBinaryReq(); + binaryReq.byteBuffer = transferReq.body; + + binaryReq.version = transferReq.version; + binaryReq.type = transferReq.type; + binaryReq.body = transferReq.body; + + return binaryReq; + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes(final ByteBuffer byteBuffer) throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_BINARY.getType(), outputStream); + return BytesUtils.concatByteArray(byteArrayOutputStream.toByteArray(), byteBuffer.array()); + } + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final PipeTransferTabletBinaryReq that = (PipeTransferTabletBinaryReq) obj; + return byteBuffer.equals(that.byteBuffer) + && version == that.version + && type == that.type + && body.equals(that.body); + } + + @Override + public int hashCode() { + return Objects.hash(byteBuffer, version, type, body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java new file mode 100644 index 00000000..40a30fb8 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class PipeTransferTabletBinaryReqV2 extends PipeTransferTabletBinaryReq { + + private transient String dataBaseName; + + protected PipeTransferTabletBinaryReqV2() { + // Do nothing + } + + public String getDataBaseName() { + return dataBaseName; + } + + // public InsertBaseStatement constructStatement() { + // final InsertNode insertNode = parseByteBuffer(); + // + // if (!(insertNode instanceof InsertRowNode + // || insertNode instanceof InsertTabletNode + // || insertNode instanceof InsertRowsNode)) { + // throw new UnsupportedOperationException( + // String.format( + // "Unknown InsertNode type %s when constructing statement from insert node.", + // insertNode)); + // } + // + // final InsertBaseStatement statement = + // (InsertBaseStatement) + // IoTDBDataNodeReceiver.PLAN_TO_STATEMENT_VISITOR.process(insertNode, null); + // + // // Tree model + // if (Objects.isNull(dataBaseName)) { + // return statement; + // } + // + // // Table model + // statement.setWriteToTable(true); + // if (statement instanceof InsertRowsStatement) { + // List rowStatements = + // ((InsertRowsStatement) statement).getInsertRowStatementList(); + // if (rowStatements != null && !rowStatements.isEmpty()) { + // for (InsertRowStatement insertRowStatement : rowStatements) { + // insertRowStatement.setWriteToTable(true); + // insertRowStatement.setDatabaseName(dataBaseName); + // } + // } + // } + // statement.setDatabaseName(dataBaseName); + // return statement; + // } + + /////////////////////////////// Batch /////////////////////////////// + + public static PipeTransferTabletBinaryReqV2 toTPipeTransferBinaryReq( + final ByteBuffer byteBuffer, final String dataBaseName) { + final PipeTransferTabletBinaryReqV2 req = new PipeTransferTabletBinaryReqV2(); + + req.byteBuffer = byteBuffer; + req.dataBaseName = dataBaseName; + req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + req.type = PipeRequestType.TRANSFER_TABLET_BINARY_V2.getType(); + + return req; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferTabletBinaryReqV2 toTPipeTransferReq( + final ByteBuffer byteBuffer, final String dataBaseName) throws IOException { + final PipeTransferTabletBinaryReqV2 req = new PipeTransferTabletBinaryReqV2(); + req.byteBuffer = byteBuffer; + req.dataBaseName = dataBaseName; + + req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + req.type = PipeRequestType.TRANSFER_TABLET_BINARY_V2.getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(byteBuffer.limit(), outputStream); + outputStream.write(byteBuffer.array(), 0, byteBuffer.limit()); + ReadWriteIOUtils.write(dataBaseName, outputStream); + req.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return req; + } + + public static PipeTransferTabletBinaryReqV2 fromTPipeTransferReq( + final org.apache.iotdb.service.rpc.thrift.TPipeTransferReq transferReq) { + final PipeTransferTabletBinaryReqV2 binaryReq = new PipeTransferTabletBinaryReqV2(); + + final int length = ReadWriteIOUtils.readInt(transferReq.body); + final byte[] body = new byte[length]; + transferReq.body.get(body); + binaryReq.byteBuffer = ByteBuffer.wrap(body); + binaryReq.dataBaseName = ReadWriteIOUtils.readString(transferReq.body); + + binaryReq.version = transferReq.version; + binaryReq.type = transferReq.type; + binaryReq.body = transferReq.body; + + return binaryReq; + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes(final ByteBuffer byteBuffer, final String dataBaseName) + throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_BINARY_V2.getType(), outputStream); + ReadWriteIOUtils.write(byteBuffer.limit(), outputStream); + outputStream.write(byteBuffer.array(), 0, byteBuffer.limit()); + ReadWriteIOUtils.write(dataBaseName, outputStream); + return byteArrayOutputStream.toByteArray(); + } + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + final PipeTransferTabletBinaryReqV2 that = (PipeTransferTabletBinaryReqV2) o; + return Objects.equals(dataBaseName, that.dataBaseName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), dataBaseName); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReq.java new file mode 100644 index 00000000..e396ebdc --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReq.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.record.Tablet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class PipeTransferTabletRawReq extends TPipeTransferReq { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTransferTabletRawReq.class); + + protected transient Tablet tablet; + protected transient boolean isAligned; + + public Tablet getTablet() { + return tablet; + } + + public boolean getIsAligned() { + return isAligned; + } + + // public InsertTabletStatement constructStatement() { + // new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + // + // try { + // if (isTabletEmpty(tablet)) { + // // Empty statement, will be filtered after construction + // return new InsertTabletStatement(); + // } + // + // final TSInsertTabletReq request = new TSInsertTabletReq(); + // + // for (final IMeasurementSchema measurementSchema : tablet.getSchemas()) { + // request.addToMeasurements(measurementSchema.getMeasurementName()); + // request.addToTypes(measurementSchema.getType().ordinal()); + // } + // + // request.setPrefixPath(tablet.getDeviceId()); + // request.setIsAligned(isAligned); + // request.setTimestamps(SessionUtils.getTimeBuffer(tablet)); + // request.setValues(SessionUtils.getValueBuffer(tablet)); + // request.setSize(tablet.getRowSize()); + // request.setMeasurements( + // PathUtils.checkIsLegalSingleMeasurementsAndUpdate(request.getMeasurements())); + // + // return StatementGenerator.createStatement(request); + // } catch (final MetadataException e) { + // LOGGER.warn("Generate Statement from tablet {} error.", tablet, e); + // return null; + // } + // } + + /////////////////////////////// WriteBack & Batch /////////////////////////////// + + public static PipeTransferTabletRawReq toTPipeTransferRawReq( + final Tablet tablet, final boolean isAligned) { + final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq(); + + tabletReq.tablet = tablet; + tabletReq.isAligned = isAligned; + + return tabletReq; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferTabletRawReq toTPipeTransferReq( + final Tablet tablet, final boolean isAligned) throws IOException { + final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq(); + + tabletReq.tablet = tablet; + tabletReq.isAligned = isAligned; + + tabletReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + tabletReq.type = PipeRequestType.TRANSFER_TABLET_RAW.getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + tablet.serialize(outputStream); + ReadWriteIOUtils.write(isAligned, outputStream); + tabletReq.body = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return tabletReq; + } + + public static PipeTransferTabletRawReq fromTPipeTransferReq(final TPipeTransferReq transferReq) { + final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq(); + + tabletReq.tablet = Tablet.deserialize(transferReq.body); + tabletReq.isAligned = ReadWriteIOUtils.readBool(transferReq.body); + + tabletReq.version = transferReq.version; + tabletReq.type = transferReq.type; + tabletReq.body = transferReq.body; + + return tabletReq; + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes(final Tablet tablet, final boolean isAligned) + throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), outputStream); + tablet.serialize(outputStream); + ReadWriteIOUtils.write(isAligned, outputStream); + return byteArrayOutputStream.toByteArray(); + } + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final PipeTransferTabletRawReq that = (PipeTransferTabletRawReq) obj; + return Objects.equals(tablet, that.tablet) + && isAligned == that.isAligned + && version == that.version + && type == that.type + && Objects.equals(body, that.body); + } + + @Override + public int hashCode() { + return Objects.hash(tablet, isAligned, version, type, body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java new file mode 100644 index 00000000..abfdfe1b --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.record.Tablet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class PipeTransferTabletRawReqV2 extends PipeTransferTabletRawReq { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTransferTabletRawReqV2.class); + + protected transient String dataBaseName; + + public String getDataBaseName() { + return dataBaseName; + } + + // @Override + // public InsertTabletStatement constructStatement() { + // if (Objects.isNull(dataBaseName)) { + // new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + // } else { + // new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByTimestampIfNecessary(); + // } + // + // try { + // if (isTabletEmpty(tablet)) { + // // Empty statement, will be filtered after construction + // return new InsertTabletStatement(); + // } + // + // final TSInsertTabletReq request = new TSInsertTabletReq(); + // + // for (final IMeasurementSchema measurementSchema : tablet.getSchemas()) { + // request.addToMeasurements(measurementSchema.getMeasurementName()); + // request.addToTypes(measurementSchema.getType().ordinal()); + // } + // + // request.setPrefixPath(tablet.getDeviceId()); + // request.setIsAligned(isAligned); + // request.setTimestamps(SessionUtils.getTimeBuffer(tablet)); + // request.setValues(SessionUtils.getValueBuffer(tablet)); + // request.setSize(tablet.getRowSize()); + // + // // Tree model + // if (Objects.isNull(dataBaseName)) { + // request.setMeasurements( + // PathUtils.checkIsLegalSingleMeasurementsAndUpdate(request.getMeasurements())); + // return StatementGenerator.createStatement(request); + // } + // + // // Table model + // request.setWriteToTable(true); + // request.columnCategories = + // tablet.getColumnTypes().stream() + // .map(t -> (byte) t.ordinal()) + // .collect(Collectors.toList()); + // final InsertTabletStatement statement = StatementGenerator.createStatement(request); + // statement.setDatabaseName(dataBaseName); + // return statement; + // } catch (final MetadataException e) { + // LOGGER.warn("Generate Statement from tablet {} error.", tablet, e); + // return null; + // } + // } + + /////////////////////////////// WriteBack & Batch /////////////////////////////// + + public static PipeTransferTabletRawReqV2 toTPipeTransferRawReq( + final Tablet tablet, final boolean isAligned, final String dataBaseName) { + final PipeTransferTabletRawReqV2 tabletReq = new PipeTransferTabletRawReqV2(); + + tabletReq.tablet = tablet; + tabletReq.isAligned = isAligned; + tabletReq.dataBaseName = dataBaseName; + tabletReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + tabletReq.type = PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(); + + return tabletReq; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferTabletRawReqV2 toTPipeTransferReq( + final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException { + final PipeTransferTabletRawReqV2 tabletReq = new PipeTransferTabletRawReqV2(); + + tabletReq.tablet = tablet; + tabletReq.isAligned = isAligned; + tabletReq.dataBaseName = dataBaseName; + + tabletReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + tabletReq.type = PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + tablet.serialize(outputStream); + ReadWriteIOUtils.write(isAligned, outputStream); + ReadWriteIOUtils.write(dataBaseName, outputStream); + tabletReq.body = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return tabletReq; + } + + public static PipeTransferTabletRawReqV2 fromTPipeTransferReq( + final TPipeTransferReq transferReq) { + final PipeTransferTabletRawReqV2 tabletReq = new PipeTransferTabletRawReqV2(); + + tabletReq.tablet = Tablet.deserialize(transferReq.body); + tabletReq.isAligned = ReadWriteIOUtils.readBool(transferReq.body); + tabletReq.dataBaseName = ReadWriteIOUtils.readString(transferReq.body); + + tabletReq.version = transferReq.version; + tabletReq.type = transferReq.type; + tabletReq.body = transferReq.body; + + return tabletReq; + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes( + final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), outputStream); + tablet.serialize(outputStream); + ReadWriteIOUtils.write(isAligned, outputStream); + ReadWriteIOUtils.write(dataBaseName, outputStream); + return byteArrayOutputStream.toByteArray(); + } + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + final PipeTransferTabletRawReqV2 that = (PipeTransferTabletRawReqV2) o; + return Objects.equals(dataBaseName, that.dataBaseName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), dataBaseName); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceReq.java new file mode 100644 index 00000000..c83b3356 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceReq.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFilePieceReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.io.IOException; + +public class PipeTransferTsFilePieceReq extends PipeTransferFilePieceReq { + + private PipeTransferTsFilePieceReq() { + // Empty constructor + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_TS_FILE_PIECE; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferTsFilePieceReq toTPipeTransferReq( + String fileName, long startWritingOffset, byte[] filePiece) throws IOException { + return (PipeTransferTsFilePieceReq) + new PipeTransferTsFilePieceReq() + .convertToTPipeTransferReq(fileName, startWritingOffset, filePiece); + } + + public static PipeTransferTsFilePieceReq fromTPipeTransferReq(TPipeTransferReq transferReq) { + return (PipeTransferTsFilePieceReq) + new PipeTransferTsFilePieceReq().translateFromTPipeTransferReq(transferReq); + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes( + String fileName, long startWritingOffset, byte[] filePiece) throws IOException { + return new PipeTransferTsFilePieceReq() + .convertToTPipeTransferBytes(fileName, startWritingOffset, filePiece); + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object obj) { + return obj instanceof PipeTransferTsFilePieceReq && super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceWithModReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceWithModReq.java new file mode 100644 index 00000000..847848ac --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFilePieceWithModReq.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFilePieceReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.io.IOException; + +public class PipeTransferTsFilePieceWithModReq extends PipeTransferFilePieceReq { + + private PipeTransferTsFilePieceWithModReq() { + // Empty constructor + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_TS_FILE_PIECE_WITH_MOD; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferTsFilePieceWithModReq toTPipeTransferReq( + String fileName, long startWritingOffset, byte[] filePiece) throws IOException { + return (PipeTransferTsFilePieceWithModReq) + new PipeTransferTsFilePieceWithModReq() + .convertToTPipeTransferReq(fileName, startWritingOffset, filePiece); + } + + public static PipeTransferTsFilePieceWithModReq fromTPipeTransferReq( + TPipeTransferReq transferReq) { + return (PipeTransferTsFilePieceWithModReq) + new PipeTransferTsFilePieceWithModReq().translateFromTPipeTransferReq(transferReq); + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes( + String fileName, long startWritingOffset, byte[] filePiece) throws IOException { + return new PipeTransferTsFilePieceWithModReq() + .convertToTPipeTransferBytes(fileName, startWritingOffset, filePiece); + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object obj) { + return obj instanceof PipeTransferTsFilePieceWithModReq && super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealReq.java new file mode 100644 index 00000000..a6c85607 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealReq.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFileSealReqV1; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.io.IOException; + +public class PipeTransferTsFileSealReq extends PipeTransferFileSealReqV1 { + + private PipeTransferTsFileSealReq() { + // Empty constructor + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_TS_FILE_SEAL; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferTsFileSealReq toTPipeTransferReq(String fileName, long fileLength) + throws IOException { + return (PipeTransferTsFileSealReq) + new PipeTransferTsFileSealReq().convertToTPipeTransferReq(fileName, fileLength); + } + + public static PipeTransferTsFileSealReq fromTPipeTransferReq(TPipeTransferReq req) { + return (PipeTransferTsFileSealReq) + new PipeTransferTsFileSealReq().translateFromTPipeTransferReq(req); + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes(String fileName, long fileLength) throws IOException { + return new PipeTransferTsFileSealReq() + .convertToTPipeTransferSnapshotSealBytes(fileName, fileLength); + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object obj) { + return obj instanceof PipeTransferTsFileSealReq && super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealWithModReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealWithModReq.java new file mode 100644 index 00000000..699181ec --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/request/PipeTransferTsFileSealWithModReq.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFileSealReqV2; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; + +public class PipeTransferTsFileSealWithModReq extends PipeTransferFileSealReqV2 { + + private PipeTransferTsFileSealWithModReq() { + // Empty constructor + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_TS_FILE_SEAL_WITH_MOD; + } + + protected static final String DATABASE_NAME_KEY_PREFIX = "DATABASE_NAME_"; + + public String getDatabaseNameByTsFileName() { + return parameters == null + ? null + : parameters.get(generateDatabaseNameWithFileNameKey(fileNames.get(fileNames.size() - 1))); + } + + protected static String generateDatabaseNameWithFileNameKey(final String fileName) { + return DATABASE_NAME_KEY_PREFIX + fileName; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferTsFileSealWithModReq toTPipeTransferReq( + final String modFileName, + final long modFileLength, + final String tsFileName, + final long tsFileLength) + throws IOException { + return toTPipeTransferReq(modFileName, modFileLength, tsFileName, tsFileLength, null); + } + + public static PipeTransferTsFileSealWithModReq toTPipeTransferReq( + final String modFileName, + final long modFileLength, + final String tsFileName, + final long tsFileLength, + final String dataBaseName) + throws IOException { + return (PipeTransferTsFileSealWithModReq) + new PipeTransferTsFileSealWithModReq() + .convertToTPipeTransferReq( + Arrays.asList(modFileName, tsFileName), + Arrays.asList(modFileLength, tsFileLength), + Collections.singletonMap( + generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName)); + } + + public static PipeTransferTsFileSealWithModReq toTPipeTransferReq( + final String tsFileName, final long tsFileLength, final String dataBaseName) + throws IOException { + return (PipeTransferTsFileSealWithModReq) + new PipeTransferTsFileSealWithModReq() + .convertToTPipeTransferReq( + Collections.singletonList(tsFileName), + Collections.singletonList(tsFileLength), + Collections.singletonMap( + generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName)); + } + + public static PipeTransferTsFileSealWithModReq fromTPipeTransferReq(final TPipeTransferReq req) { + return (PipeTransferTsFileSealWithModReq) + new PipeTransferTsFileSealWithModReq().translateFromTPipeTransferReq(req); + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public static byte[] toTPipeTransferBytes( + final String modFileName, + final long modFileLength, + final String tsFileName, + final long tsFileLength) + throws IOException { + return new PipeTransferTsFileSealWithModReq() + .convertToTPipeTransferSnapshotSealBytes( + Arrays.asList(modFileName, tsFileName), + Arrays.asList(modFileLength, tsFileLength), + new HashMap<>()); + } + + public static byte[] toTPipeTransferBytes( + final String modFileName, + final long modFileLength, + final String tsFileName, + final long tsFileLength, + final String dataBaseName) + throws IOException { + return new PipeTransferTsFileSealWithModReq() + .convertToTPipeTransferSnapshotSealBytes( + Arrays.asList(modFileName, tsFileName), + Arrays.asList(modFileLength, tsFileLength), + Collections.singletonMap( + generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName)); + } + + public static byte[] toTPipeTransferBytes( + final String tsFileName, final long tsFileLength, final String dataBaseName) + throws IOException { + return new PipeTransferTsFileSealWithModReq() + .convertToTPipeTransferSnapshotSealBytes( + Collections.singletonList(tsFileName), + Collections.singletonList(tsFileLength), + Collections.singletonMap( + generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName)); + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(final Object obj) { + return obj instanceof PipeTransferTsFileSealWithModReq && super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferHandshakeConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferHandshakeConstant.java new file mode 100644 index 00000000..f2a7b3ab --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferHandshakeConstant.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.common; + +public class PipeTransferHandshakeConstant { + + public static final String HANDSHAKE_KEY_TIME_PRECISION = "timestampPrecision"; + public static final String HANDSHAKE_KEY_CLUSTER_ID = "clusterID"; + public static final String HANDSHAKE_KEY_CONVERT_ON_TYPE_MISMATCH = "convertOnTypeMismatch"; + public static final String HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY = "loadTsFileStrategy"; + public static final String HANDSHAKE_KEY_USERNAME = "username"; + public static final String HANDSHAKE_KEY_PASSWORD = "password"; + public static final String HANDSHAKE_KEY_VALIDATE_TSFILE = "validateTsFile"; + public static final String HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST = "markAsPipeRequest"; + + private PipeTransferHandshakeConstant() { + // Utility class + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferSliceReqHandler.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferSliceReqHandler.java new file mode 100644 index 00000000..45caba3d --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/common/PipeTransferSliceReqHandler.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.common; + +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferSliceReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +public class PipeTransferSliceReqHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTransferSliceReqHandler.class); + + private int orderId = -1; + + private short originReqType = -1; + private int originBodySize = -1; + + private int sliceCount = -1; + private final List sliceBodies = new ArrayList<>(); + + public boolean receiveSlice(final PipeTransferSliceReq req) { + if (orderId == -1 + || originReqType == -1 + || originBodySize == -1 + || sliceCount == -1 + || sliceBodies.isEmpty()) { + if (orderId == -1 + && originReqType == -1 + && originBodySize == -1 + && sliceCount == -1 + && sliceBodies.isEmpty()) { + orderId = req.getOrderId(); + originReqType = req.getOriginReqType(); + originBodySize = req.getOriginBodySize(); + sliceCount = req.getSliceCount(); + } else { + LOGGER.warn( + "Invalid state: orderId={}, originReqType={}, originBodySize={}, sliceCount={}, sliceBodies.size={}", + orderId, + originReqType, + originBodySize, + sliceCount, + sliceBodies.size()); + clear(); + return false; + } + } + + if (orderId != req.getOrderId()) { + LOGGER.warn("Order ID mismatch: expected {}, actual {}", orderId, req.getOrderId()); + clear(); + return false; + } + if (originReqType != req.getOriginReqType()) { + LOGGER.warn( + "Origin request type mismatch: expected {}, actual {}", + originReqType, + req.getOriginReqType()); + clear(); + return false; + } + if (originBodySize != req.getOriginBodySize()) { + LOGGER.warn( + "Origin body size mismatch: expected {}, actual {}", + originBodySize, + req.getOriginBodySize()); + clear(); + return false; + } + if (sliceCount != req.getSliceCount()) { + LOGGER.warn("Slice count mismatch: expected {}, actual {}", sliceCount, req.getSliceCount()); + clear(); + return false; + } + if (sliceBodies.size() != req.getSliceIndex()) { + LOGGER.warn( + "Invalid slice index: expected {}, actual {}", sliceBodies.size(), req.getSliceIndex()); + clear(); + return false; + } + + sliceBodies.add(req.getSliceBody()); + return true; + } + + public Optional makeReqIfComplete() { + if (sliceBodies.size() != sliceCount) { + return Optional.empty(); + } + + final TPipeTransferReq req = new TPipeTransferReq(); + req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + req.type = originReqType; + + final ByteBuffer body = ByteBuffer.allocate(originBodySize); + sliceBodies.forEach(body::put); + body.flip(); + req.body = body; + + return Optional.of(req); + } + + public void clear() { + orderId = -1; + originReqType = -1; + originBodySize = -1; + sliceCount = -1; + sliceBodies.clear(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/IoTDBConnectorRequestVersion.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/IoTDBConnectorRequestVersion.java new file mode 100644 index 00000000..54f10f23 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/IoTDBConnectorRequestVersion.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request; + +public enum IoTDBConnectorRequestVersion { + VERSION_1((byte) 1), + VERSION_2((byte) 2), + ; + + private final byte version; + + IoTDBConnectorRequestVersion(byte type) { + this.version = type; + } + + public byte getVersion() { + return version; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeRequestType.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeRequestType.java new file mode 100644 index 00000000..aab7926a --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeRequestType.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public enum PipeRequestType { + + // Handshake + HANDSHAKE_CONFIGNODE_V1((short) 0), + HANDSHAKE_DATANODE_V1((short) 1), + HANDSHAKE_CONFIGNODE_V2((short) 50), + HANDSHAKE_DATANODE_V2((short) 51), + + // Data region + TRANSFER_TABLET_INSERT_NODE((short) 2), + TRANSFER_TABLET_RAW((short) 3), + TRANSFER_TS_FILE_PIECE((short) 4), + TRANSFER_TS_FILE_SEAL((short) 5), + TRANSFER_TABLET_BATCH((short) 6), + TRANSFER_TABLET_BINARY((short) 7), + TRANSFER_TS_FILE_PIECE_WITH_MOD((short) 8), + TRANSFER_TS_FILE_SEAL_WITH_MOD((short) 9), + + TRANSFER_TABLET_INSERT_NODE_V2((short) 10), + TRANSFER_TABLET_RAW_V2((short) 11), + TRANSFER_TABLET_BINARY_V2((short) 12), + TRANSFER_TABLET_BATCH_V2((short) 13), + + // Schema region / Delete Data + TRANSFER_PLAN_NODE((short) 100), + TRANSFER_SCHEMA_SNAPSHOT_PIECE((short) 101), + TRANSFER_SCHEMA_SNAPSHOT_SEAL((short) 102), + + // Config region + TRANSFER_CONFIG_PLAN((short) 200), + TRANSFER_CONFIG_SNAPSHOT_PIECE((short) 201), + TRANSFER_CONFIG_SNAPSHOT_SEAL((short) 202), + + // RPC Compression + TRANSFER_COMPRESSED((short) 300), + + // Fallback Handling + TRANSFER_SLICE((short) 400), + ; + + private final short type; + + PipeRequestType(short type) { + this.type = type; + } + + public short getType() { + return type; + } + + private static final Map TYPE_MAP = + Arrays.stream(PipeRequestType.values()) + .collect( + HashMap::new, + (typeMap, pipeRequestType) -> typeMap.put(pipeRequestType.getType(), pipeRequestType), + HashMap::putAll); + + public static boolean isValidatedRequestType(short type) { + return TYPE_MAP.containsKey(type); + } + + public static PipeRequestType valueOf(short type) { + return TYPE_MAP.get(type); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferCompressedReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferCompressedReq.java new file mode 100644 index 00000000..7f539328 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferCompressedReq.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request; + +import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressor; +import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressorFactory; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class PipeTransferCompressedReq extends TPipeTransferReq { + + /** Generate a compressed req with provided compressors. */ + public static TPipeTransferReq toTPipeTransferReq( + final TPipeTransferReq originalReq, final List compressors) + throws IOException { + // The generated PipeTransferCompressedReq consists of: + // version + // type: TRANSFER_COMPRESSED + // body: + // (byte) count of compressors (n) + // (n*3 bytes) for each compressor: + // (byte) compressor type + // (int) length of uncompressed bytes + // compressed req: + // (byte) version + // (2 bytes) type + // (bytes) body + final PipeTransferCompressedReq compressedReq = new PipeTransferCompressedReq(); + compressedReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + compressedReq.type = PipeRequestType.TRANSFER_COMPRESSED.getType(); + + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + byte[] body = + BytesUtils.concatByteArrayList( + Arrays.asList( + new byte[] {originalReq.version}, + BytesUtils.shortToBytes(originalReq.type), + originalReq.getBody())); + + ReadWriteIOUtils.write((byte) compressors.size(), outputStream); + for (final PipeCompressor compressor : compressors) { + ReadWriteIOUtils.write(compressor.serialize(), outputStream); + ReadWriteIOUtils.write(body.length, outputStream); + body = compressor.compress(body); + } + outputStream.write(body); + + compressedReq.body = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + return compressedReq; + } + + /** Get the original req from a compressed req. */ + public static TPipeTransferReq fromTPipeTransferReq(final TPipeTransferReq transferReq) + throws IOException { + final ByteBuffer compressedBuffer = transferReq.body; + + final List compressors = new ArrayList<>(); + final List uncompressedLengths = new ArrayList<>(); + final int compressorsSize = ReadWriteIOUtils.readByte(compressedBuffer); + for (int i = 0; i < compressorsSize; ++i) { + compressors.add( + PipeCompressorFactory.getCompressor(ReadWriteIOUtils.readByte(compressedBuffer))); + uncompressedLengths.add(ReadWriteIOUtils.readInt(compressedBuffer)); + } + + byte[] body = new byte[compressedBuffer.remaining()]; + compressedBuffer.get(body); + + for (int i = compressors.size() - 1; i >= 0; --i) { + body = compressors.get(i).decompress(body, uncompressedLengths.get(i)); + } + + final ByteBuffer decompressedBuffer = ByteBuffer.wrap(body); + + final TPipeTransferReq decompressedReq = new TPipeTransferReq(); + decompressedReq.version = ReadWriteIOUtils.readByte(decompressedBuffer); + decompressedReq.type = ReadWriteIOUtils.readShort(decompressedBuffer); + decompressedReq.body = decompressedBuffer.slice(); + + return decompressedReq; + } + + /** + * For air-gap connectors. Generate the bytes of a compressed req from the bytes of original req. + */ + public static byte[] toTPipeTransferReqBytes( + final byte[] rawReqInBytes, final List compressors) throws IOException { + // The generated bytes consists of: + // (byte) version + // (2 bytes) type: TRANSFER_COMPRESSED + // (byte) count of compressors (n) + // (n*3 bytes) for each compressor: + // (byte) compressor type + // (int) length of uncompressed bytes + // compressed req: + // (byte) version + // (2 bytes) type + // (bytes) body + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + byte[] body = rawReqInBytes; + + ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(PipeRequestType.TRANSFER_COMPRESSED.getType(), outputStream); + ReadWriteIOUtils.write((byte) compressors.size(), outputStream); + for (final PipeCompressor compressor : compressors) { + ReadWriteIOUtils.write(compressor.serialize(), outputStream); + ReadWriteIOUtils.write(body.length, outputStream); + body = compressor.compress(body); + } + outputStream.write(body); + + return byteArrayOutputStream.toByteArray(); + } + } + + private PipeTransferCompressedReq() { + // Empty constructor + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFilePieceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFilePieceReq.java new file mode 100644 index 00000000..907f5542 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFilePieceReq.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Objects; + +public abstract class PipeTransferFilePieceReq extends TPipeTransferReq { + + private transient String fileName; + private transient long startWritingOffset; + private transient byte[] filePiece; + + public final String getFileName() { + return fileName; + } + + public final long getStartWritingOffset() { + return startWritingOffset; + } + + public final byte[] getFilePiece() { + return filePiece; + } + + protected abstract PipeRequestType getPlanType(); + + /////////////////////////////// Thrift /////////////////////////////// + + protected final PipeTransferFilePieceReq convertToTPipeTransferReq( + String snapshotName, long startWritingOffset, byte[] snapshotPiece) throws IOException { + + this.fileName = snapshotName; + this.startWritingOffset = startWritingOffset; + this.filePiece = snapshotPiece; + + this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + this.type = getPlanType().getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(snapshotName, outputStream); + ReadWriteIOUtils.write(startWritingOffset, outputStream); + ReadWriteIOUtils.write(new Binary(snapshotPiece), outputStream); + body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return this; + } + + protected final PipeTransferFilePieceReq translateFromTPipeTransferReq( + TPipeTransferReq transferReq) { + + fileName = ReadWriteIOUtils.readString(transferReq.body); + startWritingOffset = ReadWriteIOUtils.readLong(transferReq.body); + filePiece = ReadWriteIOUtils.readBinary(transferReq.body).getValues(); + + version = transferReq.version; + type = transferReq.type; + body = transferReq.body; + + return this; + } + + /////////////////////////////// Air Gap /////////////////////////////// + + protected final byte[] convertToTPipeTransferBytes( + String snapshotName, long startWritingOffset, byte[] snapshotPiece) throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(getPlanType().getType(), outputStream); + ReadWriteIOUtils.write(snapshotName, outputStream); + ReadWriteIOUtils.write(startWritingOffset, outputStream); + ReadWriteIOUtils.write(new Binary(snapshotPiece), outputStream); + return byteArrayOutputStream.toByteArray(); + } + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PipeTransferFilePieceReq that = (PipeTransferFilePieceReq) obj; + return fileName.equals(that.fileName) + && startWritingOffset == that.startWritingOffset + && Arrays.equals(filePiece, that.filePiece) + && version == that.version + && type == that.type + && body.equals(that.body); + } + + @Override + public int hashCode() { + return Objects.hash( + fileName, startWritingOffset, Arrays.hashCode(filePiece), version, type, body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV1.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV1.java new file mode 100644 index 00000000..7b48eed5 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV1.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public abstract class PipeTransferFileSealReqV1 extends TPipeTransferReq { + + private transient String fileName; + private transient long fileLength; + + public final String getFileName() { + return fileName; + } + + public final long getFileLength() { + return fileLength; + } + + protected abstract PipeRequestType getPlanType(); + + /////////////////////////////// Thrift /////////////////////////////// + + protected PipeTransferFileSealReqV1 convertToTPipeTransferReq(String fileName, long fileLength) + throws IOException { + + this.fileName = fileName; + this.fileLength = fileLength; + + this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + this.type = getPlanType().getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(fileName, outputStream); + ReadWriteIOUtils.write(fileLength, outputStream); + this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return this; + } + + public PipeTransferFileSealReqV1 translateFromTPipeTransferReq(TPipeTransferReq req) { + + fileName = ReadWriteIOUtils.readString(req.body); + fileLength = ReadWriteIOUtils.readLong(req.body); + + version = req.version; + type = req.type; + body = req.body; + + return this; + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public byte[] convertToTPipeTransferSnapshotSealBytes(String fileName, long fileLength) + throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(getPlanType().getType(), outputStream); + ReadWriteIOUtils.write(fileName, outputStream); + ReadWriteIOUtils.write(fileLength, outputStream); + return byteArrayOutputStream.toByteArray(); + } + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PipeTransferFileSealReqV1 that = (PipeTransferFileSealReqV1) obj; + return fileName.equals(that.fileName) + && fileLength == that.fileLength + && version == that.version + && type == that.type + && body.equals(that.body); + } + + @Override + public int hashCode() { + return Objects.hash(fileName, fileLength, version, type, body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV2.java new file mode 100644 index 00000000..590a0a45 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV2.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public abstract class PipeTransferFileSealReqV2 extends TPipeTransferReq { + + public static final String DATABASE_PATTERN = "database_pattern"; + public static final String TREE = "tree"; + public static final String TABLE = "table"; + protected transient List fileNames; + protected transient List fileLengths; + protected transient Map parameters; + + public final List getFileNames() { + return fileNames; + } + + public final List getFileLengths() { + return fileLengths; + } + + public final Map getParameters() { + return parameters; + } + + protected abstract PipeRequestType getPlanType(); + + /////////////////////////////// Thrift /////////////////////////////// + + protected PipeTransferFileSealReqV2 convertToTPipeTransferReq( + final List fileNames, + final List fileLengths, + final Map parameters) + throws IOException { + + this.fileNames = fileNames; + this.fileLengths = fileLengths; + this.parameters = parameters; + + this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + this.type = getPlanType().getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(fileNames.size(), outputStream); + for (final String fileName : fileNames) { + ReadWriteIOUtils.write(fileName, outputStream); + } + ReadWriteIOUtils.write(fileLengths.size(), outputStream); + for (final Long fileLength : fileLengths) { + ReadWriteIOUtils.write(fileLength, outputStream); + } + ReadWriteIOUtils.write(parameters.size(), outputStream); + for (final Map.Entry entry : parameters.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue(), outputStream); + } + this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return this; + } + + public PipeTransferFileSealReqV2 translateFromTPipeTransferReq(final TPipeTransferReq req) { + fileNames = new ArrayList<>(); + int size = ReadWriteIOUtils.readInt(req.body); + for (int i = 0; i < size; ++i) { + fileNames.add(ReadWriteIOUtils.readString(req.body)); + } + + fileLengths = new ArrayList<>(); + size = ReadWriteIOUtils.readInt(req.body); + for (int i = 0; i < size; ++i) { + fileLengths.add(ReadWriteIOUtils.readLong(req.body)); + } + + parameters = new HashMap<>(); + size = ReadWriteIOUtils.readInt(req.body); + for (int i = 0; i < size; ++i) { + final String key = ReadWriteIOUtils.readString(req.body); + final String value = ReadWriteIOUtils.readString(req.body); + parameters.put(key, value); + } + + version = req.version; + type = req.type; + body = req.body; + + return this; + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public byte[] convertToTPipeTransferSnapshotSealBytes( + List fileNames, List fileLengths, Map parameters) + throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(getPlanType().getType(), outputStream); + ReadWriteIOUtils.write(fileNames.size(), outputStream); + for (String fileName : fileNames) { + ReadWriteIOUtils.write(fileName, outputStream); + } + ReadWriteIOUtils.write(fileLengths.size(), outputStream); + for (Long fileLength : fileLengths) { + ReadWriteIOUtils.write(fileLength, outputStream); + } + ReadWriteIOUtils.write(parameters.size(), outputStream); + for (final Map.Entry entry : parameters.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue(), outputStream); + } + return byteArrayOutputStream.toByteArray(); + } + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PipeTransferFileSealReqV2 that = (PipeTransferFileSealReqV2) obj; + return Objects.equals(fileNames, that.fileNames) + && Objects.equals(fileLengths, that.fileLengths) + && Objects.equals(parameters, that.parameters) + && Objects.equals(version, that.version) + && Objects.equals(type, that.type) + && Objects.equals(body, that.body); + } + + @Override + public int hashCode() { + return Objects.hash(fileNames, fileLengths, parameters, version, type, body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV1Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV1Req.java new file mode 100644 index 00000000..cec929c3 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV1Req.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public abstract class PipeTransferHandshakeV1Req extends TPipeTransferReq { + + private transient String timestampPrecision; + + public final String getTimestampPrecision() { + return timestampPrecision; + } + + protected abstract PipeRequestType getPlanType(); + + /////////////////////////////// Thrift /////////////////////////////// + + public final PipeTransferHandshakeV1Req convertToTPipeTransferReq(String timestampPrecision) + throws IOException { + this.timestampPrecision = timestampPrecision; + + this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + this.type = getPlanType().getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(timestampPrecision, outputStream); + this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return this; + } + + protected final PipeTransferHandshakeV1Req translateFromTPipeTransferReq( + TPipeTransferReq transferReq) { + timestampPrecision = ReadWriteIOUtils.readString(transferReq.body); + + version = transferReq.version; + type = transferReq.type; + body = transferReq.body; + + return this; + } + + /////////////////////////////// Air Gap /////////////////////////////// + + protected final byte[] convertToTransferHandshakeBytes(String timestampPrecision) + throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(getPlanType().getType(), outputStream); + ReadWriteIOUtils.write(timestampPrecision, outputStream); + return byteArrayOutputStream.toByteArray(); + } + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PipeTransferHandshakeV1Req that = (PipeTransferHandshakeV1Req) obj; + return timestampPrecision.equals(that.timestampPrecision) + && version == that.version + && type == that.type + && body.equals(that.body); + } + + @Override + public int hashCode() { + return Objects.hash(timestampPrecision, version, type, body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java new file mode 100644 index 00000000..83cf3d23 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public abstract class PipeTransferHandshakeV2Req extends TPipeTransferReq { + + private transient Map params; + + public Map getParams() { + return params; + } + + protected abstract PipeRequestType getPlanType(); + + /////////////////////////////// Thrift /////////////////////////////// + + protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map params) + throws IOException { + this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + this.type = getPlanType().getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(params.size(), outputStream); + for (final Map.Entry entry : params.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue(), outputStream); + } + this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + this.params = params; + return this; + } + + protected final PipeTransferHandshakeV2Req translateFromTPipeTransferReq( + TPipeTransferReq transferReq) { + Map params = new HashMap<>(); + final int size = ReadWriteIOUtils.readInt(transferReq.body); + for (int i = 0; i < size; ++i) { + final String key = ReadWriteIOUtils.readString(transferReq.body); + final String value = ReadWriteIOUtils.readString(transferReq.body); + params.put(key, value); + } + this.params = params; + + version = transferReq.version; + type = transferReq.type; + body = transferReq.body; + + return this; + } + + /////////////////////////////// Air Gap /////////////////////////////// + + public final byte[] convertToTransferHandshakeBytes(Map params) + throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(getPlanType().getType(), outputStream); + ReadWriteIOUtils.write(params.size(), outputStream); + for (final Map.Entry entry : params.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue(), outputStream); + } + return byteArrayOutputStream.toByteArray(); + } + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PipeTransferHandshakeV2Req that = (PipeTransferHandshakeV2Req) obj; + return Objects.equals(params, that.params) + && version == that.version + && type == that.type + && Objects.equals(body, that.body); + } + + @Override + public int hashCode() { + return Objects.hash(params, version, type, body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSliceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSliceReq.java new file mode 100644 index 00000000..5ca1c82e --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSliceReq.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Objects; + +public class PipeTransferSliceReq extends TPipeTransferReq { + + private transient int orderId; + + private transient short originReqType; + private transient int originBodySize; + + private transient byte[] sliceBody; + + private transient int sliceIndex; + private transient int sliceCount; + + public int getOrderId() { + return orderId; + } + + public short getOriginReqType() { + return originReqType; + } + + public int getOriginBodySize() { + return originBodySize; + } + + public byte[] getSliceBody() { + return sliceBody; + } + + public int getSliceIndex() { + return sliceIndex; + } + + public int getSliceCount() { + return sliceCount; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferSliceReq toTPipeTransferReq( + final int orderId, + final short originReqType, + final int sliceIndex, + final int sliceCount, + final ByteBuffer duplicatedOriginBody, + final int startIndexInBody, + final int endIndexInBody) + throws IOException { + final PipeTransferSliceReq sliceReq = new PipeTransferSliceReq(); + + sliceReq.orderId = orderId; + + sliceReq.originReqType = originReqType; + sliceReq.originBodySize = duplicatedOriginBody.limit(); + + sliceReq.sliceBody = new byte[endIndexInBody - startIndexInBody]; + duplicatedOriginBody.position(startIndexInBody); + duplicatedOriginBody.get(sliceReq.sliceBody); + + sliceReq.sliceIndex = sliceIndex; + sliceReq.sliceCount = sliceCount; + + sliceReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion(); + sliceReq.type = PipeRequestType.TRANSFER_SLICE.getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(sliceReq.orderId, outputStream); + + ReadWriteIOUtils.write(sliceReq.originReqType, outputStream); + ReadWriteIOUtils.write(sliceReq.originBodySize, outputStream); + + ReadWriteIOUtils.write(new Binary(sliceReq.sliceBody), outputStream); + + ReadWriteIOUtils.write(sliceReq.sliceIndex, outputStream); + ReadWriteIOUtils.write(sliceReq.sliceCount, outputStream); + + sliceReq.body = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return sliceReq; + } + + public static PipeTransferSliceReq fromTPipeTransferReq(final TPipeTransferReq transferReq) { + final PipeTransferSliceReq sliceReq = new PipeTransferSliceReq(); + + sliceReq.orderId = ReadWriteIOUtils.readInt(transferReq.body); + + sliceReq.originReqType = ReadWriteIOUtils.readShort(transferReq.body); + sliceReq.originBodySize = ReadWriteIOUtils.readInt(transferReq.body); + + sliceReq.sliceBody = ReadWriteIOUtils.readBinary(transferReq.body).getValues(); + + sliceReq.sliceIndex = ReadWriteIOUtils.readInt(transferReq.body); + sliceReq.sliceCount = ReadWriteIOUtils.readInt(transferReq.body); + + sliceReq.version = transferReq.version; + sliceReq.type = transferReq.type; + sliceReq.body = transferReq.body; + + return sliceReq; + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final PipeTransferSliceReq that = (PipeTransferSliceReq) obj; + return Objects.equals(orderId, that.orderId) + && Objects.equals(originReqType, that.originReqType) + && Objects.equals(originBodySize, that.originBodySize) + && Arrays.equals(sliceBody, that.sliceBody) + && Objects.equals(sliceIndex, that.sliceIndex) + && Objects.equals(sliceCount, that.sliceCount) + && Objects.equals(version, that.version) + && Objects.equals(type, that.type) + && Objects.equals(body, that.body); + } + + @Override + public int hashCode() { + return Objects.hash( + orderId, + originReqType, + originBodySize, + Arrays.hashCode(sliceBody), + sliceIndex, + sliceCount, + version, + type, + body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/response/PipeTransferFilePieceResp.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/response/PipeTransferFilePieceResp.java new file mode 100644 index 00000000..b0c497ab --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/response/PipeTransferFilePieceResp.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.response; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class PipeTransferFilePieceResp extends TPipeTransferResp { + + public static final long ERROR_END_OFFSET = -1; + + private long endWritingOffset; + + private PipeTransferFilePieceResp() { + // Empty constructor + } + + public long getEndWritingOffset() { + return endWritingOffset; + } + + /////////////////////////////// Thrift /////////////////////////////// + + public static PipeTransferFilePieceResp toTPipeTransferResp( + TSStatus status, long endWritingOffset) throws IOException { + final PipeTransferFilePieceResp filePieceResp = new PipeTransferFilePieceResp(); + + filePieceResp.status = status; + + filePieceResp.endWritingOffset = endWritingOffset; + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(endWritingOffset, outputStream); + filePieceResp.body = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + + return filePieceResp; + } + + public static PipeTransferFilePieceResp toTPipeTransferResp(TSStatus status) { + final PipeTransferFilePieceResp filePieceResp = new PipeTransferFilePieceResp(); + + filePieceResp.status = status; + + return filePieceResp; + } + + public static PipeTransferFilePieceResp fromTPipeTransferResp(TPipeTransferResp transferResp) { + final PipeTransferFilePieceResp filePieceResp = new PipeTransferFilePieceResp(); + + filePieceResp.status = transferResp.status; + + if (transferResp.isSetBody()) { + filePieceResp.endWritingOffset = ReadWriteIOUtils.readLong(transferResp.body); + filePieceResp.body = transferResp.body; + } + + return filePieceResp; + } + + /////////////////////////////// Object /////////////////////////////// + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PipeTransferFilePieceResp that = (PipeTransferFilePieceResp) obj; + return endWritingOffset == that.endWritingOffset + && status.equals(that.status) + && body.equals(that.body); + } + + @Override + public int hashCode() { + return Objects.hash(endWritingOffset, status, body); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/DemoSink.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/DemoSink.java similarity index 97% rename from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/DemoSink.java rename to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/DemoSink.java index 8373b868..d5b8becf 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/DemoSink.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/DemoSink.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.collector.plugin.builtin.sink; +package org.apache.iotdb.collector.plugin.builtin.sink.protocol; import org.apache.iotdb.pipe.api.PipeSink; import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBConnector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBConnector.java new file mode 100644 index 00000000..70923143 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBConnector.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.protocol; + +import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressor; +import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressorConfig; +import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressorFactory; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferCompressedReq; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.pipe.api.PipeConnector; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.iotdb.rpc.UrlUtils; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_SET; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_TABLET_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_SIZE_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_HOST_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_IP_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_NODE_URLS_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PASSWORD_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PORT_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_USERNAME_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_USER_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_USER_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_STRATEGY_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_STRATEGY_SET; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_STRATEGY_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_STRATEGY_SET; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_VALIDATION_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_VALIDATION_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_MARK_AS_PIPE_REQUEST_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_MARK_AS_PIPE_REQUEST_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_COMPRESSOR_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_COMPRESSOR_ZSTD_LEVEL_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_FORMAT_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_MODE_ENABLE_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_SIZE_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_HOST_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_IP_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_NODE_URLS_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_PASSWORD_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_PORT_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_USERNAME_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_USER_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LOAD_BALANCE_STRATEGY_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LOAD_TSFILE_STRATEGY_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LOAD_TSFILE_VALIDATION_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_MARK_AS_PIPE_REQUEST_KEY; + +public abstract class IoTDBConnector implements PipeConnector { + + private static final String PARSE_URL_ERROR_FORMATTER = + "Exception occurred while parsing node urls from target servers: {}"; + private static final String PARSE_URL_ERROR_MESSAGE = + "Error occurred while parsing node urls from target servers, please check the specified 'host':'port' or 'node-urls'"; + + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConnector.class); + + protected final List nodeUrls = new ArrayList<>(); + + protected String username = CONNECTOR_IOTDB_USER_DEFAULT_VALUE; + protected String password = CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE; + + protected String loadBalanceStrategy; + + protected String loadTsFileStrategy; + protected boolean loadTsFileValidation; + + protected boolean shouldMarkAsPipeRequest; + + private boolean isRpcCompressionEnabled; + private final List compressors = new ArrayList<>(); + + protected boolean isTabletBatchModeEnabled = true; + + protected PipeReceiverStatusHandler receiverStatusHandler; + protected boolean shouldReceiverConvertOnTypeMismatch = + CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE; + + @Override + public void validate(final PipeParameterValidator validator) throws Exception { + final PipeParameters parameters = validator.getParameters(); + + validator.validate( + args -> + (boolean) args[0] + || (((boolean) args[1] || (boolean) args[2]) && (boolean) args[3]) + || (boolean) args[4] + || (((boolean) args[5] || (boolean) args[6]) && (boolean) args[7]), + String.format( + "One of %s, %s:%s, %s, %s:%s must be specified", + CONNECTOR_IOTDB_NODE_URLS_KEY, + CONNECTOR_IOTDB_HOST_KEY, + CONNECTOR_IOTDB_PORT_KEY, + SINK_IOTDB_NODE_URLS_KEY, + SINK_IOTDB_HOST_KEY, + SINK_IOTDB_PORT_KEY), + parameters.hasAttribute(CONNECTOR_IOTDB_NODE_URLS_KEY), + parameters.hasAttribute(CONNECTOR_IOTDB_IP_KEY), + parameters.hasAttribute(CONNECTOR_IOTDB_HOST_KEY), + parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY), + parameters.hasAttribute(SINK_IOTDB_NODE_URLS_KEY), + parameters.hasAttribute(SINK_IOTDB_IP_KEY), + parameters.hasAttribute(SINK_IOTDB_HOST_KEY), + parameters.hasAttribute(SINK_IOTDB_PORT_KEY)); + + validator.validate( + requestMaxBatchSizeInBytes -> (long) requestMaxBatchSizeInBytes > 0, + String.format( + "%s must be > 0, but got %s", + SINK_IOTDB_BATCH_SIZE_KEY, + parameters.getLongOrDefault( + Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY), + CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE)), + parameters.getLongOrDefault( + Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY), + CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE)); + + // Check coexistence of user and username + validator.validateSynonymAttributes( + Arrays.asList(CONNECTOR_IOTDB_USER_KEY, SINK_IOTDB_USER_KEY), + Arrays.asList(CONNECTOR_IOTDB_USERNAME_KEY, SINK_IOTDB_USERNAME_KEY), + false); + + username = + parameters.getStringOrDefault( + Arrays.asList( + CONNECTOR_IOTDB_USER_KEY, + SINK_IOTDB_USER_KEY, + CONNECTOR_IOTDB_USERNAME_KEY, + SINK_IOTDB_USERNAME_KEY), + CONNECTOR_IOTDB_USER_DEFAULT_VALUE); + password = + parameters.getStringOrDefault( + Arrays.asList(CONNECTOR_IOTDB_PASSWORD_KEY, SINK_IOTDB_PASSWORD_KEY), + CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE); + + loadBalanceStrategy = + parameters + .getStringOrDefault( + Arrays.asList(CONNECTOR_LOAD_BALANCE_STRATEGY_KEY, SINK_LOAD_BALANCE_STRATEGY_KEY), + CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY) + .trim() + .toLowerCase(); + validator.validate( + arg -> CONNECTOR_LOAD_BALANCE_STRATEGY_SET.contains(loadBalanceStrategy), + String.format( + "Load balance strategy should be one of %s, but got %s.", + CONNECTOR_LOAD_BALANCE_STRATEGY_SET, loadBalanceStrategy), + loadBalanceStrategy); + + loadTsFileStrategy = + parameters + .getStringOrDefault( + Arrays.asList(CONNECTOR_LOAD_TSFILE_STRATEGY_KEY, SINK_LOAD_TSFILE_STRATEGY_KEY), + CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE) + .trim() + .toLowerCase(); + validator.validate( + arg -> CONNECTOR_LOAD_TSFILE_STRATEGY_SET.contains(loadTsFileStrategy), + String.format( + "Load tsfile strategy should be one of %s, but got %s.", + CONNECTOR_LOAD_TSFILE_STRATEGY_SET, loadTsFileStrategy), + loadTsFileStrategy); + loadTsFileValidation = + parameters.getBooleanOrDefault( + Arrays.asList(CONNECTOR_LOAD_TSFILE_VALIDATION_KEY, SINK_LOAD_TSFILE_VALIDATION_KEY), + CONNECTOR_LOAD_TSFILE_VALIDATION_DEFAULT_VALUE); + + final int zstdCompressionLevel = + parameters.getIntOrDefault( + Arrays.asList(CONNECTOR_COMPRESSOR_ZSTD_LEVEL_KEY, SINK_COMPRESSOR_ZSTD_LEVEL_KEY), + CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE); + validator.validate( + arg -> + (int) arg >= CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE + && (int) arg <= CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE, + String.format( + "Zstd compression level should be in the range [%d, %d], but got %d.", + CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE, + CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE, + zstdCompressionLevel), + zstdCompressionLevel); + + final String compressionTypes = + parameters + .getStringOrDefault( + Arrays.asList(CONNECTOR_COMPRESSOR_KEY, SINK_COMPRESSOR_KEY), + CONNECTOR_COMPRESSOR_DEFAULT_VALUE) + .toLowerCase(); + if (!compressionTypes.isEmpty()) { + for (final String compressionType : compressionTypes.split(",")) { + final String trimmedCompressionType = compressionType.trim(); + if (trimmedCompressionType.isEmpty()) { + continue; + } + + validator.validate( + arg -> CONNECTOR_COMPRESSOR_SET.contains(trimmedCompressionType), + String.format( + "Compressor should be one of %s, but got %s.", + CONNECTOR_COMPRESSOR_SET, trimmedCompressionType), + trimmedCompressionType); + compressors.add( + PipeCompressorFactory.getCompressor( + new PipeCompressorConfig(trimmedCompressionType, zstdCompressionLevel))); + } + } + validator.validate( + arg -> compressors.size() <= Byte.MAX_VALUE, + String.format( + "The number of compressors should be less than or equal to %d, but got %d.", + Byte.MAX_VALUE, compressors.size()), + compressors.size()); + isRpcCompressionEnabled = !compressors.isEmpty(); + + validator.validate( + arg -> arg.equals("retry") || arg.equals("ignore"), + String.format( + "The value of key %s or %s must be either 'retry' or 'ignore'.", + CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY, + SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY), + parameters + .getStringOrDefault( + Arrays.asList( + CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY, + SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY), + CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE) + .trim() + .toLowerCase()); + + validator.validateAttributeValueRange( + validator.getParameters().hasAttribute(CONNECTOR_FORMAT_KEY) + ? CONNECTOR_FORMAT_KEY + : SINK_FORMAT_KEY, + true, + CONNECTOR_FORMAT_TABLET_VALUE, + CONNECTOR_FORMAT_HYBRID_VALUE, + CONNECTOR_FORMAT_TS_FILE_VALUE); + } + + @Override + public void customize( + final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) + throws Exception { + nodeUrls.clear(); + nodeUrls.addAll(parseNodeUrls(parameters)); + LOGGER.info("IoTDBConnector nodeUrls: {}", nodeUrls); + + isTabletBatchModeEnabled = + parameters.getBooleanOrDefault( + Arrays.asList( + CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY, SINK_IOTDB_BATCH_MODE_ENABLE_KEY), + CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE) + || parameters + .getStringOrDefault( + Arrays.asList(CONNECTOR_FORMAT_KEY, SINK_FORMAT_KEY), + CONNECTOR_FORMAT_HYBRID_VALUE) + .equals(CONNECTOR_FORMAT_TS_FILE_VALUE); + LOGGER.info("IoTDBConnector isTabletBatchModeEnabled: {}", isTabletBatchModeEnabled); + + shouldMarkAsPipeRequest = + parameters.getBooleanOrDefault( + Arrays.asList(CONNECTOR_MARK_AS_PIPE_REQUEST_KEY, SINK_MARK_AS_PIPE_REQUEST_KEY), + CONNECTOR_MARK_AS_PIPE_REQUEST_DEFAULT_VALUE); + LOGGER.info("IoTDBConnector shouldMarkAsPipeRequest: {}", shouldMarkAsPipeRequest); + + shouldReceiverConvertOnTypeMismatch = + parameters.getBooleanOrDefault( + Arrays.asList( + CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY, + SINK_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY), + CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE); + LOGGER.info( + "IoTDBConnector {} = {}", + CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY, + shouldReceiverConvertOnTypeMismatch); + } + + protected LinkedHashSet parseNodeUrls(final PipeParameters parameters) + throws PipeParameterNotValidException { + final LinkedHashSet givenNodeUrls = new LinkedHashSet<>(nodeUrls); + + try { + if (parameters.hasAttribute(CONNECTOR_IOTDB_IP_KEY) + && parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY)) { + givenNodeUrls.add( + new TEndPoint( + parameters.getStringByKeys(CONNECTOR_IOTDB_IP_KEY), + parameters.getIntByKeys(CONNECTOR_IOTDB_PORT_KEY))); + } + + if (parameters.hasAttribute(SINK_IOTDB_IP_KEY) + && parameters.hasAttribute(SINK_IOTDB_PORT_KEY)) { + givenNodeUrls.add( + new TEndPoint( + parameters.getStringByKeys(SINK_IOTDB_IP_KEY), + parameters.getIntByKeys(SINK_IOTDB_PORT_KEY))); + } + + if (parameters.hasAttribute(CONNECTOR_IOTDB_HOST_KEY) + && parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY)) { + givenNodeUrls.add( + new TEndPoint( + parameters.getStringByKeys(CONNECTOR_IOTDB_HOST_KEY), + parameters.getIntByKeys(CONNECTOR_IOTDB_PORT_KEY))); + } + + if (parameters.hasAttribute(SINK_IOTDB_HOST_KEY) + && parameters.hasAttribute(SINK_IOTDB_PORT_KEY)) { + givenNodeUrls.add( + new TEndPoint( + parameters.getStringByKeys(SINK_IOTDB_HOST_KEY), + parameters.getIntByKeys(SINK_IOTDB_PORT_KEY))); + } + + if (parameters.hasAttribute(CONNECTOR_IOTDB_NODE_URLS_KEY)) { + givenNodeUrls.addAll( + parseTEndPointUrls( + Arrays.asList( + parameters + .getStringByKeys(CONNECTOR_IOTDB_NODE_URLS_KEY) + .replace(" ", "") + .split(",")))); + } + + if (parameters.hasAttribute(SINK_IOTDB_NODE_URLS_KEY)) { + givenNodeUrls.addAll( + parseTEndPointUrls( + Arrays.asList( + parameters + .getStringByKeys(SINK_IOTDB_NODE_URLS_KEY) + .replace(" ", "") + .split(",")))); + } + } catch (final Exception e) { + LOGGER.warn(PARSE_URL_ERROR_FORMATTER, e.toString()); + throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE); + } + + checkNodeUrls(givenNodeUrls); + + return givenNodeUrls; + } + + private static List parseTEndPointUrls(List endPointUrls) { + if (endPointUrls == null) { + throw new NumberFormatException("endPointUrls is null"); + } + List result = new ArrayList<>(); + for (String url : endPointUrls) { + result.add(UrlUtils.parseTEndPointIpv4AndIpv6Url(url)); + } + return result; + } + + private static void checkNodeUrls(final Set nodeUrls) throws PipeParameterNotValidException { + for (final TEndPoint nodeUrl : nodeUrls) { + if (Objects.isNull(nodeUrl.ip) || nodeUrl.ip.isEmpty()) { + LOGGER.warn(PARSE_URL_ERROR_FORMATTER, "host cannot be empty"); + throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE); + } + if (nodeUrl.port == 0) { + LOGGER.warn(PARSE_URL_ERROR_FORMATTER, "port cannot be empty"); + throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE); + } + } + } + + protected TPipeTransferReq compressIfNeeded(final TPipeTransferReq req) throws IOException { + return isRpcCompressionEnabled + ? PipeTransferCompressedReq.toTPipeTransferReq(req, compressors) + : req; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBDataRegionSyncConnector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBDataRegionSyncConnector.java new file mode 100644 index 00000000..04b9844b --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBDataRegionSyncConnector.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.protocol; + +import org.apache.iotdb.collector.plugin.builtin.sink.client.IoTDBDataNodeSyncClientManager; +import org.apache.iotdb.collector.plugin.builtin.sink.client.IoTDBSyncClient; +import org.apache.iotdb.collector.plugin.builtin.sink.client.IoTDBSyncClientManager; +import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeRawTabletInsertionEvent; +import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeTsFileInsertionEvent; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch.PipeTabletEventBatch; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch.PipeTabletEventPlainBatch; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch.PipeTabletEventTsFileBatch; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch.PipeTransferBatchReqBuilder; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request.PipeTransferTabletRawReqV2; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request.PipeTransferTsFilePieceReq; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request.PipeTransferTsFilePieceWithModReq; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.request.PipeTransferTsFileSealWithModReq; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFilePieceReq; +import org.apache.iotdb.collector.plugin.builtin.sink.utils.cacher.LeaderCacheUtils; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; +import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; +import org.apache.iotdb.pipe.api.exception.PipeConnectionException; +import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; + +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class IoTDBDataRegionSyncConnector extends IoTDBSslSyncConnector { + + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDataRegionSyncConnector.class); + + private PipeTransferBatchReqBuilder tabletBatchBuilder; + + protected IoTDBDataNodeSyncClientManager clientManager; + + @Override + protected IoTDBSyncClientManager constructClient( + final List nodeUrls, + final boolean useSSL, + final String trustStorePath, + final String trustStorePwd, + /* The following parameters are used locally. */ + final boolean useLeaderCache, + final String loadBalanceStrategy, + /* The following parameters are used to handshake with the receiver. */ + final String username, + final String password, + final boolean shouldReceiverConvertOnTypeMismatch, + final String loadTsFileStrategy, + final boolean validateTsFile, + final boolean shouldMarkAsPipeRequest) { + clientManager = + new IoTDBDataNodeSyncClientManager( + nodeUrls, + useSSL, + Objects.nonNull(trustStorePath) + ? /*IoTDBConfig.addDataHomeDir(trustStorePath)*/ "" + : null, + trustStorePwd, + useLeaderCache, + loadBalanceStrategy, + username, + password, + shouldReceiverConvertOnTypeMismatch, + loadTsFileStrategy, + validateTsFile, + shouldMarkAsPipeRequest); + return clientManager; + } + + @Override + public void customize( + final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) + throws Exception { + super.customize(parameters, configuration); + + // tablet batch mode configuration + if (isTabletBatchModeEnabled) { + tabletBatchBuilder = new PipeTransferBatchReqBuilder(parameters); + } + } + + @Override + protected PipeTransferFilePieceReq getTransferSingleFilePieceReq( + final String fileName, final long position, final byte[] payLoad) throws IOException { + return PipeTransferTsFilePieceReq.toTPipeTransferReq(fileName, position, payLoad); + } + + @Override + protected PipeTransferFilePieceReq getTransferMultiFilePieceReq( + final String fileName, final long position, final byte[] payLoad) throws IOException { + return PipeTransferTsFilePieceWithModReq.toTPipeTransferReq(fileName, position, payLoad); + } + + @Override + public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exception { + if (isTabletBatchModeEnabled) { + final Pair endPointAndBatch = + tabletBatchBuilder.onEvent(tabletInsertionEvent); + if (Objects.nonNull(endPointAndBatch)) { + doTransferWrapper(endPointAndBatch); + } + } else { + doTransferWrapper((PipeRawTabletInsertionEvent) tabletInsertionEvent); + } + } + + @Override + public void transfer(final TsFileInsertionEvent tsFileInsertionEvent) throws Exception { + // In order to commit in order + if (isTabletBatchModeEnabled && !tabletBatchBuilder.isEmpty()) { + doTransferWrapper(); + } + + doTransferWrapper((PipeTsFileInsertionEvent) tsFileInsertionEvent); + } + + @Override + public void transfer(final Event event) throws Exception { + if (isTabletBatchModeEnabled && !tabletBatchBuilder.isEmpty()) { + doTransferWrapper(); + } + } + + private void doTransferWrapper() throws IOException, WriteProcessException { + for (final Pair nonEmptyBatch : + tabletBatchBuilder.getAllNonEmptyBatches()) { + doTransferWrapper(nonEmptyBatch); + } + } + + private void doTransferWrapper(final Pair endPointAndBatch) + throws IOException, WriteProcessException { + final PipeTabletEventBatch batch = endPointAndBatch.getRight(); + if (batch instanceof PipeTabletEventPlainBatch) { + doTransfer(endPointAndBatch.getLeft(), (PipeTabletEventPlainBatch) batch); + } else if (batch instanceof PipeTabletEventTsFileBatch) { + doTransfer((PipeTabletEventTsFileBatch) batch); + } else { + LOGGER.warn("Unsupported batch type {}.", batch.getClass()); + } + batch.onSuccess(); + } + + private void doTransfer( + final TEndPoint endPoint, final PipeTabletEventPlainBatch batchToTransfer) { + final Pair clientAndStatus = clientManager.getClient(endPoint); + + final TPipeTransferResp resp; + try { + final TPipeTransferReq uncompressedReq = batchToTransfer.toTPipeTransferReq(); + final long uncompressedSize = uncompressedReq.getBody().length; + + final TPipeTransferReq req = compressIfNeeded(uncompressedReq); + final long compressedSize = req.getBody().length; + + final double compressionRatio = (double) compressedSize / uncompressedSize; + + resp = clientAndStatus.getLeft().pipeTransfer(req); + } catch (final Exception e) { + clientAndStatus.setRight(false); + throw new PipeConnectionException( + String.format("Network error when transfer tablet batch, because %s.", e.getMessage()), + e); + } + + final TSStatus status = resp.getStatus(); + // Only handle the failed statuses to avoid string format performance overhead + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { + receiverStatusHandler.handle( + resp.getStatus(), + String.format("Transfer PipeTransferTabletBatchReq error, result status %s", resp.status), + batchToTransfer.deepCopyEvents().toString()); + } + + for (final Pair redirectPair : + LeaderCacheUtils.parseRecommendedRedirections(status)) { + clientManager.updateLeaderCache(redirectPair.getLeft(), redirectPair.getRight()); + } + } + + private void doTransfer(final PipeTabletEventTsFileBatch batchToTransfer) + throws IOException, WriteProcessException { + final List> dbTsFilePairs = batchToTransfer.sealTsFiles(); + final Map, Double> pipe2WeightMap = batchToTransfer.deepCopyPipe2WeightMap(); + + for (final Pair dbTsFile : dbTsFilePairs) { + doTransfer(dbTsFile.right, null, dbTsFile.left); + try { + FileUtils.delete(dbTsFile.right); + } catch (final NoSuchFileException e) { + LOGGER.info("The file {} is not found, may already be deleted.", dbTsFile); + } catch (final Exception e) { + LOGGER.warn( + "Failed to delete batch file {}, this file should be deleted manually later", dbTsFile); + } + } + } + + private void doTransferWrapper(final PipeRawTabletInsertionEvent pipeRawTabletInsertionEvent) + throws PipeException { + doTransfer(pipeRawTabletInsertionEvent); + } + + private void doTransfer(final PipeRawTabletInsertionEvent pipeRawTabletInsertionEvent) + throws PipeException { + final Pair clientAndStatus = + clientManager.getClient(pipeRawTabletInsertionEvent.getDeviceId()); + final TPipeTransferResp resp; + + try { + final TPipeTransferReq req = + compressIfNeeded( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + pipeRawTabletInsertionEvent.getTablet(), + pipeRawTabletInsertionEvent.isAligned(), + pipeRawTabletInsertionEvent.isTableModelEvent() + ? pipeRawTabletInsertionEvent.getTableModelDatabaseName() + : null)); + resp = clientAndStatus.getLeft().pipeTransfer(req); + } catch (final Exception e) { + clientAndStatus.setRight(false); + throw new PipeConnectionException( + String.format( + "Network error when transfer raw tablet insertion event, because %s.", + e.getMessage()), + e); + } + + final TSStatus status = resp.getStatus(); + // Only handle the failed statuses to avoid string format performance overhead + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { + receiverStatusHandler.handle( + status, + String.format( + "Transfer PipeRawTabletInsertionEvent %s error, result status %s", + pipeRawTabletInsertionEvent, status), + pipeRawTabletInsertionEvent.toString()); + } + if (status.isSetRedirectNode()) { + clientManager.updateLeaderCache( + pipeRawTabletInsertionEvent.getDeviceId(), status.getRedirectNode()); + } + } + + private void doTransferWrapper(final PipeTsFileInsertionEvent pipeTsFileInsertionEvent) + throws PipeException, IOException { + doTransfer( + pipeTsFileInsertionEvent.getTsFile(), + null, + pipeTsFileInsertionEvent.isTableModelEvent() + ? pipeTsFileInsertionEvent.getTableModelDatabaseName() + : null); + } + + private void doTransfer( + final File tsFile, + final File modFile, + final String dataBaseName) + throws PipeException, IOException { + + final Pair clientAndStatus = clientManager.getClient(); + final TPipeTransferResp resp; + + // 1. Transfer tsFile, and mod file if exists and receiver's version >= 2 + if (Objects.nonNull(modFile) && clientManager.supportModsIfIsDataNodeReceiver()) { + transferFilePieces(modFile, clientAndStatus, true); + transferFilePieces(tsFile, clientAndStatus, true); + + // 2. Transfer file seal signal with mod, which means the file is transferred completely + try { + final TPipeTransferReq req = + compressIfNeeded( + PipeTransferTsFileSealWithModReq.toTPipeTransferReq( + modFile.getName(), + modFile.length(), + tsFile.getName(), + tsFile.length(), + dataBaseName)); + + resp = clientAndStatus.getLeft().pipeTransfer(req); + } catch (final Exception e) { + clientAndStatus.setRight(false); + clientManager.adjustTimeoutIfNecessary(e); + throw new PipeConnectionException( + String.format("Network error when seal file %s, because %s.", tsFile, e.getMessage()), + e); + } + } else { + transferFilePieces(tsFile, clientAndStatus, false); + + // 2. Transfer file seal signal without mod, which means the file is transferred completely + try { + final TPipeTransferReq req = + compressIfNeeded( + PipeTransferTsFileSealWithModReq.toTPipeTransferReq( + tsFile.getName(), tsFile.length(), dataBaseName)); + + resp = clientAndStatus.getLeft().pipeTransfer(req); + } catch (final Exception e) { + clientAndStatus.setRight(false); + clientManager.adjustTimeoutIfNecessary(e); + throw new PipeConnectionException( + String.format("Network error when seal file %s, because %s.", tsFile, e.getMessage()), + e); + } + } + + final TSStatus status = resp.getStatus(); + // Only handle the failed statuses to avoid string format performance overhead + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { + receiverStatusHandler.handle( + resp.getStatus(), + String.format("Seal file %s error, result status %s.", tsFile, resp.getStatus()), + tsFile.getName()); + } + + LOGGER.info("Successfully transferred file {}.", tsFile); + } + + @Override + public void close() throws Exception { + if (tabletBatchBuilder != null) { + tabletBatchBuilder.close(); + } + + super.close(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBSslSyncConnector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBSslSyncConnector.java new file mode 100644 index 00000000..c49cc2c6 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBSslSyncConnector.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.protocol; + +import org.apache.iotdb.collector.config.PipeOptions; +import org.apache.iotdb.collector.plugin.builtin.sink.client.IoTDBSyncClient; +import org.apache.iotdb.collector.plugin.builtin.sink.client.IoTDBSyncClientManager; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFilePieceReq; +import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.response.PipeTransferFilePieceResp; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeConnectionException; +import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_SSL_ENABLE_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_KEY; +import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LEADER_CACHE_ENABLE_KEY; + +public abstract class IoTDBSslSyncConnector extends IoTDBConnector { + + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSslSyncConnector.class); + + protected IoTDBSyncClientManager clientManager; + + @Override + public void validate(final PipeParameterValidator validator) throws Exception { + super.validate(validator); + + final PipeParameters parameters = validator.getParameters(); + + final String userSpecifiedConnectorName = + parameters + .getStringOrDefault( + ImmutableList.of(CONNECTOR_KEY, SINK_KEY), "" + /*IOTDB_THRIFT_CONNECTOR.getPipePluginName()*/ ) + .toLowerCase(); + + validator.validate( + args -> !((boolean) args[0]) || ((boolean) args[1] && (boolean) args[2]), + String.format( + "When ssl transport is enabled, %s and %s must be specified", + SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY, SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY), + // IOTDB_THRIFT_SSL_CONNECTOR.getPipePluginName().equals(userSpecifiedConnectorName) + // || IOTDB_THRIFT_SSL_SINK.getPipePluginName().equals(userSpecifiedConnectorName) + /*|| */ parameters.getBooleanOrDefault(SINK_IOTDB_SSL_ENABLE_KEY, false), + parameters.hasAttribute(SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY), + parameters.hasAttribute(SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY)); + } + + @Override + public void customize( + final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) + throws Exception { + super.customize(parameters, configuration); + + // ssl transport configuration + // final String userSpecifiedConnectorName = + // parameters + // .getStringOrDefault( + // ImmutableList.of(CONNECTOR_KEY, SINK_KEY), + // IOTDB_THRIFT_CONNECTOR.getPipePluginName()) + // .toLowerCase(); + // final boolean useSSL = + // IOTDB_THRIFT_SSL_CONNECTOR.getPipePluginName().equals(userSpecifiedConnectorName) + // || IOTDB_THRIFT_SSL_SINK.getPipePluginName().equals(userSpecifiedConnectorName) + // || parameters.getBooleanOrDefault(SINK_IOTDB_SSL_ENABLE_KEY, false); + final String trustStorePath = parameters.getString(SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY); + final String trustStorePwd = parameters.getString(SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY); + + // leader cache configuration + final boolean useLeaderCache = + parameters.getBooleanOrDefault( + Arrays.asList(SINK_LEADER_CACHE_ENABLE_KEY, CONNECTOR_LEADER_CACHE_ENABLE_KEY), + CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE); + + clientManager = + constructClient( + nodeUrls, + // useSSL, + true, + trustStorePath, + trustStorePwd, + useLeaderCache, + loadBalanceStrategy, + username, + password, + shouldReceiverConvertOnTypeMismatch, + loadTsFileStrategy, + loadTsFileValidation, + shouldMarkAsPipeRequest); + } + + protected abstract IoTDBSyncClientManager constructClient( + final List nodeUrls, + final boolean useSSL, + final String trustStorePath, + final String trustStorePwd, + /* The following parameters are used locally. */ + final boolean useLeaderCache, + final String loadBalanceStrategy, + /* The following parameters are used to handshake with the receiver. */ + final String username, + final String password, + final boolean shouldReceiverConvertOnTypeMismatch, + final String loadTsFileStrategy, + final boolean validateTsFile, + final boolean shouldMarkAsPipeRequest); + + @Override + public void handshake() throws Exception { + clientManager.checkClientStatusAndTryReconstructIfNecessary(); + } + + @Override + public void heartbeat() { + try { + handshake(); + } catch (final Exception e) { + LOGGER.warn( + "Failed to reconnect to target server, because: {}. Try to reconnect later.", + e.getMessage(), + e); + } + } + + protected void transferFilePieces( + final File file, + final Pair clientAndStatus, + final boolean isMultiFile) + throws PipeException, IOException { + final int readFileBufferSize = PipeOptions.PIPE_CONNECTOR_READ_FILE_BUFFER_SIZE.value(); + final byte[] readBuffer = new byte[readFileBufferSize]; + long position = 0; + try (final RandomAccessFile reader = new RandomAccessFile(file, "r")) { + while (true) { + final int readLength = reader.read(readBuffer); + if (readLength == -1) { + break; + } + + final byte[] payLoad = + readLength == readFileBufferSize + ? readBuffer + : Arrays.copyOfRange(readBuffer, 0, readLength); + final PipeTransferFilePieceResp resp; + try { + final TPipeTransferReq req = + compressIfNeeded( + isMultiFile + ? getTransferMultiFilePieceReq(file.getName(), position, payLoad) + : getTransferSingleFilePieceReq(file.getName(), position, payLoad)); + + resp = + PipeTransferFilePieceResp.fromTPipeTransferResp( + clientAndStatus.getLeft().pipeTransfer(req)); + } catch (final Exception e) { + clientAndStatus.setRight(false); + throw new PipeConnectionException( + String.format( + "Network error when transfer file %s, because %s.", file, e.getMessage()), + e); + } + + position += readLength; + + final TSStatus status = resp.getStatus(); + // This case only happens when the connection is broken, and the connector is reconnected + // to the receiver, then the receiver will redirect the file position to the last position + if (status.getCode() == TSStatusCode.PIPE_TRANSFER_FILE_OFFSET_RESET.getStatusCode()) { + position = resp.getEndWritingOffset(); + reader.seek(position); + LOGGER.info("Redirect file position to {}.", position); + continue; + } + + // Send handshake req and then re-transfer the event + if (status.getCode() + == TSStatusCode.PIPE_CONFIG_RECEIVER_HANDSHAKE_NEEDED.getStatusCode()) { + clientManager.sendHandshakeReq(clientAndStatus); + } + // Only handle the failed statuses to avoid string format performance overhead + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { + receiverStatusHandler.handle( + resp.getStatus(), + String.format("Transfer file %s error, result status %s.", file, resp.getStatus()), + file.getName()); + } + } + } + } + + protected abstract PipeTransferFilePieceReq getTransferSingleFilePieceReq( + final String fileName, final long position, final byte[] payLoad) throws IOException; + + protected abstract PipeTransferFilePieceReq getTransferMultiFilePieceReq( + final String fileName, final long position, final byte[] payLoad) throws IOException; + + @Override + public void close() throws Exception { + if (clientManager != null) { + clientManager.close(); + } + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/PipeReceiverStatusHandler.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/PipeReceiverStatusHandler.java new file mode 100644 index 00000000..12f8e090 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/PipeReceiverStatusHandler.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.protocol; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +public class PipeReceiverStatusHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeReceiverStatusHandler.class); + + private static final int CONFLICT_RETRY_MAX_TIMES = 100; + + private final boolean isRetryAllowedWhenConflictOccurs; + private final long retryMaxMillisWhenConflictOccurs; + private final boolean shouldRecordIgnoredDataWhenConflictOccurs; + + private final long retryMaxMillisWhenOtherExceptionsOccur; + private final boolean shouldRecordIgnoredDataWhenOtherExceptionsOccur; + + private final AtomicLong exceptionFirstEncounteredTime = new AtomicLong(0); + private final AtomicBoolean exceptionEventHasBeenRetried = new AtomicBoolean(false); + private final AtomicReference exceptionRecordedMessage = new AtomicReference<>(""); + + public PipeReceiverStatusHandler( + final boolean isRetryAllowedWhenConflictOccurs, + final long retryMaxSecondsWhenConflictOccurs, + final boolean shouldRecordIgnoredDataWhenConflictOccurs, + final long retryMaxSecondsWhenOtherExceptionsOccur, + final boolean shouldRecordIgnoredDataWhenOtherExceptionsOccur) { + this.isRetryAllowedWhenConflictOccurs = isRetryAllowedWhenConflictOccurs; + this.retryMaxMillisWhenConflictOccurs = + retryMaxSecondsWhenConflictOccurs < 0 + ? Long.MAX_VALUE + : retryMaxSecondsWhenConflictOccurs * 1000; + this.shouldRecordIgnoredDataWhenConflictOccurs = shouldRecordIgnoredDataWhenConflictOccurs; + + this.retryMaxMillisWhenOtherExceptionsOccur = + retryMaxSecondsWhenOtherExceptionsOccur < 0 + ? Long.MAX_VALUE + : retryMaxSecondsWhenOtherExceptionsOccur * 1000; + this.shouldRecordIgnoredDataWhenOtherExceptionsOccur = + shouldRecordIgnoredDataWhenOtherExceptionsOccur; + } + + /** + * Handle {@link TSStatus} returned by receiver. Do nothing if ignore the {@link Event}, and throw + * exception if retry the {@link Event}. Upper class must ensure that the method is invoked only + * by a single thread. + * + * @throws PipeException to retry the current {@link Event} + * @param status the {@link TSStatus} to judge + * @param exceptionMessage The exception message to throw + * @param recordMessage The message to record an ignored {@link Event}, the caller should assure + * that the same {@link Event} generates always the same record message, for instance, do not + * put any time-related info here + */ + public void handle( + final TSStatus status, final String exceptionMessage, final String recordMessage) { + switch (status.getCode()) { + case 200: // SUCCESS_STATUS + case 400: // REDIRECTION_RECOMMEND + { + return; + } + + case 1809: // PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION + { + LOGGER.info("Idempotent conflict exception: will be ignored. status: {}", status); + return; + } + + case 1808: // PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION + { + LOGGER.info("Temporary unavailable exception: will retry forever. status: {}", status); + throw new PipeException(exceptionMessage); + } + + case 1810: // PIPE_RECEIVER_USER_CONFLICT_EXCEPTION + if (!isRetryAllowedWhenConflictOccurs) { + LOGGER.warn( + "User conflict exception: will be ignored because retry is not allowed. event: {}. status: {}", + shouldRecordIgnoredDataWhenConflictOccurs ? recordMessage : "not recorded", + status); + return; + } + + synchronized (this) { + recordExceptionStatusIfNecessary(recordMessage); + + if (exceptionEventHasBeenRetried.get() + && System.currentTimeMillis() - exceptionFirstEncounteredTime.get() + > retryMaxMillisWhenConflictOccurs) { + LOGGER.warn( + "User conflict exception: retry timeout. will be ignored. event: {}. status: {}", + shouldRecordIgnoredDataWhenConflictOccurs ? recordMessage : "not recorded", + status); + resetExceptionStatus(); + return; + } + + LOGGER.warn( + "User conflict exception: will retry {}. status: {}", + retryMaxMillisWhenConflictOccurs == Long.MAX_VALUE + ? "forever" + : "for at least " + + (retryMaxMillisWhenConflictOccurs + + exceptionFirstEncounteredTime.get() + - System.currentTimeMillis()) + / 1000.0 + + " seconds", + status); + exceptionEventHasBeenRetried.set(true); + throw new PipeException( + exceptionMessage, + (int) + Math.max( + 5, + Math.min(CONFLICT_RETRY_MAX_TIMES, retryMaxMillisWhenConflictOccurs * 1.1))); + } + + default: // Other exceptions + synchronized (this) { + recordExceptionStatusIfNecessary(recordMessage); + + if (exceptionEventHasBeenRetried.get() + && System.currentTimeMillis() - exceptionFirstEncounteredTime.get() + > retryMaxMillisWhenOtherExceptionsOccur) { + LOGGER.warn( + "Unclassified exception: retry timeout. will be ignored. event: {}. status: {}", + shouldRecordIgnoredDataWhenOtherExceptionsOccur ? recordMessage : "not recorded", + status); + resetExceptionStatus(); + return; + } + + LOGGER.warn( + "Unclassified exception: will retry {}. status: {}", + retryMaxMillisWhenOtherExceptionsOccur == Long.MAX_VALUE + ? "forever" + : "for at least " + + (retryMaxMillisWhenOtherExceptionsOccur + + exceptionFirstEncounteredTime.get() + - System.currentTimeMillis()) + / 1000.0 + + " seconds", + status); + exceptionEventHasBeenRetried.set(true); + throw new PipeException( + exceptionMessage, + (int) + Math.max( + 5, + Math.min( + CONFLICT_RETRY_MAX_TIMES, retryMaxMillisWhenOtherExceptionsOccur * 1.1))); + } + } + } + + private void recordExceptionStatusIfNecessary(final String message) { + if (!Objects.equals(exceptionRecordedMessage.get(), message)) { + exceptionFirstEncounteredTime.set(System.currentTimeMillis()); + exceptionEventHasBeenRetried.set(false); + exceptionRecordedMessage.set(message); + } + } + + private void resetExceptionStatus() { + exceptionFirstEncounteredTime.set(0); + exceptionEventHasBeenRetried.set(false); + exceptionRecordedMessage.set(""); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/builder/PipeTableModeTsFileBuilder.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/builder/PipeTableModeTsFileBuilder.java new file mode 100644 index 00000000..74030d87 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/builder/PipeTableModeTsFileBuilder.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.utils.builder; + +import org.apache.iotdb.pipe.api.exception.PipeException; + +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.WriteUtils; +import org.apache.tsfile.write.record.Tablet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +public class PipeTableModeTsFileBuilder extends PipeTsFileBuilder { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTableModeTsFileBuilder.class); + + private final Map> dataBase2TabletList = new HashMap<>(); + + public PipeTableModeTsFileBuilder(AtomicLong currentBatchId, AtomicLong tsFileIdGenerator) { + super(currentBatchId, tsFileIdGenerator); + } + + @Override + public void bufferTableModelTablet(String dataBase, Tablet tablet) { + dataBase2TabletList.computeIfAbsent(dataBase, db -> new ArrayList<>()).add(tablet); + } + + @Override + public void bufferTreeModelTablet(Tablet tablet, Boolean isAligned) { + throw new UnsupportedOperationException( + "PipeTableModeTsFileBuilder does not support tree model tablet to build TSFile"); + } + + @Override + public List> convertTabletToTsFileWithDBInfo() throws IOException { + if (dataBase2TabletList.isEmpty()) { + return new ArrayList<>(0); + } + List> pairList = new ArrayList<>(); + for (Map.Entry> entry : dataBase2TabletList.entrySet()) { + final LinkedHashSet>>>> linkedHashSet = + new LinkedHashSet<>(); + pairList.addAll( + writeTableModelTabletsToTsFiles(entry.getValue(), entry.getKey(), linkedHashSet)); + } + return pairList; + } + + @Override + public boolean isEmpty() { + return dataBase2TabletList.isEmpty(); + } + + @Override + public synchronized void onSuccess() { + super.onSuccess(); + dataBase2TabletList.clear(); + } + + @Override + public synchronized void close() { + super.close(); + dataBase2TabletList.clear(); + } + + private >>> + List> writeTableModelTabletsToTsFiles( + final List tabletList, + final String dataBase, + LinkedHashSet> linkedHashSet) + throws IOException { + + final Map> tableName2Tablets = new HashMap<>(); + + // Sort the tablets by dataBaseName + for (final Tablet tablet : tabletList) { + tableName2Tablets + .computeIfAbsent(tablet.getTableName(), k -> new ArrayList<>()) + .add((T) new Pair<>(tablet, WriteUtils.splitTabletByDevice(tablet))); + } + + // Replace ArrayList with LinkedList to improve performance + final LinkedHashSet> table2Tablets = new LinkedHashSet<>(); + + // Sort the tablets by start time in first device + for (final List tablets : tableName2Tablets.values()) { + tablets.sort( + (o1, o2) -> { + final IDeviceID deviceID = o1.right.get(0).left; + final int result; + if ((result = deviceID.compareTo(o2.right.get(0).left)) == 0) { + return Long.compare(o1.left.getTimestamp(0), o2.left.getTimestamp(0)); + } + return result; + }); + } + + // Sort the tables by table name + tableName2Tablets.entrySet().stream() + .sorted(Map.Entry.comparingByKey(Comparator.naturalOrder())) + .forEach(entry -> linkedHashSet.add(new LinkedList<>(entry.getValue()))); + + // Help GC + tableName2Tablets.clear(); + + final List> sealedFiles = new ArrayList<>(); + + // Try making the tsfile size as large as possible + while (!linkedHashSet.isEmpty()) { + if (Objects.isNull(fileWriter)) { + createFileWriter(); + } + + try { + tryBestToWriteTabletsIntoOneFile(linkedHashSet); + } catch (final Exception e) { + LOGGER.warn( + "Batch id = {}: Failed to write tablets into tsfile, because {}", + currentBatchId.get(), + e.getMessage(), + e); + + try { + fileWriter.close(); + } catch (final Exception closeException) { + LOGGER.warn( + "Batch id = {}: Failed to close the tsfile {} after failed to write tablets into, because {}", + currentBatchId.get(), + fileWriter.getIOWriter().getFile().getPath(), + closeException.getMessage(), + closeException); + } finally { + // Add current writing file to the list and delete the file + sealedFiles.add(new Pair<>(dataBase, fileWriter.getIOWriter().getFile())); + } + + for (final Pair sealedFile : sealedFiles) { + final boolean deleteSuccess = FileUtils.deleteQuietly(sealedFile.right); + LOGGER.warn( + "Batch id = {}: {} delete the tsfile {} after failed to write tablets into {}. {}", + currentBatchId.get(), + deleteSuccess ? "Successfully" : "Failed to", + sealedFile.right.getPath(), + fileWriter.getIOWriter().getFile().getPath(), + deleteSuccess ? "" : "Maybe the tsfile needs to be deleted manually."); + } + sealedFiles.clear(); + + fileWriter = null; + + throw e; + } + + fileWriter.close(); + final File sealedFile = fileWriter.getIOWriter().getFile(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Batch id = {}: Seal tsfile {} successfully.", + currentBatchId.get(), + sealedFile.getPath()); + } + sealedFiles.add(new Pair<>(dataBase, sealedFile)); + fileWriter = null; + } + + return sealedFiles; + } + + private >>> + void tryBestToWriteTabletsIntoOneFile( + final LinkedHashSet> device2TabletsLinkedList) throws IOException { + final Iterator> iterator = device2TabletsLinkedList.iterator(); + + while (iterator.hasNext()) { + final LinkedList tablets = iterator.next(); + + final List tabletsToWrite = new ArrayList<>(); + final Map deviceLastTimestampMap = new HashMap<>(); + while (!tablets.isEmpty()) { + final T pair = tablets.peekFirst(); + if (timestampsAreNonOverlapping( + (Pair>>) pair, deviceLastTimestampMap)) { + tabletsToWrite.add(pair); + tablets.pollFirst(); + continue; + } + break; + } + + if (tablets.isEmpty()) { + iterator.remove(); + } + boolean schemaNotRegistered = true; + for (final Pair>> pair : tabletsToWrite) { + final Tablet tablet = pair.left; + if (schemaNotRegistered) { + fileWriter.registerTableSchema( + new TableSchema(tablet.getTableName(), tablet.getSchemas(), tablet.getColumnTypes())); + schemaNotRegistered = false; + } + try { + fileWriter.writeTable(tablet, pair.right); + } catch (WriteProcessException e) { + LOGGER.warn( + "Batch id = {}: Failed to build the table model TSFile. Please check whether the written Tablet has time overlap and whether the Table Schema is correct.", + currentBatchId.get(), + e); + throw new PipeException( + "The written Tablet time may overlap or the Schema may be incorrect"); + } + } + } + } + + /** + * A Map is used to record the maximum time each {@link IDeviceID} is written. {@link Pair} + * records the Index+1 of the maximum timestamp of IDevice in each {@link Tablet}. + * + * @return If false, the tablet overlaps with the previous tablet; if true, there is no time + * overlap. + */ + private >>> + boolean timestampsAreNonOverlapping( + final T tabletPair, final Map deviceLastTimestampMap) { + int currentTimestampIndex = 0; + for (Pair deviceTimestampIndexPair : tabletPair.right) { + final Long lastDeviceTimestamp = deviceLastTimestampMap.get(deviceTimestampIndexPair.left); + if (lastDeviceTimestamp != null + && lastDeviceTimestamp >= tabletPair.left.getTimestamp(currentTimestampIndex)) { + return false; + } + currentTimestampIndex = deviceTimestampIndexPair.right; + deviceLastTimestampMap.put( + deviceTimestampIndexPair.left, tabletPair.left.getTimestamp(currentTimestampIndex - 1)); + } + + return true; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/builder/PipeTreeModelTsFileBuilder.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/builder/PipeTreeModelTsFileBuilder.java new file mode 100644 index 00000000..8f5d48cf --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/builder/PipeTreeModelTsFileBuilder.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.utils.builder; + +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +public class PipeTreeModelTsFileBuilder extends PipeTsFileBuilder { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTreeModelTsFileBuilder.class); + + private final List tabletList = new ArrayList<>(); + private final List isTabletAlignedList = new ArrayList<>(); + + public PipeTreeModelTsFileBuilder( + final AtomicLong currentBatchId, final AtomicLong tsFileIdGenerator) { + super(currentBatchId, tsFileIdGenerator); + } + + @Override + public void bufferTableModelTablet(final String dataBase, final Tablet tablet) { + throw new UnsupportedOperationException( + "PipeTreeModelTsFileBuilder does not support table model tablet to build TSFile"); + } + + @Override + public void bufferTreeModelTablet(final Tablet tablet, final Boolean isAligned) { + tabletList.add(tablet); + isTabletAlignedList.add(isAligned); + } + + @Override + public List> convertTabletToTsFileWithDBInfo() + throws IOException, WriteProcessException { + return writeTabletsToTsFiles(); + } + + @Override + public boolean isEmpty() { + return tabletList.isEmpty(); + } + + @Override + public void onSuccess() { + super.onSuccess(); + tabletList.clear(); + isTabletAlignedList.clear(); + } + + @Override + public synchronized void close() { + super.close(); + tabletList.clear(); + isTabletAlignedList.clear(); + } + + private List> writeTabletsToTsFiles() + throws IOException, WriteProcessException { + final Map> device2Tablets = new HashMap<>(); + final Map device2Aligned = new HashMap<>(); + + // Sort the tablets by device id + for (int i = 0, size = tabletList.size(); i < size; ++i) { + final Tablet tablet = tabletList.get(i); + final String deviceId = tablet.getDeviceId(); + device2Tablets.computeIfAbsent(deviceId, k -> new ArrayList<>()).add(tablet); + device2Aligned.put(deviceId, isTabletAlignedList.get(i)); + } + + // Sort the tablets by start time in each device + for (final List tablets : device2Tablets.values()) { + tablets.sort( + // Each tablet has at least one timestamp + Comparator.comparingLong(tablet -> tablet.getTimestamp(0))); + } + + // Sort the devices by device id + final List devices = new ArrayList<>(device2Tablets.keySet()); + devices.sort(Comparator.naturalOrder()); + + // Replace ArrayList with LinkedList to improve performance + final LinkedHashMap> device2TabletsLinkedList = + new LinkedHashMap<>(); + for (final String device : devices) { + device2TabletsLinkedList.put(device, new LinkedList<>(device2Tablets.get(device))); + } + + // Help GC + devices.clear(); + device2Tablets.clear(); + + // Write the tablets to the tsfile device by device, and the tablets + // in the same device are written in order of start time. Tablets in + // the same device should not be written if their time ranges overlap. + // If overlapped, we try to write the tablets whose device id is not + // the same as the previous one. For the tablets not written in the + // previous round, we write them in a new tsfile. + final List> sealedFiles = new ArrayList<>(); + + // Try making the tsfile size as large as possible + while (!device2TabletsLinkedList.isEmpty()) { + if (Objects.isNull(fileWriter)) { + createFileWriter(); + } + try { + tryBestToWriteTabletsIntoOneFile(device2TabletsLinkedList, device2Aligned); + } catch (final Exception e) { + LOGGER.warn( + "Batch id = {}: Failed to write tablets into tsfile, because {}", + currentBatchId.get(), + e.getMessage(), + e); + + try { + fileWriter.close(); + } catch (final Exception closeException) { + LOGGER.warn( + "Batch id = {}: Failed to close the tsfile {} after failed to write tablets into, because {}", + currentBatchId.get(), + fileWriter.getIOWriter().getFile().getPath(), + closeException.getMessage(), + closeException); + } finally { + // Add current writing file to the list and delete the file + sealedFiles.add(new Pair<>(null, fileWriter.getIOWriter().getFile())); + } + + for (final Pair sealedFile : sealedFiles) { + final boolean deleteSuccess = FileUtils.deleteQuietly(sealedFile.right); + LOGGER.warn( + "Batch id = {}: {} delete the tsfile {} after failed to write tablets into {}. {}", + currentBatchId.get(), + deleteSuccess ? "Successfully" : "Failed to", + sealedFile.right.getPath(), + fileWriter.getIOWriter().getFile().getPath(), + deleteSuccess ? "" : "Maybe the tsfile needs to be deleted manually."); + } + sealedFiles.clear(); + + fileWriter = null; + + throw e; + } + + fileWriter.close(); + final File sealedFile = fileWriter.getIOWriter().getFile(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Batch id = {}: Seal tsfile {} successfully.", + currentBatchId.get(), + sealedFile.getPath()); + } + sealedFiles.add(new Pair<>(null, sealedFile)); + fileWriter = null; + } + + return sealedFiles; + } + + private void tryBestToWriteTabletsIntoOneFile( + final LinkedHashMap> device2TabletsLinkedList, + final Map device2Aligned) + throws IOException, WriteProcessException { + final Iterator>> iterator = + device2TabletsLinkedList.entrySet().iterator(); + + while (iterator.hasNext()) { + final Map.Entry> entry = iterator.next(); + final String deviceId = entry.getKey(); + final LinkedList tablets = entry.getValue(); + + final List tabletsToWrite = new ArrayList<>(); + + Tablet lastTablet = null; + while (!tablets.isEmpty()) { + final Tablet tablet = tablets.peekFirst(); + if (Objects.isNull(lastTablet) + // lastTablet.rowSize is not 0 + || lastTablet.getTimestamp(lastTablet.getRowSize() - 1) < tablet.getTimestamp(0)) { + tabletsToWrite.add(tablet); + lastTablet = tablet; + tablets.pollFirst(); + } else { + break; + } + } + + if (tablets.isEmpty()) { + iterator.remove(); + } + + final boolean isAligned = device2Aligned.get(deviceId); + if (isAligned) { + final Map> deviceId2MeasurementSchemas = new HashMap<>(); + tabletsToWrite.forEach( + tablet -> + deviceId2MeasurementSchemas.compute( + tablet.getDeviceId(), + (k, v) -> { + if (Objects.isNull(v)) { + return new ArrayList<>(tablet.getSchemas()); + } + v.addAll(tablet.getSchemas()); + return v; + })); + for (final Map.Entry> deviceIdWithMeasurementSchemas : + deviceId2MeasurementSchemas.entrySet()) { + fileWriter.registerAlignedTimeseries( + new Path(deviceIdWithMeasurementSchemas.getKey()), + deviceIdWithMeasurementSchemas.getValue()); + } + for (final Tablet tablet : tabletsToWrite) { + fileWriter.writeAligned(tablet); + } + } else { + for (final Tablet tablet : tabletsToWrite) { + for (final IMeasurementSchema schema : tablet.getSchemas()) { + try { + fileWriter.registerTimeseries( + IDeviceID.Factory.DEFAULT_FACTORY.create(tablet.getDeviceId()), schema); + } catch (final WriteProcessException ignore) { + // Do nothing if the timeSeries has been registered + } + } + + fileWriter.writeTree(tablet); + } + } + } + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/builder/PipeTsFileBuilder.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/builder/PipeTsFileBuilder.java new file mode 100644 index 00000000..e4b45065 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/builder/PipeTsFileBuilder.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.utils.builder; + +import org.apache.iotdb.collector.config.PipeOptions; +import org.apache.iotdb.pipe.api.exception.PipeException; + +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +public abstract class PipeTsFileBuilder { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileBuilder.class); + + // private static final AtomicReference FOLDER_MANAGER = new AtomicReference<>(); + protected final AtomicLong currentBatchId; + // private final File batchFileBaseDir; + + private static final String TS_FILE_PREFIX = "tb"; // tb means tablet batch + private final AtomicLong tsFileIdGenerator; + + @SuppressWarnings("java:S3077") + protected volatile TsFileWriter fileWriter; + + public PipeTsFileBuilder(final AtomicLong currentBatchId, final AtomicLong tsFileIdGenerator) { + this.currentBatchId = currentBatchId; + this.tsFileIdGenerator = tsFileIdGenerator; + try { + // this.batchFileBaseDir = getNextBaseDir(); + } catch (final Exception e) { + throw new PipeException( + String.format("Failed to create file dir for batch: %s", e.getMessage())); + } + } + + // private File getNextBaseDir() throws DiskSpaceInsufficientException { + // if (FOLDER_MANAGER.get() == null) { + // synchronized (FOLDER_MANAGER) { + // if (FOLDER_MANAGER.get() == null) { + // FOLDER_MANAGER.set( + // new FolderManager( + // Arrays.stream(PipeOptions.PIPE_RECEIVER_FILE_DIRS.value()) + // .map(fileDir -> fileDir + File.separator + ".batch") + // .collect(Collectors.toList()), + // DirectoryStrategyType.SEQUENCE_STRATEGY)); + // } + // } + // } + // + // final File baseDir = + // new File(FOLDER_MANAGER.get().getNextFolder(), Long.toString(currentBatchId.get())); + // if (baseDir.exists()) { + // FileUtils.deleteQuietly(baseDir); + // } + // if (!baseDir.exists() && !baseDir.mkdirs()) { + // LOGGER.warn( + // "Batch id = {}: Failed to create batch file dir {}.", + // currentBatchId.get(), + // baseDir.getPath()); + // throw new PipeException( + // String.format( + // "Failed to create batch file dir %s. (Batch id = %s)", + // baseDir.getPath(), currentBatchId.get())); + // } + // LOGGER.info( + // "Batch id = {}: Create batch dir successfully, batch file dir = {}.", + // currentBatchId.get(), + // baseDir.getPath()); + // return baseDir; + // } + + public abstract void bufferTableModelTablet(String dataBase, Tablet tablet); + + public abstract void bufferTreeModelTablet(Tablet tablet, Boolean isAligned); + + public abstract List> convertTabletToTsFileWithDBInfo() + throws IOException, WriteProcessException; + + public abstract boolean isEmpty(); + + public synchronized void onSuccess() { + fileWriter = null; + } + + public synchronized void close() { + if (Objects.nonNull(fileWriter)) { + try { + fileWriter.close(); + } catch (final Exception e) { + LOGGER.info( + "Batch id = {}: Failed to close the tsfile {} when trying to close batch, because {}", + currentBatchId.get(), + fileWriter.getIOWriter().getFile().getPath(), + e.getMessage(), + e); + } + + try { + FileUtils.delete(fileWriter.getIOWriter().getFile()); + } catch (final Exception e) { + LOGGER.info( + "Batch id = {}: Failed to delete the tsfile {} when trying to close batch, because {}", + currentBatchId.get(), + fileWriter.getIOWriter().getFile().getPath(), + e.getMessage(), + e); + } + + fileWriter = null; + } + } + + protected void createFileWriter() throws IOException { + fileWriter = + new TsFileWriter( + new File( + // batchFileBaseDir, + TS_FILE_PREFIX + + "_" + + PipeOptions.DATA_NODE_ID.value() + + "_" + + currentBatchId.get() + + "_" + + tsFileIdGenerator.getAndIncrement() + + TsFileConstant.TSFILE_SUFFIX)); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/cacher/LeaderCacheUtils.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/cacher/LeaderCacheUtils.java new file mode 100644 index 00000000..3a8a1dfd --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/cacher/LeaderCacheUtils.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.utils.cacher; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.utils.Pair; + +import java.util.ArrayList; +import java.util.List; + +public class LeaderCacheUtils { + + private LeaderCacheUtils() { + // Do nothing + } + + /** + * Get all redirection recommends after transferring a batch event to update leader cache. + * + * @param status is the returned status after transferring a batch event. + * @return a list of pairs, each pair contains a device path and its redirect endpoint. + */ + public static List> parseRecommendedRedirections(TSStatus status) { + // If there is no exception, there should be 2 sub-statuses, one for InsertRowsStatement and one + // for InsertMultiTabletsStatement (see IoTDBDataNodeReceiver#handleTransferTabletBatch). + final List> redirectList = new ArrayList<>(); + + if (status.getSubStatusSize() != 2) { + return redirectList; + } + + for (final TSStatus subStatus : status.getSubStatus()) { + if (subStatus.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { + continue; + } + + for (final TSStatus innerSubStatus : subStatus.getSubStatus()) { + if (innerSubStatus.isSetRedirectNode()) { + // We assume that innerSubStatus.getMessage() is a device path. + // The message field should be a device path. + redirectList.add( + new Pair<>(innerSubStatus.getMessage(), innerSubStatus.getRedirectNode())); + } + } + } + + return redirectList; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/sorter/PipeTableModelTabletEventSorter.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/sorter/PipeTableModelTabletEventSorter.java new file mode 100644 index 00000000..e1694738 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/sorter/PipeTableModelTabletEventSorter.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.utils.sorter; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class PipeTableModelTabletEventSorter { + + private final Tablet tablet; + + private Integer[] index; + private boolean isUnSorted = false; + private boolean hasDuplicates = false; + private int deduplicatedSize; + private int initIndexSize; + + public PipeTableModelTabletEventSorter(final Tablet tablet) { + this.tablet = tablet; + deduplicatedSize = tablet == null ? 0 : tablet.getRowSize(); + } + + /** + * For the sorting and deduplication needs of the table model tablet, it is done according to the + * {@link IDeviceID}. For sorting, it is necessary to sort the {@link IDeviceID} first, and then + * sort by time. Deduplication is to remove the same timestamp in the same {@link IDeviceID}, and + * the same timestamp in different {@link IDeviceID} will not be processed. + */ + public void sortAndDeduplicateByDevIdTimestamp() { + if (tablet == null || tablet.getRowSize() < 1) { + return; + } + + HashMap>> deviceIDToIndexMap = new HashMap<>(); + final long[] timestamps = tablet.getTimestamps(); + + IDeviceID lastDevice = tablet.getDeviceID(0); + long previousTimestamp = tablet.getTimestamp(0); + int lasIndex = 0; + for (int i = 1, size = tablet.getRowSize(); i < size; ++i) { + final IDeviceID deviceID = tablet.getDeviceID(i); + final long currentTimestamp = timestamps[i]; + final int deviceComparison = deviceID.compareTo(lastDevice); + if (deviceComparison == 0) { + if (previousTimestamp == currentTimestamp) { + hasDuplicates = true; + continue; + } + if (previousTimestamp > currentTimestamp) { + isUnSorted = true; + } + previousTimestamp = currentTimestamp; + continue; + } + if (deviceComparison < 0) { + isUnSorted = true; + } + + List> list = + deviceIDToIndexMap.computeIfAbsent(lastDevice, k -> new ArrayList<>()); + + if (!list.isEmpty()) { + isUnSorted = true; + } + list.add(new Pair<>(lasIndex, i)); + lastDevice = deviceID; + lasIndex = i; + previousTimestamp = currentTimestamp; + } + + List> list = + deviceIDToIndexMap.computeIfAbsent(lastDevice, k -> new ArrayList<>()); + if (!list.isEmpty()) { + isUnSorted = true; + } + list.add(new Pair<>(lasIndex, tablet.getRowSize())); + + if (!isUnSorted && !hasDuplicates) { + return; + } + + initIndexSize = 0; + deduplicatedSize = 0; + index = new Integer[tablet.getRowSize()]; + deviceIDToIndexMap.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach( + entry -> { + final int start = initIndexSize; + int i = initIndexSize; + for (Pair pair : entry.getValue()) { + for (int j = pair.left; j < pair.right; j++) { + index[i++] = j; + } + } + if (isUnSorted) { + sortTimestamps(start, i); + deduplicateTimestamps(start, i); + initIndexSize = i; + return; + } + + if (hasDuplicates) { + deduplicateTimestamps(start, i); + } + initIndexSize = i; + }); + + sortAndDeduplicateValuesAndBitMaps(); + } + + private void sortAndDeduplicateValuesAndBitMaps() { + int columnIndex = 0; + tablet.setTimestamps( + (long[]) + PipeTabletEventSorter.reorderValueList( + deduplicatedSize, tablet.getTimestamps(), TSDataType.TIMESTAMP, index)); + for (int i = 0, size = tablet.getSchemas().size(); i < size; i++) { + final IMeasurementSchema schema = tablet.getSchemas().get(i); + if (schema != null) { + tablet.getValues()[columnIndex] = + PipeTabletEventSorter.reorderValueList( + deduplicatedSize, tablet.getValues()[columnIndex], schema.getType(), index); + if (tablet.getBitMaps() != null && tablet.getBitMaps()[columnIndex] != null) { + tablet.getBitMaps()[columnIndex] = + PipeTabletEventSorter.reorderBitMap( + deduplicatedSize, tablet.getBitMaps()[columnIndex], index); + } + columnIndex++; + } + } + + tablet.setRowSize(deduplicatedSize); + } + + private void sortTimestamps(final int startIndex, final int endIndex) { + Arrays.sort(this.index, startIndex, endIndex, Comparator.comparingLong(tablet::getTimestamp)); + } + + private void deduplicateTimestamps(final int startIndex, final int endIndex) { + long[] timestamps = tablet.getTimestamps(); + long lastTime = timestamps[index[startIndex]]; + index[deduplicatedSize++] = index[startIndex]; + for (int i = startIndex + 1; i < endIndex; i++) { + if (lastTime != (lastTime = timestamps[index[i]])) { + index[deduplicatedSize++] = index[i]; + } + } + } + + /** Sort by time only, and remove only rows with the same DeviceID and time. */ + public void sortAndDeduplicateByTimestampIfNecessary() { + if (tablet == null || tablet.getRowSize() == 0) { + return; + } + + long[] timestamps = tablet.getTimestamps(); + for (int i = 1, size = tablet.getRowSize(); i < size; ++i) { + final long currentTimestamp = timestamps[i]; + final long previousTimestamp = timestamps[i - 1]; + + if (currentTimestamp < previousTimestamp) { + isUnSorted = true; + break; + } + if (currentTimestamp == previousTimestamp) { + hasDuplicates = true; + } + } + + if (!isUnSorted && !hasDuplicates) { + return; + } + + index = new Integer[tablet.getRowSize()]; + for (int i = 0, size = tablet.getRowSize(); i < size; i++) { + index[i] = i; + } + + if (isUnSorted) { + sortTimestamps(); + + // Do deduplicate anyway. + // isDeduplicated may be false positive when isUnSorted is true. + deduplicateTimestamps(); + hasDuplicates = false; + } + + if (hasDuplicates) { + deduplicateTimestamps(); + } + + sortAndDeduplicateValuesAndBitMapsIgnoreTimestamp(); + } + + private void sortTimestamps() { + Arrays.sort(this.index, Comparator.comparingLong(tablet::getTimestamp)); + Arrays.sort(tablet.getTimestamps(), 0, tablet.getRowSize()); + } + + private void deduplicateTimestamps() { + deduplicatedSize = 1; + long[] timestamps = tablet.getTimestamps(); + long lastTime = timestamps[0]; + IDeviceID deviceID = tablet.getDeviceID(index[0]); + final Set deviceIDSet = new HashSet<>(); + deviceIDSet.add(deviceID); + for (int i = 1, size = tablet.getRowSize(); i < size; i++) { + deviceID = tablet.getDeviceID(index[i]); + if ((lastTime == (lastTime = timestamps[i]))) { + if (!deviceIDSet.contains(deviceID)) { + timestamps[deduplicatedSize] = lastTime; + index[deduplicatedSize++] = index[i]; + deviceIDSet.add(deviceID); + } + } else { + timestamps[deduplicatedSize] = lastTime; + index[deduplicatedSize++] = index[i]; + deviceIDSet.clear(); + deviceIDSet.add(deviceID); + } + } + } + + private void sortAndDeduplicateValuesAndBitMapsIgnoreTimestamp() { + int columnIndex = 0; + for (int i = 0, size = tablet.getSchemas().size(); i < size; i++) { + final IMeasurementSchema schema = tablet.getSchemas().get(i); + if (schema != null) { + tablet.getValues()[columnIndex] = + PipeTabletEventSorter.reorderValueList( + deduplicatedSize, tablet.getValues()[columnIndex], schema.getType(), index); + if (tablet.getBitMaps() != null && tablet.getBitMaps()[columnIndex] != null) { + tablet.getBitMaps()[columnIndex] = + PipeTabletEventSorter.reorderBitMap( + deduplicatedSize, tablet.getBitMaps()[columnIndex], index); + } + columnIndex++; + } + } + + tablet.setRowSize(deduplicatedSize); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/sorter/PipeTabletEventSorter.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/sorter/PipeTabletEventSorter.java new file mode 100644 index 00000000..05597347 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/sorter/PipeTabletEventSorter.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.utils.sorter; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +import java.time.LocalDate; + +public class PipeTabletEventSorter { + + public static Object reorderValueList( + final int deduplicatedSize, + final Object valueList, + final TSDataType dataType, + final Integer[] index) { + switch (dataType) { + case BOOLEAN: + final boolean[] boolValues = (boolean[]) valueList; + final boolean[] deduplicatedBoolValues = new boolean[boolValues.length]; + for (int i = 0; i < deduplicatedSize; i++) { + deduplicatedBoolValues[i] = boolValues[index[i]]; + } + return deduplicatedBoolValues; + case INT32: + final int[] intValues = (int[]) valueList; + final int[] deduplicatedIntValues = new int[intValues.length]; + for (int i = 0; i < deduplicatedSize; i++) { + deduplicatedIntValues[i] = intValues[index[i]]; + } + return deduplicatedIntValues; + case DATE: + final LocalDate[] dateValues = (LocalDate[]) valueList; + final LocalDate[] deduplicatedDateValues = new LocalDate[dateValues.length]; + for (int i = 0; i < deduplicatedSize; i++) { + deduplicatedDateValues[i] = dateValues[index[i]]; + } + return deduplicatedDateValues; + case INT64: + case TIMESTAMP: + final long[] longValues = (long[]) valueList; + final long[] deduplicatedLongValues = new long[longValues.length]; + for (int i = 0; i < deduplicatedSize; i++) { + deduplicatedLongValues[i] = longValues[index[i]]; + } + return deduplicatedLongValues; + case FLOAT: + final float[] floatValues = (float[]) valueList; + final float[] deduplicatedFloatValues = new float[floatValues.length]; + for (int i = 0; i < deduplicatedSize; i++) { + deduplicatedFloatValues[i] = floatValues[index[i]]; + } + return deduplicatedFloatValues; + case DOUBLE: + final double[] doubleValues = (double[]) valueList; + final double[] deduplicatedDoubleValues = new double[doubleValues.length]; + for (int i = 0; i < deduplicatedSize; i++) { + deduplicatedDoubleValues[i] = doubleValues[index[i]]; + } + return deduplicatedDoubleValues; + case TEXT: + case BLOB: + case STRING: + final Binary[] binaryValues = (Binary[]) valueList; + final Binary[] deduplicatedBinaryValues = new Binary[binaryValues.length]; + for (int i = 0; i < deduplicatedSize; i++) { + deduplicatedBinaryValues[i] = binaryValues[index[i]]; + } + return deduplicatedBinaryValues; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", dataType)); + } + } + + public static BitMap reorderBitMap( + final int deduplicatedSize, final BitMap bitMap, final Integer[] index) { + final BitMap deduplicatedBitMap = new BitMap(bitMap.getSize()); + for (int i = 0; i < deduplicatedSize; i++) { + if (bitMap.isMarked(index[i])) { + deduplicatedBitMap.mark(i); + } + } + return deduplicatedBitMap; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/sorter/PipeTreeModelTabletEventSorter.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/sorter/PipeTreeModelTabletEventSorter.java new file mode 100644 index 00000000..7dee7093 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/utils/sorter/PipeTreeModelTabletEventSorter.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink.utils.sorter; + +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; + +import java.util.Arrays; +import java.util.Comparator; + +public class PipeTreeModelTabletEventSorter { + + private final Tablet tablet; + + private boolean isSorted = true; + private boolean isDeduplicated = true; + + private Integer[] index; + private int deduplicatedSize; + + public PipeTreeModelTabletEventSorter(final Tablet tablet) { + this.tablet = tablet; + deduplicatedSize = tablet == null ? 0 : tablet.getRowSize(); + } + + public void deduplicateAndSortTimestampsIfNecessary() { + if (tablet == null || tablet.getRowSize() == 0) { + return; + } + + long[] timestamps = tablet.getTimestamps(); + for (int i = 1, size = tablet.getRowSize(); i < size; ++i) { + final long currentTimestamp = timestamps[i]; + final long previousTimestamp = timestamps[i - 1]; + + if (currentTimestamp < previousTimestamp) { + isSorted = false; + break; + } + if (currentTimestamp == previousTimestamp) { + isDeduplicated = false; + } + } + + if (isSorted && isDeduplicated) { + return; + } + + index = new Integer[tablet.getRowSize()]; + for (int i = 0, size = tablet.getRowSize(); i < size; i++) { + index[i] = i; + } + + if (!isSorted) { + sortTimestamps(); + + // Do deduplicate anyway. + // isDeduplicated may be false positive when isSorted is false. + deduplicateTimestamps(); + isDeduplicated = true; + } + + if (!isDeduplicated) { + deduplicateTimestamps(); + } + + sortAndDeduplicateValuesAndBitMaps(); + } + + private void sortTimestamps() { + Arrays.sort(index, Comparator.comparingLong(tablet::getTimestamp)); + Arrays.sort(tablet.getTimestamps(), 0, tablet.getRowSize()); + } + + private void deduplicateTimestamps() { + deduplicatedSize = 1; + long[] timestamps = tablet.getTimestamps(); + for (int i = 1, size = tablet.getRowSize(); i < size; i++) { + if (timestamps[i] != timestamps[i - 1]) { + index[deduplicatedSize] = index[i]; + timestamps[deduplicatedSize] = timestamps[i]; + + ++deduplicatedSize; + } + } + tablet.setRowSize(deduplicatedSize); + } + + private void sortAndDeduplicateValuesAndBitMaps() { + int columnIndex = 0; + for (int i = 0, size = tablet.getSchemas().size(); i < size; i++) { + final IMeasurementSchema schema = tablet.getSchemas().get(i); + if (schema != null) { + tablet.getValues()[columnIndex] = + PipeTabletEventSorter.reorderValueList( + deduplicatedSize, tablet.getValues()[columnIndex], schema.getType(), index); + if (tablet.getBitMaps() != null && tablet.getBitMaps()[columnIndex] != null) { + tablet.getBitMaps()[columnIndex] = + PipeTabletEventSorter.reorderBitMap( + deduplicatedSize, tablet.getBitMaps()[columnIndex], index); + } + columnIndex++; + } + } + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/plugin/constructor/SinkConstructor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/plugin/constructor/SinkConstructor.java index 066cf08e..5e24cfd5 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/plugin/constructor/SinkConstructor.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/plugin/constructor/SinkConstructor.java @@ -20,7 +20,7 @@ package org.apache.iotdb.collector.runtime.plugin.constructor; import org.apache.iotdb.collector.plugin.builtin.BuiltinPlugin; -import org.apache.iotdb.collector.plugin.builtin.sink.DemoSink; +import org.apache.iotdb.collector.plugin.builtin.sink.protocol.DemoSink; import org.apache.iotdb.collector.runtime.plugin.meta.PluginMetaKeeper; import org.apache.iotdb.pipe.api.PipeSink; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/processor/ProcessorTask.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/processor/ProcessorTask.java index e671ea00..dad94046 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/processor/ProcessorTask.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/processor/ProcessorTask.java @@ -25,8 +25,6 @@ import org.apache.iotdb.collector.runtime.task.event.EventCollector; import org.apache.iotdb.collector.runtime.task.event.EventContainer; import org.apache.iotdb.collector.service.RuntimeService; -import org.apache.iotdb.commons.concurrent.IoTThreadFactory; -import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import com.lmax.disruptor.BlockingWaitStrategy; @@ -39,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import static org.apache.iotdb.collector.config.TaskRuntimeOptions.TASK_PROCESSOR_RING_BUFFER_SIZE; @@ -67,14 +66,12 @@ public ProcessorTask( REGISTERED_EXECUTOR_SERVICES.putIfAbsent( taskId, - new WrappedThreadPoolExecutor( + new ThreadPoolExecutor( parallelism, parallelism, 0L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(parallelism), - new IoTThreadFactory(taskId), // TODO: thread name - taskId)); + new LinkedBlockingQueue<>(parallelism))); disruptor = new Disruptor<>( diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/sink/SinkTask.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/sink/SinkTask.java index e3e66db9..7e66d9e0 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/sink/SinkTask.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/sink/SinkTask.java @@ -25,8 +25,6 @@ import org.apache.iotdb.collector.runtime.task.event.EventCollector; import org.apache.iotdb.collector.runtime.task.event.EventContainer; import org.apache.iotdb.collector.service.RuntimeService; -import org.apache.iotdb.commons.concurrent.IoTThreadFactory; -import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import com.lmax.disruptor.BlockingWaitStrategy; @@ -39,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import static org.apache.iotdb.collector.config.TaskRuntimeOptions.TASK_SINK_PARALLELISM_NUM; @@ -59,14 +58,12 @@ public SinkTask(final String taskId, final Map attributes) { REGISTERED_EXECUTOR_SERVICES.putIfAbsent( taskId, - new WrappedThreadPoolExecutor( + new ThreadPoolExecutor( parallelism, parallelism, 0L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(parallelism), - new IoTThreadFactory(taskId), // TODO: thread name - taskId)); + new LinkedBlockingQueue<>(parallelism))); disruptor = new Disruptor<>( diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/source/pull/PullSourceTask.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/source/pull/PullSourceTask.java index b47054a9..f4857433 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/source/pull/PullSourceTask.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/runtime/task/source/pull/PullSourceTask.java @@ -25,8 +25,6 @@ import org.apache.iotdb.collector.runtime.task.event.EventCollector; import org.apache.iotdb.collector.runtime.task.source.SourceTask; import org.apache.iotdb.collector.service.RuntimeService; -import org.apache.iotdb.commons.concurrent.IoTThreadFactory; -import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.slf4j.Logger; @@ -36,6 +34,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; public class PullSourceTask extends SourceTask { @@ -64,14 +63,12 @@ public void createInternal() throws Exception { REGISTERED_EXECUTOR_SERVICES.putIfAbsent( taskId, - new WrappedThreadPoolExecutor( + new ThreadPoolExecutor( parallelism, parallelism, 0L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(parallelism), - new IoTThreadFactory(taskId), // TODO: thread name - taskId)); + new LinkedBlockingQueue<>(parallelism))); final long creationTime = System.currentTimeMillis(); consumers = new PullSourceConsumer[parallelism]; diff --git a/pom.xml b/pom.xml index 44237084..751cd5a6 100644 --- a/pom.xml +++ b/pom.xml @@ -183,6 +183,7 @@ 0.14.1 9.0.86 + 2.1.0-250219-SNAPSHOT 1.9 0.11.1 1.5.5-5 @@ -724,6 +725,33 @@ + + org.eclipse.milo + stack-core + ${milo.version} + + + com.sun.activation + jakarta.activation + + + + + org.eclipse.milo + sdk-core + ${milo.version} + + + com.sun.activation + jakarta.activation + + + + + org.eclipse.milo + stack-server + ${milo.version} + org.eclipse.milo sdk-server @@ -760,6 +788,11 @@ logback-classic ${logback.version} + + ch.qos.logback + logback-core + ${logback.version} + com.rabbitmq amqp-client @@ -861,6 +894,56 @@ disruptor ${disruptor.version} + + org.apache.commons + commons-collections4 + ${commons.collections4.version} + + + org.apache.commons + commons-pool2 + ${commons-pool2.version} + + + org.apache.tsfile + tsfile + ${tsfile.version} + + + org.apache.tsfile + common + ${tsfile.version} + + + net.java.dev.jna + jna + ${jna.version} + + + net.java.dev.jna + jna-platform + ${jna.version} + + + commons-io + commons-io + ${commons-io.version} + + + com.github.luben + zstd-jni + ${zstd-jni.version} + + + com.github.ben-manes.caffeine + caffeine + ${caffeine.version} + + + io.moquette + moquette-broker + ${moquette.version} +