You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2025/04/24 04:21:15 UTC
(iotdb-extras) branch master updated: Collector: Add collector-distribution packaging module & Migrate IoTDBDataRegionSyncConnector & Add scheduled triggering for batch accumulation operations & Add subscribe source demo & Update iotdb dependence version (#61)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb-extras.git
The following commit(s) were added to refs/heads/master by this push:
new 85e8b44 Collector: Add collector-distribution packaging module & Migrate IoTDBDataRegionSyncConnector & Add scheduled triggering for batch accumulation operations & Add subscribe source demo & Update iotdb dependence version (#61)
85e8b44 is described below
commit 85e8b44e8c0129b2c420b34069cfbff2af83d40f
Author: 0xB <76...@users.noreply.github.com>
AuthorDate: Thu Apr 24 12:21:11 2025 +0800
Collector: Add collector-distribution packaging module & Migrate IoTDBDataRegionSyncConnector & Add scheduled triggering for batch accumulation operations & Add subscribe source demo & Update iotdb dependence version (#61)
---
.github/workflows/code-analysis.yml | 2 +-
.github/workflows/compile-check.yml | 2 +-
iotdb-collector/collector-core/pom.xml | 141 +++++-
.../src/assembly/core.xml} | 34 +-
.../assembly/resources/conf/application.properties | 154 +++++++
.../src/assembly/resources/conf/logback.xml | 126 ++++++
.../src/assembly/resources/sbin/collector-env.sh | 94 ++++
.../src/assembly/resources/sbin/common.sh | 181 ++++++++
.../assembly/resources/sbin/start-collector.bat | 208 +++++++++
.../src/assembly/resources/sbin/start-collector.sh | 237 +++++++++++
.../src/assembly/resources/sbin/stop-collector.bat | 67 +++
.../src/assembly/resources/sbin/stop-collector.sh | 86 ++++
.../org/apache/iotdb/collector/Application.java | 2 +
.../iotdb/collector/config/Configuration.java | 45 +-
.../org/apache/iotdb/collector/config/Options.java | 13 +
.../iotdb/collector/config/PipeRuntimeOptions.java | 283 ++++++++++++
.../collector/config/PluginRuntimeOptions.java | 16 +-
.../iotdb/collector/config/TaskRuntimeOptions.java | 16 +-
.../iotdb/collector/persistence/DBConstant.java | 13 +-
.../iotdb/collector/persistence/Persistence.java | 1 +
.../collector/persistence/PluginPersistence.java | 6 +-
.../collector/persistence/TaskPersistence.java | 11 +-
.../iotdb/collector/plugin/api/PullSource.java | 12 +-
.../iotdb/collector/plugin/api/PushSource.java | 10 +
.../customizer/CollectorRuntimeEnvironment.java | 5 +
.../PeriodicalEvent.java} | 6 +-
.../collector/plugin/builtin/BuiltinPlugin.java | 8 +-
.../builtin/processor/SubscriptionProcessor.java | 56 +++
.../builtin/sink/client/IoTDBClientManager.java | 114 +++++
.../IoTDBDataNodeCacheLeaderClientManager.java | 79 ++++
.../client/IoTDBDataNodeSyncClientManager.java | 130 ++++++
.../builtin/sink/client/IoTDBSyncClient.java | 180 ++++++++
.../sink/client/IoTDBSyncClientManager.java | 373 ++++++++++++++++
.../plugin/builtin/sink/client/ThriftClient.java | 124 ++++++
.../builtin/sink/client/ThriftClientProperty.java | 123 ++++++
.../builtin/sink/compressor/PipeCompressor.java | 76 ++++
.../sink/compressor/PipeCompressorConfig.java} | 23 +-
.../sink/compressor/PipeCompressorFactory.java | 116 +++++
.../sink/compressor/PipeGZIPCompressor.java | 54 +++
.../builtin/sink/compressor/PipeLZ4Compressor.java | 54 +++
.../sink/compressor/PipeLZMA2Compressor.java | 54 +++
.../sink/compressor/PipeSnappyCompressor.java | 54 +++
.../sink/compressor/PipeZSTDCompressor.java | 49 +++
.../sink/constant/ColumnHeaderConstant.java} | 17 +-
.../sink/constant/PipeConnectorConstant.java | 261 ++++++++++++
.../constant/PipeTransferHandshakeConstant.java} | 34 +-
.../sink/event/PipeInsertionEvent.java} | 22 +-
.../sink/event/PipeRawTabletInsertionEvent.java | 81 ++++
.../sink/event/PipeTsFileInsertionEvent.java | 58 +++
.../PipeRuntimeConnectorCriticalException.java | 103 +++++
.../exception/PipeRuntimeCriticalException.java | 96 +++++
.../sink/exception/PipeRuntimeException.java | 54 +++
.../sink/exception/PipeRuntimeExceptionType.java | 89 ++++
.../sink/exception/PipeRuntimeMetaVersion.java | 77 ++++
.../exception/PipeRuntimeNonCriticalException.java | 101 +++++
.../PipeRuntimeOutOfMemoryCriticalException.java | 104 +++++
.../builtin/sink/exception/PipeStatus.java} | 41 +-
.../evolvable/batch/PipeTabletEventBatch.java | 114 +++++
.../evolvable/batch/PipeTabletEventPlainBatch.java | 119 ++++++
.../batch/PipeTabletEventTsFileBatch.java | 170 ++++++++
.../batch/PipeTransferBatchReqBuilder.java | 179 ++++++++
.../request/IoTDBConnectorRequestVersion.java} | 18 +-
.../payload/thrift/request/PipeRequestType.java | 90 ++++
.../thrift/request/PipeTransferCompressedReq.java | 150 +++++++
.../PipeTransferDataNodeHandshakeV1Req.java | 69 +++
.../PipeTransferDataNodeHandshakeV2Req.java | 69 +++
.../thrift/request/PipeTransferFilePieceReq.java | 117 +++++
.../thrift/request/PipeTransferFileSealReqV1.java | 107 +++++
.../thrift/request/PipeTransferFileSealReqV2.java | 163 +++++++
.../thrift/request/PipeTransferHandshakeV1Req.java | 100 +++++
.../thrift/request/PipeTransferHandshakeV2Req.java | 116 +++++
.../PipeTransferSchemaSnapshotPieceReq.java | 71 ++++
.../request/PipeTransferSchemaSnapshotSealReq.java | 166 ++++++++
.../thrift/request/PipeTransferSliceReq.java | 147 +++++++
.../request/PipeTransferTabletBatchReqV2.java | 148 +++++++
.../request/PipeTransferTabletBinaryReq.java | 98 +++++
.../request/PipeTransferTabletBinaryReqV2.java | 126 ++++++
.../thrift/request/PipeTransferTabletRawReq.java | 126 ++++++
.../thrift/request/PipeTransferTabletRawReqV2.java | 121 ++++++
.../thrift/request/PipeTransferTsFilePieceReq.java | 70 +++
.../request/PipeTransferTsFilePieceWithModReq.java | 71 ++++
.../request/PipeTransferTsFileSealWithModReq.java | 142 +++++++
.../thrift/response/PipeTransferFilePieceResp.java | 74 ++++
.../builtin/sink/protocol/IoTDBConnector.java | 472 +++++++++++++++++++++
.../protocol/IoTDBDataRegionSyncConnector.java | 365 ++++++++++++++++
.../sink/protocol/IoTDBSslSyncConnector.java | 229 ++++++++++
.../sink/protocol/PipeReceiverStatusHandler.java | 199 +++++++++
.../sink/protocol/session/IClientSession.java | 75 ++++
.../plugin/builtin/source/HttpPullSource.java | 6 -
.../plugin/builtin/source/HttpPushSource.java | 5 -
.../plugin/builtin/source/IoTDBPushSource.java | 129 ++++++
.../source/constant/IoTDBPushSourceConstant.java} | 39 +-
.../plugin/builtin/source/event/SubDemoEvent.java | 44 ++
.../event/common/PipeBinaryTransformer.java} | 30 +-
.../event/common/PipeDataTypeTransformer.java | 74 ++++
.../source/event/common/PipeResetTabletRow.java | 54 +++
.../builtin/source/event/common/PipeRow.java | 211 +++++++++
.../source/event/common/PipeRowCollector.java | 103 +++++
.../collector/runtime/plugin/PluginRuntime.java | 2 +-
.../plugin/constructor/ProcessorConstructor.java | 3 +
.../plugin/constructor/SinkConstructor.java | 7 +-
.../plugin/constructor/SourceConstructor.java | 2 +
.../iotdb/collector/runtime/task/TaskRuntime.java | 2 +
.../runtime/task/processor/ProcessorTask.java | 23 +-
.../collector/runtime/task/sink/SinkTask.java | 9 +-
.../runtime/task/source/pull/PullSourceTask.java | 9 +-
.../collector/service/PeriodicalJobService.java | 129 ++++++
.../api/PullSource.java => utils/PathUtil.java} | 6 +-
.../collector/utils/PipeMemoryWeightUtil.java | 304 +++++++++++++
.../plugin => }/utils/PluginFileUtils.java | 2 +-
.../utils/builder/PipeTableModeTsFileBuilder.java | 273 ++++++++++++
.../utils/builder/PipeTreeModelTsFileBuilder.java | 268 ++++++++++++
.../collector/utils/builder/PipeTsFileBuilder.java | 112 +++++
.../collector/utils/cacher/LeaderCacheUtils.java | 69 +++
.../utils/preiodical/ScheduledExecutorUtil.java | 195 +++++++++
.../preiodical/WrappedRunnable.java} | 41 +-
.../sorter/PipeTableModelTabletEventSorter.java | 273 ++++++++++++
.../utils/sorter/PipeTabletEventSorter.java | 105 +++++
.../sorter/PipeTreeModelTabletEventSorter.java | 123 ++++++
.../src/main/resources/application.properties | 70 ---
.../collector-core/src/main/resources/logback.xml | 44 --
iotdb-collector/collector-distribution/pom.xml | 93 ++++
.../src/assembly/collector-core.xml | 77 ++++
iotdb-collector/pom.xml | 3 +-
pom.xml | 114 +++++
125 files changed, 11592 insertions(+), 318 deletions(-)
diff --git a/.github/workflows/code-analysis.yml b/.github/workflows/code-analysis.yml
index b8f15bc..f890970 100644
--- a/.github/workflows/code-analysis.yml
+++ b/.github/workflows/code-analysis.yml
@@ -34,7 +34,7 @@ jobs:
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
- queries: +security-extended
+ queries: +security-and-quality
- name: Auto Build
uses: github/codeql-action/autobuild@v3
diff --git a/.github/workflows/compile-check.yml b/.github/workflows/compile-check.yml
index 6205f98..e54c438 100644
--- a/.github/workflows/compile-check.yml
+++ b/.github/workflows/compile-check.yml
@@ -47,4 +47,4 @@ jobs:
- name: Compiler Test
shell: bash
run: |
- mvn clean compile -P with-integration-tests -ntp
+ mvn clean package -P with-integration-tests -ntp
diff --git a/iotdb-collector/collector-core/pom.xml b/iotdb-collector/collector-core/pom.xml
index c6517b8..c29937e 100644
--- a/iotdb-collector/collector-core/pom.xml
+++ b/iotdb-collector/collector-core/pom.xml
@@ -50,17 +50,20 @@
</dependency>
<dependency>
<groupId>org.apache.iotdb</groupId>
- <artifactId>node-commons</artifactId>
- <version>1.3.2</version>
+ <artifactId>service-rpc</artifactId>
</dependency>
<dependency>
<groupId>org.apache.iotdb</groupId>
- <artifactId>service-rpc</artifactId>
- <version>1.3.2</version>
+ <artifactId>pipe-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tsfile</groupId>
+ <artifactId>tsfile</artifactId>
+ <version>2.1.0-250325-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.iotdb</groupId>
- <artifactId>pipe-api</artifactId>
+ <artifactId>tsfile</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
@@ -103,16 +106,110 @@
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.java-websocket</groupId>
+ <artifactId>Java-WebSocket</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-collections4</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-pool2</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-thrift-commons</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-thrift</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-session</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna-platform</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.milo</groupId>
+ <artifactId>stack-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.milo</groupId>
+ <artifactId>sdk-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.milo</groupId>
+ <artifactId>stack-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.milo</groupId>
+ <artifactId>sdk-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tsfile</groupId>
+ <artifactId>common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.moquette</groupId>
+ <artifactId>moquette-broker</artifactId>
+ </dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </dependency>
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ </dependency>
</dependencies>
<build>
+ <resources>
+ <resource>
+ <directory>${project.basedir}/src/assembly/resources/conf</directory>
+ <includes>
+ <include>application.properties</include>
+ <include>logback.xml</include>
+ </includes>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -126,6 +223,40 @@
</usedDependencies>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>8</source>
+ <target>8</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <!-- Package binaries-->
+ <execution>
+ <id>server-assembly</id>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <descriptors>
+ <descriptor>src/assembly/core.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <archive>
+ <manifest>
+ <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+ <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+ </manifest>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/iotdb-collector/pom.xml b/iotdb-collector/collector-core/src/assembly/core.xml
similarity index 57%
copy from iotdb-collector/pom.xml
copy to iotdb-collector/collector-core/src/assembly/core.xml
index 4bf95ac..4db5225 100644
--- a/iotdb-collector/pom.xml
+++ b/iotdb-collector/collector-core/src/assembly/core.xml
@@ -19,18 +19,22 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.iotdb</groupId>
- <artifactId>iotdb-extras-parent</artifactId>
- <version>2.0.2-SNAPSHOT</version>
- </parent>
- <artifactId>iotdb-collector</artifactId>
- <packaging>pom</packaging>
- <name>IoTDB: Collector</name>
- <modules>
- <module>collector-core</module>
- <module>collector-openapi</module>
- </modules>
-</project>
+<assembly>
+ <id>server</id>
+ <formats>
+ <format>dir</format>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>src/assembly/resources</directory>
+ <outputDirectory>${file.separator}</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/iotdb-collector/collector-core/src/assembly/resources/conf/application.properties b/iotdb-collector/collector-core/src/assembly/resources/conf/application.properties
new file mode 100644
index 0000000..7cfcea4
--- /dev/null
+++ b/iotdb-collector/collector-core/src/assembly/resources/conf/application.properties
@@ -0,0 +1,154 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+####################
+### API Service Configuration
+####################
+
+# The port on which the Jetty service runs.
+# Effective mode: on every start
+# Data type: int
+api_service_port=17070
+
+####################
+### Task Runtime Configuration
+####################
+
+# The number of concurrent threads for the source task.
+# Effective mode: on every start
+# Data type: int
+task_source_parallelism_num=1
+
+# The number of concurrent threads for the process task.
+# Effective mode: on every start
+# Data type: int
+task_process_parallelism_num=4
+
+# The number of concurrent threads for the sink task.
+# Effective mode: on every start
+# Data type: int
+task_sink_parallelism_num=4
+
+# The ring buffer size for the processor task.
+# Effective mode: on every start
+# Data type: int
+task_processor_ring_buffer_size=1024
+
+# The ring buffer size for the sink task.
+# Effective mode: on every start
+# Data type: int
+task_sink_ring_buffer_size=1024
+
+# Database file location of task
+# Effective mode: on every start
+# Data type: string
+task_database_file_path=system/database/task.db
+
+####################
+### Plugin Configuration
+####################
+
+# The location of plugin jar file
+# Effective mode: on every start
+# Data type: string
+plugin_lib_dir=system/plugin
+
+# Installation location of plugin jar file
+# Effective mode: on every start
+# Data type: string
+plugin_install_lib_dir=system/plugin/install
+
+# Database file location of plugin
+# Effective mode: on every start
+# Data type: string
+plugin_database_file_path=system/database/plugin.db
+
+####################
+### Pipe Configuration
+####################
+
+# The total bytes that all pipe sinks can transfer per second.
+# When given a value less than or equal to 0, it means no limit.
+# default value is -1, which means no limit.
+# Effective mode: on every start
+# Data type: double
+pipe_all_sinks_rate_limit_bytes_per_second=-1
+
+# Rate limiter configuration interval in milliseconds for hot reloading
+# Effective mode: on every start
+# Data type: int
+rate_limiter_hot_reload_check_interval_ms=1000
+
+# Maximum number of retry attempts for operations
+# Effective mode: on every start
+# Data type: int
+max_retry_times=5
+
+# Used for connection of IoTDB native clients
+# Bind with rpc_address
+# Effective mode: on every start
+# Data type: int
+rpc_port=6667
+
+# Used for connection of IoTDB native clients(Session)
+# Could set 127.0.0.1(for local test) or ipv4 address
+# Effective mode: on every start
+# Data type: String
+rpc_address=0.0.0.0
+
+# Buffer size for reading files in pipe connector (8MB default)
+# Effective mode: on every start
+# Data type: int
+pipe_connector_read_file_buffer_size=8388608
+
+# Timeout duration for pipe connector data transfer in milliseconds
+# Effective mode: on every start
+# Data type: int
+pipe_connector_transfer_timeout_ms=900000
+
+# Maximum allowed frame size for Thrift communication
+# Effective mode: on every start
+# Data type: int
+thrift_frame_max_size=536870912
+
+# Enable/disable thrift compression for pipe connector RPC
+# Effective mode: on every start
+# Data type: boolean
+is_pipe_connector_rpc_thrift_compression_enabled=false
+
+# Use this value to set timestamp precision as "ms", "us" or "ns".
+# Once the precision has been set, it can not be changed.
+# Effective mode: on every start
+# Data type: string
+timestamp_precision=ms
+
+# Memory allocation ratio for pipe leader cache management
+# Effective mode: on every start
+# Data type: float
+pipe_leader_cache_memory_usage_percentage=0.1
+
+# Enable/disable reference tracking for pipe events
+# Effective mode: on every start
+# Data type: boolean
+pipe_event_reference_tracking_enabled=true
+
+# Proactively triggers the interval for batch deliveries
+# Effective mode: on every start
+# Data type: long
+executor_cron_heartbeat_event_interval_seconds=20
\ No newline at end of file
diff --git a/iotdb-collector/collector-core/src/assembly/resources/conf/logback.xml b/iotdb-collector/collector-core/src/assembly/resources/conf/logback.xml
new file mode 100644
index 0000000..c3e66ca
--- /dev/null
+++ b/iotdb-collector/collector-core/src/assembly/resources/conf/logback.xml
@@ -0,0 +1,126 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration scan="true" scanPeriod="60 seconds">
+ <statusListener class="ch.qos.logback.core.status.NopStatusListener"/>
+ <property name="COLLECTOR_HOME" value="${COLLECTOR_HOME:-}"/>
+ <appender class="ch.qos.logback.core.ConsoleAppender" name="CONSOLE">
+ <encoder>
+ <!-- <pattern>%d{yyyy-MM-dd HH:mm:ss} %-5level %logger - %msg%n</pattern>-->
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ </encoder>
+ </appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEERROR">
+ <file>${COLLECTOR_HOME}/logs/application_error.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${COLLECTOR_HOME}/logs/log-application-error-%d{yyyyMMd}.log.gz</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.LevelFilter">
+ <level>ERROR</level>
+ <onMatch>ACCEPT</onMatch>
+ <onMismatch>DENY</onMismatch>
+ </filter>
+ </appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEWARN">
+ <file>${COLLECTOR_HOME}/logs/application_warn.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${COLLECTOR_HOME}/logs/log-application-warn-%d{yyyyMMd}.log.gz</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.LevelFilter">
+ <level>WARN</level>
+ <onMatch>ACCEPT</onMatch>
+ <onMismatch>DENY</onMismatch>
+ </filter>
+ </appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEDEBUG">
+ <file>${COLLECTOR_HOME}/logs/application_debug.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${COLLECTOR_HOME}/logs/log-application-debug-%d{yyyyMMd}.log.gz</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.LevelFilter">
+ <level>DEBUG</level>
+ <onMatch>ACCEPT</onMatch>
+ <onMismatch>DENY</onMismatch>
+ </filter>
+ </appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILETRACE">
+ <file>${COLLECTOR_HOME}/logs/application_trace.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${COLLECTOR_HOME}/logs/log-application-trace-%d{yyyyMMdd}.log.gz</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.LevelFilter">
+ <level>TRACE</level>
+ <onMatch>ACCEPT</onMatch>
+ <onMismatch>DENY</onMismatch>
+ </filter>
+ </appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEALL">
+ <file>${COLLECTOR_HOME}/logs/application_all.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${COLLECTOR_HOME}/logs/log-application-all-%d{yyyyMMd}.log.gz</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.LevelFilter">
+ <level>INFO</level>
+ <onMatch>ACCEPT</onMatch>
+ <onMismatch>DENY</onMismatch>
+ </filter>
+ </appender>
+ <root level="info">
+ <appender-ref ref="CONSOLE"/>
+ <appender-ref ref="FILEERROR"/>
+ <appender-ref ref="FILEWARN"/>
+ <appender-ref ref="FILEDEBUG"/>
+ <appender-ref ref="FILETRACE"/>
+ <appender-ref ref="FILEALL"/>
+ </root>
+ <logger name="org.eclipse.jetty" level="warn"/>
+ <logger name="javax.websocket.server" level="warn"/>
+</configuration>
diff --git a/iotdb-collector/collector-core/src/assembly/resources/sbin/collector-env.sh b/iotdb-collector/collector-core/src/assembly/resources/sbin/collector-env.sh
new file mode 100644
index 0000000..2e265c3
--- /dev/null
+++ b/iotdb-collector/collector-core/src/assembly/resources/sbin/collector-env.sh
@@ -0,0 +1,94 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# find java in JAVA_HOME
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+if [ -z $JAVA ] ; then
+ echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr
+ exit 1;
+fi
+
+# Determine the sort of JVM we'll be running on.
+java_ver_output=`"$JAVA" -version 2>&1`
+jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1`
+JVM_VERSION=${jvmver%_*}
+JVM_PATCH_VERSION=${jvmver#*_}
+if [ "$JVM_VERSION" \< "1.8" ] ; then
+ echo "IoTDB requires Java 8u92 or later."
+ exit 1;
+fi
+
+if [ "$JVM_VERSION" \< "1.8" ] && [ "$JVM_PATCH_VERSION" -lt 92 ] ; then
+ echo "IoTDB requires Java 8u92 or later."
+ exit 1;
+fi
+
+version_arr=(${JVM_VERSION//./ })
+
+illegal_access_params=""
+#GC log path has to be defined here because it needs to access COLLECTOR_HOME
+if [ "${version_arr[0]}" = "1" ] ; then
+ # Java 8
+ MAJOR_VERSION=${version_arr[1]}
+ echo "$COLLECTOR_JMX_OPTS" | grep -q "^-[X]loggc"
+ if [ "$?" = "1" ] ; then # [X] to prevent ccm from replacing this line
+ # only add -Xlog:gc if it's not mentioned in jvm-server.options file
+ mkdir -p ${COLLECTOR_HOME}/logs
+ if [ "$#" -ge "1" -a "$1" == "printgc" ]; then
+ COLLECTOR_JMX_OPTS="$COLLECTOR_JMX_OPTS -Xloggc:${COLLECTOR_HOME}/logs/gc.log -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCApplicationStoppedTime -XX:+PrintPromotionFailure -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M"
+ # For more detailed GC information, you can uncomment option below.
+ # NOTE: more detailed GC information may bring larger GC log files.
+ # COLLECTOR_JMX_OPTS="$COLLECTOR_JMX_OPTS -Xloggc:${COLLECTOR_HOME}/logs/gc.log -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCApplicationStoppedTime -XX:+PrintPromotionFailure -XX:+UseGCLogFileRotation -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC -XX:+PrintReferenceGC -XX:+PrintSafepointStatistics -XX:PrintSafepointStatisticsCount=1 -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M"
+ fi
+ fi
+else
+ #JDK 11 and others
+ MAJOR_VERSION=${version_arr[0]}
+ # See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
+ # The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
+ echo "$COLLECTOR_JMX_OPTS" | grep -q "^-[X]log:gc"
+ if [ "$?" = "1" ] ; then # [X] to prevent ccm from replacing this line
+ # only add -Xlog:gc if it's not mentioned in jvm-server.options file
+ mkdir -p ${COLLECTOR_HOME}/logs
+ if [ "$#" -ge "1" -a "$1" == "printgc" ]; then
+ COLLECTOR_JMX_OPTS="$COLLECTOR_JMX_OPTS -Xlog:gc=info,heap*=info,age*=info,safepoint=info,promotion*=info:file=${COLLECTOR_HOME}/logs/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760"
+ # For more detailed GC information, you can uncomment option below.
+ # NOTE: more detailed GC information may bring larger GC log files.
+ # COLLECTOR_JMX_OPTS="$COLLECTOR_JMX_OPTS -Xlog:gc*=debug,heap*=debug,age*=trace,metaspace*=info,safepoint*=debug,promotion*=info:file=${COLLECTOR_HOME}/logs/gc.log:time,uptime,pid,tid,level,tags:filecount=10,filesize=100M"
+ fi
+ fi
+ # Add argLine for Java 11 and above, due to [JEP 396: Strongly Encapsulate JDK Internals by Default] (https://openjdk.java.net/jeps/396)
+ illegal_access_params="$illegal_access_params --add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
+ illegal_access_params="$illegal_access_params --add-opens=java.base/java.lang=ALL-UNNAMED"
+ illegal_access_params="$illegal_access_params --add-opens=java.base/java.util=ALL-UNNAMED"
+ illegal_access_params="$illegal_access_params --add-opens=java.base/java.nio=ALL-UNNAMED"
+ illegal_access_params="$illegal_access_params --add-opens=java.base/java.io=ALL-UNNAMED"
+ illegal_access_params="$illegal_access_params --add-opens=java.base/java.net=ALL-UNNAMED"
+fi
\ No newline at end of file
diff --git a/iotdb-collector/collector-core/src/assembly/resources/sbin/common.sh b/iotdb-collector/collector-core/src/assembly/resources/sbin/common.sh
new file mode 100644
index 0000000..fe56821
--- /dev/null
+++ b/iotdb-collector/collector-core/src/assembly/resources/sbin/common.sh
@@ -0,0 +1,181 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# this function is for parsing the variables like "A=B" in `start-server.sh -D A=B`
+# The command just parse COLLECTOR-prefixed variables and ignore all other variables
+check_env_variables() {
+ string="$1"
+ array=$(echo $string | tr '=' ' ')
+ eval set -- "$array"
+ case "$1" in
+ COLLECTOR_INCLUDE)
+ COLLECTOR_INCLUDE="$2"
+ ;;
+ COLLECTOR_HOME)
+ COLLECTOR_HOME="$2"
+ ;;
+ COLLECTOR_DATA_HOME)
+ COLLECTOR_DATA_HOME="$2"
+ ;;
+ COLLECTOR_CONF)
+ COLLECTOR_CONF="$2"
+ ;;
+ COLLECTOR_LOG_DIR)
+ COLLECTOR_LOG_DIR="$2"
+ ;;
+ COLLECTOR_LOG_CONFIG)
+ COLLECTOR_LOG_CONFIG="$2"
+ ;;
+ COLLECTOR_CLI_CONF)
+ COLLECTOR_CLI_CONF="$2"
+ ;;
+ *)
+ #we can not process it, so that we return back.
+ echo "$1=$2"
+ ;;
+ esac
+ echo ""
+}
+
+check_all_variables() {
+ if [ -z "${COLLECTOR_INCLUDE}" ]; then
+ #do nothing
+ :
+ elif [ -r "$COLLECTOR_INCLUDE" ]; then
+ . "$COLLECTOR_INCLUDE"
+ fi
+
+ if [ -z "${COLLECTOR_HOME}" ]; then
+ export COLLECTOR_HOME="`dirname "$0"`/.."
+ fi
+
+ if [ -z "${COLLECTOR_DATA_HOME}" ]; then
+ export COLLECTOR_DATA_HOME=${COLLECTOR_HOME}
+ fi
+
+ if [ -z "${COLLECTOR_CONF}" ]; then
+ export COLLECTOR_CONF=${COLLECTOR_HOME}/conf
+ fi
+
+ if [ -z "${COLLECTOR_LOG_DIR}" ]; then
+ export COLLECTOR_LOG_DIR=${COLLECTOR_HOME}/logs
+ fi
+
+ if [ -z "${COLLECTOR_LOG_CONFIG}" ]; then
+ export COLLECTOR_LOG_CONFIG="${COLLECTOR_CONF}/logback.xml"
+ fi
+}
+
+check_config_unique() {
+ local key=$1
+ local values=$2
+
+ line_count=$(echo "$values" | wc -l)
+
+ if [ "$line_count" -gt 1 ]; then
+ echo "Error: Duplicate $key entries found"
+ exit 1
+ fi
+}
+
+check_collector_port_usages() {
+ echo "Checking whether the ports are already occupied..."
+ if [ "$(id -u)" -ne 0 ]; then
+ echo "Warning: If you do not use sudo, the checking may not detect all the occupied ports."
+ fi
+ occupied=false
+ if [ -f "$COLLECTOR_CONF/application.properties" ]; then
+ api_service_port=$(sed '/^api_service_port=/!d;s/.*=//' "${COLLECTOR_CONF}"/application.properties | tr -d '\r')
+ elif [ -f "$COLLECTOR_HOME/conf/application.properties" ]; then
+ api_service_port=$(sed '/^api_service_port=/!d;s/.*=//' "${COLLECTOR_HOME}"/conf/application.properties | tr -d '\r')
+ elif [ -f "$COLLECTOR_CONF/application.properties" ]; then
+ api_service_port=$(sed '/^api_service_port=/!d;s/.*=//' "${COLLECTOR_CONF}"/application.properties | tr -d '\r')
+ elif [ -f "$COLLECTOR_HOME/conf/application.properties" ]; then
+ api_service_port=$(sed '/^api_service_port=/!d;s/.*=//' "${COLLECTOR_HOME}"/conf/application.properties | tr -d '\r')
+ else
+ echo "Warning: cannot find application.properties, check the default configuration"
+ fi
+
+ check_config_unique "api_service_port" "$api_service_port"
+
+ api_service_port=${api_service_port:-17070}
+ if type lsof >/dev/null 2>&1; then
+ PID=$(lsof -t -i:"${api_service_port}" -sTCP:LISTEN)
+ if [ -n "$PID" ]; then
+ echo "The api_service_port" "$api_service_port" "is already occupied, PID:" "$PID"
+ occupied=true
+ fi
+ elif type netstat >/dev/null 2>&1; then
+ PID=$(netstat -anp 2>/dev/null | grep ":${api_service_port} " | grep ' LISTEN ' | awk '{print $NF}' | sed "s|/.*||g")
+ if [ -n "$PID" ]; then
+ echo "The api_service_port" "$api_service_port" "is already occupied, PID:" "$PID"
+ occupied=true
+ fi
+ else
+ echo " Error: No necessary tool to check whether given port is occupied, stop ports checking"
+ echo " Please install 'lsof' or 'netstat'."
+ fi
+ if [ $occupied = true ]; then
+ echo "Exit because there are occupied ports."
+ exit 0
+ fi
+}
+
+init_env() {
+ if [ -f "$COLLECTOR_CONF/collector-env.sh" ]; then
+ if [ "x$PRINT_GC" != "x" ]; then
+ . "$COLLECTOR_CONF/collector-env.sh" "printgc"
+ else
+ . "$COLLECTOR_CONF/collector-env.sh"
+ fi
+ elif [ -f "${COLLECTOR_HOME}/sbin/collector-env.sh" ]; then
+ if [ "x$PRINT_GC" != "x" ]; then
+ . "${COLLECTOR_HOME}/sbin/collector-env.sh" "printgc"
+ else
+ . "${COLLECTOR_HOME}/sbin/collector-env.sh"
+ fi
+ else
+ echo "Can't find $COLLECTOR_CONF/collector-env.sh"
+ fi
+}
+
+get_iotdb_collector_include() {
+ #reset $1 to $* for this command
+ eval set -- "$1"
+ VARS=""
+ while true; do
+ case "$1" in
+ -D)
+ VARS="$VARS $(checkEnvVariables $2)"
+ shift 2
+ ;;
+ "")
+ #if we do not use getopt, we then have to process the case that there is no argument.
+ #in some systems, when there is no argument, shift command may throw error, so we skip directly
+ break
+ ;;
+ *)
+ VARS="$VARS $1"
+ shift
+ ;;
+ esac
+ done
+ echo "$VARS"
+}
\ No newline at end of file
diff --git a/iotdb-collector/collector-core/src/assembly/resources/sbin/start-collector.bat b/iotdb-collector/collector-core/src/assembly/resources/sbin/start-collector.bat
new file mode 100644
index 0000000..d40c579
--- /dev/null
+++ b/iotdb-collector/collector-core/src/assembly/resources/sbin/start-collector.bat
@@ -0,0 +1,208 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@echo off
+@REM set cmd format
+chcp 65001
+
+title IoTDB Collector
+
+echo ````````````````````````
+echo Starting IoTDB Collector
+echo ````````````````````````
+
+@REM -----------------------------------------------------------------------------
+@REM SET JAVA
+set PATH="%JAVA_HOME%\bin\";%PATH%
+set "FULL_VERSION="
+set "MAJOR_VERSION="
+set "MINOR_VERSION="
+
+
+for /f tokens^=2-5^ delims^=.-_+^" %%j in ('java -fullversion 2^>^&1') do (
+ set "FULL_VERSION=%%j-%%k-%%l-%%m"
+ IF "%%j" == "1" (
+ set "MAJOR_VERSION=%%k"
+ set "MINOR_VERSION=%%l"
+ ) else (
+ set "MAJOR_VERSION=%%j"
+ set "MINOR_VERSION=%%k"
+ )
+)
+
+set JAVA_VERSION=%MAJOR_VERSION%
+
+@REM we do not check jdk that version less than 1.8 because they are too stale...
+IF "%JAVA_VERSION%" == "6" (
+ echo IoTDB only supports jdk >= 8, please check your java version.
+ goto finally
+)
+IF "%JAVA_VERSION%" == "7" (
+ echo IoTDB only supports jdk >= 8, please check your java version.
+ goto finally
+)
+
+@REM -----------------------------------------------------------------------------
+@REM SET DIR
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..
+if NOT DEFINED COLLECTOR_HOME set COLLECTOR_HOME=%cd%
+popd
+
+SET enable_printgc=false
+IF "%1" == "printgc" (
+ SET enable_printgc=true
+ SHIFT
+)
+
+SET COLLECTOR_CONF=%COLLECTOR_HOME%\conf
+SET COLLECTOR_LOGS=%COLLECTOR_HOME%\logs
+
+@setlocal ENABLEDELAYEDEXPANSION ENABLEEXTENSIONS
+set is_conf_path=false
+for %%i in (%*) do (
+ IF "%%i" == "-c" (
+ set is_conf_path=true
+ ) ELSE IF "!is_conf_path!" == "true" (
+ set is_conf_path=false
+ set COLLECTOR_CONF=%%i
+ )
+)
+
+@REM SET CONFIG FILE
+IF EXIST "%COLLECTOR_CONF%\application.properties" (
+ set CONFIG_FILE="%COLLECTOR_CONF%\application.properties"
+) ELSE IF EXIST "%COLLECTOR_HOME%\conf\application.properties" (
+ set CONFIG_FILE="%COLLECTOR_HOME%\conf\application.properties"
+) ELSE (
+ set CONFIG_FILE=
+)
+
+@REM CHECK THE PORT USAGES
+IF DEFINED CONFIG_FILE (
+ for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^api_service_port"
+ "%CONFIG_FILE%"') do (
+ set api_service_port=%%i
+ )
+) ELSE (
+ echo "Can't find application.properties, check the default ports"
+ set api_service_port=17070
+)
+
+echo Check whether the ports are occupied....
+set occupied=0
+set api_service_port_occupied=0
+
+for /f "tokens=1,3,7 delims=: " %%i in ('netstat /ano') do (
+ if %%i==TCP (
+ if %%j==%api_service_port% (
+ if !api_service_port_occupied!==0 (
+ echo The api_service_port %api_service_port% is already occupied, pid:%%k
+ set occupied=1
+ set api_service_port_occupied=1
+ )
+ )
+ )
+)
+
+if %occupied%==1 (
+ echo There exists occupied port, please change the configuration.
+ TIMEOUT /T 10 /NOBREAK
+ exit 0
+)
+
+@setlocal ENABLEDELAYEDEXPANSION ENABLEEXTENSIONS
+set CONF_PARAMS=-s
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.collector.Application
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM -----------------------------------------------------------------------------
+@REM JVM Opts we'll use in legacy run or installation
+@REM -DTSFILE_HOME="%COLLECTOR_HOME%"^
+@REM -DTSFILE_CONF="%COLLECTOR_CONF%"^
+set JAVA_OPTS=-ea^
+ -Dlogback.configurationFile="%COLLECTOR_CONF%\logback.xml"^
+ -DCOLLECTOR_HOME="%COLLECTOR_HOME%"^
+ -DCOLLECTOR_CONF="%COLLECTOR_CONF%"^
+ -DOFF_HEAP_MEMORY="%OFF_HEAP_MEMORY%"^
+ -Dsun.jnu.encoding=UTF-8^
+ -Dfile.encoding=UTF-8
+
+@REM ----------------------------------------------------------------------------
+@REM ***** CLASSPATH library setting *****
+@REM Ensure that any user defined CLASSPATH variables are not used on startup
+if EXIST "%COLLECTOR_HOME%\lib" (set CLASSPATH="%COLLECTOR_HOME%\lib\*") else set CLASSPATH="%COLLECTOR_HOME%\..\lib\*"
+
+@REM this special suffix 'iotdb.DataNode' is mandatory as stop-node.bat uses it to filter the process id.
+set CLASSPATH=%CLASSPATH%;collector.Application
+goto okClasspath
+
+:append
+set CLASSPATH=%CLASSPATH%;%1
+goto :eof
+
+@REM -----------------------------------------------------------------------------
+:okClasspath
+
+rem echo CLASSPATH: %CLASSPATH%
+
+@REM ----------------------------------------------------------------------------
+@REM SET PARA
+
+@REM iotdb-server runs in foreground by default
+@REM set foreground=0
+set foreground=yes
+
+:checkPara
+set COMMANSLINE=%*
+@REM setlocal ENABLEDELAYEDEXPANSION
+:STR_VISTOR
+for /f "tokens=1* delims= " %%a in ("%COMMANSLINE%") do (
+@REM -----more para-----
+for /f "tokens=1* delims==" %%1 in ("%%a") do (
+@REM echo 1=%%1 "|||" 2=%%2
+@REM if "%%1"=="-v" ( java %JAVA_OPTS% -Dlogback.configurationFile="%COLLECTOR_CONF%/logback-tool.xml" -cp %CLASSPATH% org.apache.iotdb.db.service.GetVersion & goto finally )
+if "%%1"=="-f" ( set foreground=yes)
+if "%%1"=="-d" ( set foreground=0)
+)
+set COMMANSLINE=%%b
+goto STR_VISTOR
+)
+
+@REM ----------------------------------------------------------------------------
+@REM START
+:start
+if %foreground%==yes (
+ java %ILLEGAL_ACCESS_PARAMS% %JAVA_OPTS% %COLLECTOR_HEAP_OPTS% -cp %CLASSPATH% %COLLECTOR_JMX_OPTS% %MAIN_CLASS% %CONF_PARAMS%
+ ) ELSE (
+ start javaw %ILLEGAL_ACCESS_PARAMS% %JAVA_OPTS% %COLLECTOR_HEAP_OPTS% -cp %CLASSPATH% %COLLECTOR_JMX_OPTS% %MAIN_CLASS% %CONF_PARAMS%
+ )
+
+goto finally
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+
+@REM -----------------------------------------------------------------------------
+:finally
+@ENDLOCAL
+pause
\ No newline at end of file
diff --git a/iotdb-collector/collector-core/src/assembly/resources/sbin/start-collector.sh b/iotdb-collector/collector-core/src/assembly/resources/sbin/start-collector.sh
new file mode 100644
index 0000000..61935ac
--- /dev/null
+++ b/iotdb-collector/collector-core/src/assembly/resources/sbin/start-collector.sh
@@ -0,0 +1,237 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+source "$(dirname "$0")/common.sh"
+
+# iotdb collector server runs on foreground by default
+foreground="yes"
+
+if [ $# -ne 0 ]; then
+ echo "All parameters are $*"
+fi
+
+while true; do
+ case "$1" in
+ -c)
+ COLLECTOR_CONF="$2"
+ shift 2
+ ;;
+ -p)
+ pidfile="$2"
+ shift 2
+ ;;
+ -f)
+ foreground="yes"
+ shift
+ ;;
+ -d)
+ foreground=""
+ shift
+ ;;
+ -g)
+ PRINT_GC="yes"
+ shift
+ ;;
+ -h)
+ echo "Usage: $0 [-v] [-f] [-d] [-h] [-p pidfile] [-c configFolder] [-H HeapDumpPath] [-E JvmErrorFile] [printgc]"
+ exit 0
+ ;;
+ -v)
+ SHOW_VERSION="yes"
+ break
+ ;;
+ --)
+ shift
+ #all others are args to the program
+ PARAMS=$*
+ break
+ ;;
+ "")
+ #if we do not use getopt, we then have to process the case that there is no argument.
+ #in some systems, when there is no argument, shift command may throw error, so we skip directly
+ #all others are args to the program
+ PARAMS=$*
+ break
+ ;;
+ *)
+ echo "Error parsing arguments! Unknown argument \"$1\"" >&2
+ exit 1
+ ;;
+ esac
+done
+
+if [ "$(id -u)" -ne 0 ]; then
+ echo "Notice: in some systems, DataNode must run in sudo mode to write data. The process may fail."
+fi
+
+#check_all_variables is in common.sh
+check_all_variables
+
+#check_collector_port_usages is in common.sh
+check_collector_port_usages
+
+CLASSPATH=""
+for f in "${COLLECTOR_HOME}"/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}":"$f
+done
+
+classname=org.apache.iotdb.collector.Application
+
+if [ "x$SHOW_VERSION" != "x" ]; then
+ COLLECTOR_LOG_CONFIG="${COLLECTOR_CONF}/logback.xml"
+ # find java in JAVA_HOME
+ if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+ else
+ JAVA=java
+ fi
+ exit 0
+fi
+
+echo ---------------------
+echo "Starting IoTDB Collector"
+echo ---------------------
+
+#init_env is in common.sh
+init_env
+
+# check whether we can enable heap dump when oom
+if [ "x$COLLECTOR_ALLOW_HEAP_DUMP" == "xtrue" ]; then
+ COLLECTOR_JVM_OPTS="$COLLECTOR_JVM_OPTS $COLLECTOR_HEAP_DUMP_COMMAND"
+fi
+
+PARAMS="-s $PARAMS"
+
+classname=org.apache.iotdb.collector.Application
+
+launch_service() {
+ class="$1"
+ collector_parms="-Dlogback.configurationFile=${COLLECTOR_LOG_CONFIG}"
+ collector_parms="$collector_parms -DCOLLECTOR_HOME=${COLLECTOR_HOME}"
+ collector_parms="$collector_parms -DCOLLECTOR_DATA_HOME=${COLLECTOR_DATA_HOME}"
+ collector_parms="$collector_parms -DTSFILE_HOME=${COLLECTOR_HOME}"
+ collector_parms="$collector_parms -DCOLLECTOR_CONF=${COLLECTOR_CONF}"
+ collector_parms="$collector_parms -DTSFILE_CONF=${COLLECTOR_CONF}"
+ collector_parms="$collector_parms -Dname=collector\.IoTDB"
+ collector_parms="$collector_parms -DCOLLECTOR_LOG_DIR=${COLLECTOR_LOG_DIR}"
+ collector_parms="$collector_parms -DOFF_HEAP_MEMORY=${OFF_HEAP_MEMORY}"
+
+ if [ "x$pidfile" != "x" ]; then
+ collector_parms="collector_parms -Dcollector-pidfile=$pidfile"
+ fi
+
+ if [ "x$foreground" == "xyes" ]; then
+ collector_parms="$collector_parms -Dcollector-foreground=yes"
+ if [ "x$JVM_ON_OUT_OF_MEMORY_ERROR_OPT" != "x" ]; then
+ [ ! -z "$pidfile" ] && printf "%d" $! > "$pidfile"
+ exec $NUMACTL "$JAVA" $JVM_OPTS "$JVM_ON_OUT_OF_MEMORY_ERROR_OPT" $illegal_access_params $collector_parms $COLLECTOR_JMX_OPTS -cp "$CLASSPATH" $COLLECTOR_JVM_OPTS "$class" $PARAMS
+ else
+ [ ! -z "$pidfile" ] && printf "%d" $! > "$pidfile"
+ exec $NUMACTL "$JAVA" $JVM_OPTS $illegal_access_params $collector_parms $COLLECTOR_JMX_OPTS -cp "$CLASSPATH" $COLLECTOR_JVM_OPTS "$class" $PARAMS
+ fi
+ # Startup IoTDB, background it, and write the pid.
+ else
+ if [ "x$JVM_ON_OUT_OF_MEMORY_ERROR_OPT" != "x" ]; then
+ exec $NUMACTL "$JAVA" $JVM_OPTS "$JVM_ON_OUT_OF_MEMORY_ERROR_OPT" $illegal_access_params $collector_parms $COLLECTOR_JMX_OPTS -cp "$CLASSPATH" $COLLECTOR_JVM_OPTS "$class" $PARAMS 2>&1 > /dev/null <&- &
+ [ ! -z "$pidfile" ] && printf "%d" $! > "$pidfile"
+ true
+ else
+ exec $NUMACTL "$JAVA" $JVM_OPTS $illegal_access_params $collector_parms $COLLECTOR_JMX_OPTS -cp "$CLASSPATH" $COLLECTOR_JVM_OPTS "$class" $PARAMS 2>&1 > /dev/null <&- &
+ [ ! -z "$pidfile" ] && printf "%d" $! > "$pidfile"
+ true
+ fi
+ fi
+
+ return $?
+}
+
+# check whether tool 'lsof' exists
+check_tool_env() {
+ if ! type lsof > /dev/null 2>&1 ; then
+ echo ""
+ echo " Warning: No tool 'lsof', Please install it."
+ echo " Note: Some checking function need 'lsof'."
+ echo ""
+ return 1
+ else
+ return 0
+ fi
+}
+
+# convert path to real full-path.
+# e.g., /a/b/c/.. will return /a/b
+# If path has been deleted, return ""
+get_real_path() {
+ local path=$1
+ local real_path=""
+ cd $path > /dev/null 2>&1
+ if [ $? -eq 0 ] ; then
+ real_path=$(pwd -P)
+ cd - > /dev/null 2>&1
+ fi
+ echo "${real_path}"
+}
+
+# check whether same directory's IoTDB node process has been running
+check_running_process() {
+ check_tool_env
+
+ PIDS=$(ps ax | grep "$classname" | grep java | grep DCOLLECTOR_HOME | grep -v grep | awk '{print $1}')
+ for pid in ${PIDS}
+ do
+ run_conf_path=""
+ # find the abstract path of the process
+ run_cwd=$(lsof -p $pid 2>/dev/null | awk '$4~/cwd/ {print $NF}')
+ # find "-DCOLLECTOR_HOME=XXX" from the process command
+ run_home_path=$(ps -fp $pid | sed "s/ /\n/g" | sed -n "s/-DCOLLECTOR_HOME=//p")
+ run_home_path=$(get_real_path "${run_cwd}/${run_home_path}")
+
+ #if dir ${run_home_path} has been deleted
+ if [ "${run_home_path}" == "" ]; then
+ continue
+ fi
+
+ current_home_path=$(get_real_path ${COLLECTOR_HOME})
+ if [ "${run_home_path}" == "${current_home_path}" ]; then
+ echo ""
+ echo " Found running IoTDB Collector (PID=$pid)." >&2
+ echo " Can not run duplicated IoTDB Collector!" >&2
+ echo " Exit..." >&2
+ echo ""
+ exit 1
+ fi
+ done
+}
+
+check_tool_env
+# If needed tool is ready, check whether same directory's IoTDB node is running
+if [ $? -eq 0 ]; then
+ check_running_process
+fi
+
+# Start up the service
+launch_service "$classname"
+
+exit $?
diff --git a/iotdb-collector/collector-core/src/assembly/resources/sbin/stop-collector.bat b/iotdb-collector/collector-core/src/assembly/resources/sbin/stop-collector.bat
new file mode 100644
index 0000000..e8daee2
--- /dev/null
+++ b/iotdb-collector/collector-core/src/assembly/resources/sbin/stop-collector.bat
@@ -0,0 +1,67 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@echo off
+
+set current_dir=%~dp0
+set superior_dir=%current_dir%\..\
+
+IF EXIST "%superior_dir%\conf\application.properties" (
+ set config_file="%superior_dir%\conf\application.properties"
+) ELSE (
+ IF EXIST "%superior_dir%\conf\application.properties" (
+ set config_file=%superior_dir%\conf\application.properties
+ ) ELSE (
+ echo No configuration file found. Exiting.
+ exit /b 1
+ )
+)
+
+if not defined config_file (
+ echo No configuration file found. Exiting.
+ exit /b 1
+)
+
+for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^api_service_port"
+"%config_file%"') do (
+ set api_service_port=%%i
+)
+
+if not defined api_service_port (
+ echo "WARNING: api_service_port not found in the configuration file. Using default value api_service_port = 17070"
+ set api_service_port=17070
+)
+
+echo Check whether the rpc_port is used..., port is %dn_rpc_port%
+
+for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "api_service_address"
+"%config_file%"') do (
+ set api_service_address=%%i
+)
+
+if not defined api_service_address (
+ echo "WARNING: api_service_address not found in the configuration file. Using default value api_service_address = 0.0.0.0"
+ set api_service_address=0.0.0.0
+)
+
+for /f "tokens=5" %%a in ('netstat /ano ^| findstr %api_service_address%:%api_service_port%') do (
+ taskkill /f /pid %%a
+ echo Close DataNode, PID: %%a
+)
+rem ps ax | grep -i 'iotdb.Application' | grep -v grep | awk '{print $1}' | xargs kill -SIGTERM
diff --git a/iotdb-collector/collector-core/src/assembly/resources/sbin/stop-collector.sh b/iotdb-collector/collector-core/src/assembly/resources/sbin/stop-collector.sh
new file mode 100644
index 0000000..51322fa
--- /dev/null
+++ b/iotdb-collector/collector-core/src/assembly/resources/sbin/stop-collector.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+source "$(dirname "$0")/common.sh"
+COLLECTOR_CONF="`dirname "$0"`/../conf"
+
+if [ -f "${COLLECTOR_CONF}/application.properties" ]; then
+ api_service_port=`sed '/^api_service_port=/!d;s/.*=//' ${COLLECTOR_CONF}/application.properties | tr -d '\r'`
+else
+ api_service_port=`sed '/^api_service_port=/!d;s/.*=//' ${COLLECTOR_CONF}/application.properties | tr -d '\r'`
+fi
+
+if [ -z "$api_service_port" ]; then
+ echo "WARNING: api_service_port not found in the configuration file. Using default value api_service_port=17070"
+ api_service_port=17070
+fi
+
+check_config_unique "api_service_port" "$api_service_port"
+
+force=""
+
+while true; do
+ case "$1" in
+ -f)
+ force="yes"
+ break
+ ;;
+ "")
+ #if we do not use getopt, we then have to process the case that there is no argument.
+ #in some systems, when there is no argument, shift command may throw error, so we skip directly
+ #all others are args to the program
+ PARAMS=$*
+ break
+ ;;
+ esac
+done
+
+echo "Check whether the api_service_port is used..., port is" $api_service_port
+
+if type lsof > /dev/null 2>&1 ; then
+ PID=$(lsof -t -i:"${api_service_port}" -sTCP:LISTEN)
+elif type netstat > /dev/null 2>&1 ; then
+ PID=$(netstat -anp 2>/dev/null | grep ":${api_service_port} " | grep ' LISTEN ' | awk '{print $NF}' | sed "s|/.*||g" )
+else
+ echo ""
+ echo " Error: No necessary tool."
+ echo " Please install 'lsof' or 'netstat'."
+ exit 1
+fi
+
+PID_VERIFY=$(ps ax | grep -i 'Application' | grep java | grep -v grep | awk '{print $1}')
+if [ -z "$PID" ]; then
+ echo "No Application to stop"
+ if [ "$(id -u)" -ne 0 ]; then
+ echo "Maybe you can try to run in sudo mode to detect the process."
+ fi
+ exit 1
+elif [[ "${PID_VERIFY}" =~ ${PID} ]]; then
+ if [[ "${force}" == "yes" ]]; then
+ kill -9 "$PID"
+ echo "Force to stop Application, PID:" "$PID"
+ else
+ kill -s TERM "$PID"
+ echo "Stop Application, PID:" "$PID"
+ fi
+else
+ echo "No Application to stop"
+ exit 1
+fi
\ No newline at end of file
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java
index 650caee..80b5b42 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.collector;
import org.apache.iotdb.collector.config.Configuration;
import org.apache.iotdb.collector.service.ApiService;
import org.apache.iotdb.collector.service.IService;
+import org.apache.iotdb.collector.service.PeriodicalJobService;
import org.apache.iotdb.collector.service.PersistenceService;
import org.apache.iotdb.collector.service.RuntimeService;
@@ -41,6 +42,7 @@ public class Application {
services.add(new RuntimeService());
services.add(new ApiService());
services.add(new PersistenceService());
+ services.add(new PeriodicalJobService());
}
public static void main(String[] args) {
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Configuration.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Configuration.java
index 115c1c0..83b84d3 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Configuration.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Configuration.java
@@ -22,10 +22,12 @@ package org.apache.iotdb.collector.config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
@@ -36,6 +38,8 @@ public class Configuration {
private static final Logger LOGGER = LoggerFactory.getLogger(Configuration.class);
private static final String CONFIG_FILE_NAME = "application.properties";
+ public static final String COLLECTOR_CONF = "COLLECTOR_CONF";
+ public static final String COLLECTOR_HOME = "COLLECTOR_HOME";
private final Options options = new Options();
@@ -69,19 +73,44 @@ public class Configuration {
}
private Optional<URL> getPropsUrl() {
- final URL url = Options.class.getResource("/" + CONFIG_FILE_NAME);
+ String urlString = getConfDir();
+ if (urlString == null) {
+ final URL uri = Options.class.getResource("/" + CONFIG_FILE_NAME);
+ if (uri != null) {
+ return Optional.of(uri);
+ } else {
+ LOGGER.warn(
+ "Cannot find IOTDB_COLLECTOR_HOME or IOTDB_COLLECTOR_CONF environment variable when loading "
+ + "config file {}, use default configuration",
+ CONFIG_FILE_NAME);
+ return Optional.empty();
+ }
+ } else if (!urlString.endsWith(".properties")) {
+ urlString += (File.separatorChar + CONFIG_FILE_NAME);
+ }
- if (url != null) {
- return Optional.of(url);
- } else {
- LOGGER.warn(
- "Cannot find IOTDB_COLLECTOR_HOME or IOTDB_COLLECTOR_CONF environment variable when loading "
- + "config file {}, use default configuration",
- CONFIG_FILE_NAME);
+ try {
+ if (!urlString.startsWith("file:") && !urlString.startsWith("classpath:")) {
+ urlString = "file:" + urlString;
+ }
+ return Optional.of(new URL(urlString));
+ } catch (final MalformedURLException e) {
+ LOGGER.warn("get url failed", e);
return Optional.empty();
}
}
+ public String getConfDir() {
+ String confString = System.getProperty(COLLECTOR_CONF, null);
+ if (confString == null) {
+ confString = System.getProperty(COLLECTOR_HOME, null);
+ if (confString != null) {
+ confString = confString + File.separatorChar + "conf";
+ }
+ }
+ return confString;
+ }
+
public void logAllOptions() {
options.logAllOptions();
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java
index 16fd814..ce262d6 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java
@@ -22,10 +22,13 @@ package org.apache.iotdb.collector.config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
+import static org.apache.iotdb.collector.config.Configuration.COLLECTOR_HOME;
+
public class Options {
private static final Logger LOGGER = LoggerFactory.getLogger(Options.class);
@@ -37,6 +40,7 @@ public class Options {
Class.forName(ApiServiceOptions.class.getName());
Class.forName(TaskRuntimeOptions.class.getName());
Class.forName(PluginRuntimeOptions.class.getName());
+ Class.forName(PipeRuntimeOptions.class.getName());
} catch (final ClassNotFoundException e) {
throw new RuntimeException("Failed to load options", e);
}
@@ -73,6 +77,15 @@ public class Options {
public abstract void setValue(final String valueString);
+ protected String addHomeDir(String dir) {
+ final String homeDir = System.getProperty(COLLECTOR_HOME, null);
+ if (!new File(dir).isAbsolute() && homeDir != null && !homeDir.isEmpty()) {
+ dir =
+ !homeDir.endsWith(File.separator) ? homeDir + File.separatorChar + dir : homeDir + dir;
+ }
+ return dir;
+ }
+
@Override
public String toString() {
return key + " = " + value();
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/PipeRuntimeOptions.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/PipeRuntimeOptions.java
new file mode 100644
index 0000000..0abd49a
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/PipeRuntimeOptions.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.config;
+
+import org.apache.iotdb.rpc.RpcUtils;
+
+import java.util.concurrent.TimeUnit;
+
+public class PipeRuntimeOptions extends Options {
+ public static final Option<Double> PIPE_ALL_SINK_RATE_LIMIT_BYTES_PER_SECOND =
+ new Option<Double>("pipe_all_sinks_rate_limit_bytes_per_second", -1d) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Double.parseDouble(valueString);
+ }
+ };
+
+ public static final Option<Integer> RATE_LIMITER_HOT_RELOAD_CHECK_INTERVAL_MS =
+ new Option<Integer>("rate_limiter_hot_reload_check_interval_ms", 1000) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Integer> MAX_RETRY_TIMES =
+ new Option<Integer>("max_retry_times", 5) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Integer> RPC_PORT =
+ new Option<Integer>("rpc_port", 6667) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<String> RPC_ADDRESS =
+ new Option<String>("rpc_address", "0.0.0.0") {
+ @Override
+ public void setValue(final String valueString) {
+ value = valueString;
+ }
+ };
+
+ public static final Option<Integer> PIPE_CONNECTOR_READ_FILE_BUFFER_SIZE =
+ new Option<Integer>("pipe_connector_read_file_buffer_size", 8388608) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Integer> PIPE_CONNECTOR_TRANSFER_TIMEOUT_MS =
+ new Option<Integer>("pipe_connector_transfer_timeout_ms", 15 * 60 * 1000) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Integer> THRIFT_FRAME_MAX_SIZE =
+ new Option<Integer>("thrift_frame_max_size", 546870912) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Boolean> IS_PIPE_CONNECTOR_RPC_THRIFT_COMPRESSION_ENABLED =
+ new Option<Boolean>("is_pipe_connector_rpc_thrift_compression_enabled", false) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Boolean.parseBoolean(valueString);
+ }
+ };
+
+ public static final Option<String> TIMESTAMP_PRECISION =
+ new Option<String>("timestamp_precision", "ms") {
+ @Override
+ public void setValue(final String valueString) {
+ value = valueString;
+ }
+ };
+
+ public static final Option<String> CLUSTER_ID =
+ new Option<String>("cluster_id", "") {
+ @Override
+ public void setValue(final String valueString) {
+ value = valueString;
+ }
+ };
+
+ public static final Option<Float> PIPE_LEADER_CACHE_MEMORY_USAGE_PERCENTAGE =
+ new Option<Float>("pipe_leader_cache_memory_usage_percentage", 0.1f) {
+ @Override
+ public void setValue(String valueString) {
+ value = Float.parseFloat(valueString);
+ }
+ };
+
+ public static final Option<Boolean> PIPE_EVENT_REFERENCE_TRACKING_ENABLED =
+ new Option<Boolean>("pipe_event_reference_tracking_enabled", true) {
+ @Override
+ public void setValue(String valueString) {
+ value = Boolean.parseBoolean(valueString);
+ }
+ };
+
+ public static volatile Option<Long> PIPE_CHECK_MEMORY_ENOUGH_INTERVAL_MS =
+ new Option<Long>("pipe_check_memory_enough_interval_ms", 10L) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Long.parseLong(valueString);
+ }
+ };
+
+ public static final Option<Boolean> PIPE_CONNECTOR_READ_FILE_BUFFER_MEMORY_CONTROL =
+ new Option<Boolean>("pipe_connector_read_file_buffer_memory_control", false) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Boolean.parseBoolean(valueString);
+ }
+ };
+
+ public static final Option<Integer> PIPE_SUBTASK_EXECUTOR_MAX_THREAD_NUM =
+ new Option<Integer>(
+ "pipe_subtask_executor_max_thread_num",
+ Math.max(5, Runtime.getRuntime().availableProcessors() / 2)) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Integer> PIPE_DATA_STRUCTURE_TABLET_SIZE_IN_BYTES =
+ new Option<Integer>("pipe_data_structure_tablet_size_in_bytes", 2097152) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Integer> PIPE_DATA_STRUCTURE_TABLET_ROW_SIZE =
+ new Option<Integer>("pipe_data_structure_tablet_row_size", 2048) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Boolean> PIPE_AIR_GAP_RECEIVER_ENABLED =
+ new Option<Boolean>("pipe_air_gap_receiver_enabled", true) {
+
+ @Override
+ public void setValue(final String valueString) {
+ value = Boolean.parseBoolean(valueString);
+ }
+ };
+
+ public static final Option<Integer> PIPE_AIR_GAP_RECEIVER_PORT =
+ new Option<Integer>("pipe_air_gap_receiver_port", 9780) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Long> PIPE_SUBTASK_EXECUTOR_PENDING_QUEUE_MAX_BLOCKING_TIME_MS =
+ new Option<Long>("pipe_subtask_executor_pending_queue_max_blocking_time_ms", 1000L) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Long.parseLong(valueString);
+ }
+ };
+
+ public static final Option<Boolean> TIMESTAMP_PRECISION_CHECK_ENABLED =
+ new Option<Boolean>("timestamp_precision_check_enabled", true) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Boolean.parseBoolean(valueString);
+ }
+ };
+
+ public static final Option<Integer> DN_CONNECTION_TIMEOUT_IN_MS =
+ new Option<Integer>("dn_connection_timeout_in_ms", (int) TimeUnit.SECONDS.toMillis(60)) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Boolean> IS_RPC_THRIFT_COMPRESSION_ENABLED =
+ new Option<Boolean>("is_rpc_thrift_compression_enabled", false) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Boolean.parseBoolean(valueString);
+ }
+ };
+
+ public static final Option<Integer> PIPE_CONNECTOR_REQUEST_SLICE_THRESHOLD_BYTES =
+ new Option<Integer>(
+ "pipe_connector_request_slice_threshold_bytes",
+ (int) (RpcUtils.THRIFT_FRAME_MAX_SIZE * 0.8)) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Integer> PIPE_CONNECTOR_HANDSHAKE_TIMEOUT_MS =
+ new Option<Integer>("pipe_connector_handshake_timeout_ms", 10 * 1000) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Boolean> PIPE_CONNECTOR_RPC_THRIFT_COMPRESSION_ENABLED =
+ new Option<Boolean>("pipe_connector_rpc_thrift_compression_enabled", false) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Boolean.parseBoolean(valueString);
+ }
+ };
+
+ public static final Option<Integer> PIPE_ASYNC_CONNECTOR_SELECTOR_NUMBER =
+ new Option<Integer>(
+ "pipe_async_connector_selector_number",
+ Math.max(4, Runtime.getRuntime().availableProcessors() / 2)) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Integer> PIPE_ASYNC_CONNECTOR_MAX_CLIENT_NUMBER =
+ new Option<Integer>(
+ "pipe_async_connector_max_client_number",
+ Math.max(16, Runtime.getRuntime().availableProcessors() / 2)) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static volatile Option<Integer> DATA_NODE_ID =
+ new Option<Integer>("data_node_id", -1) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Integer.parseInt(valueString);
+ }
+ };
+
+ public static final Option<Long> EXECUTOR_CRON_HEARTBEAT_EVENT_INTERVAL_SECONDS =
+ new Option<Long>("executor_cron_heartbeat_event_interval_seconds", 20L) {
+ @Override
+ public void setValue(final String valueString) {
+ value = Long.parseLong(valueString);
+ }
+ };
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/PluginRuntimeOptions.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/PluginRuntimeOptions.java
index 92eb04b..92a745e 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/PluginRuntimeOptions.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/PluginRuntimeOptions.java
@@ -23,10 +23,10 @@ import java.io.File;
public class PluginRuntimeOptions extends Options {
public static final Option<String> PLUGIN_LIB_DIR =
- new Option<String>("plugin_lib_dir", "ext" + File.separator + "plugin") {
+ new Option<String>("plugin_lib_dir", "system" + File.separator + "plugin") {
@Override
public void setValue(final String valueString) {
- value = valueString;
+ value = addHomeDir(valueString);
}
};
@@ -35,7 +35,17 @@ public class PluginRuntimeOptions extends Options {
"plugin_install_lib_dir", PLUGIN_LIB_DIR.value() + File.separator + "install") {
@Override
public void setValue(final String valueString) {
- value = valueString;
+ value = addHomeDir(valueString);
+ }
+ };
+
+ public static final Option<String> PLUGIN_DATABASE_FILE_PATH =
+ new Option<String>(
+ "plugin_database_file_path",
+ "system" + File.separator + "database" + File.separator + "plugin.db") {
+ @Override
+ public void setValue(final String valueString) {
+ value = addHomeDir(valueString);
}
};
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/TaskRuntimeOptions.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/TaskRuntimeOptions.java
index 70efcc9..dc35f33 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/TaskRuntimeOptions.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/TaskRuntimeOptions.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.collector.config;
+import java.io.File;
+
public class TaskRuntimeOptions extends Options {
public static final Option<Integer> TASK_SOURCE_PARALLELISM_NUM =
@@ -48,7 +50,7 @@ public class TaskRuntimeOptions extends Options {
public static final Option<Integer> TASK_PROCESSOR_RING_BUFFER_SIZE =
new Option<Integer>("task_processor_ring_buffer_size", 1024) {
@Override
- public void setValue(String valueString) {
+ public void setValue(final String valueString) {
value = Integer.parseInt(valueString);
}
};
@@ -56,8 +58,18 @@ public class TaskRuntimeOptions extends Options {
public static final Option<Integer> TASK_SINK_RING_BUFFER_SIZE =
new Option<Integer>("task_sink_ring_buffer_size", 1024) {
@Override
- public void setValue(String valueString) {
+ public void setValue(final String valueString) {
value = Integer.parseInt(valueString);
}
};
+
+ public static final Option<String> TASK_DATABASE_FILE_PATH =
+ new Option<String>(
+ "task_database_file_path",
+ "system" + File.separator + "database" + File.separator + "task.db") {
+ @Override
+ public void setValue(final String valueString) {
+ value = addHomeDir(valueString);
+ }
+ };
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/DBConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/DBConstant.java
index 2e8c3bb..5e18ce1 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/DBConstant.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/DBConstant.java
@@ -19,6 +19,9 @@
package org.apache.iotdb.collector.persistence;
+import org.apache.iotdb.collector.config.PluginRuntimeOptions;
+import org.apache.iotdb.collector.config.TaskRuntimeOptions;
+
public class DBConstant {
public static final String CREATE_PLUGIN_TABLE_SQL =
@@ -41,9 +44,11 @@ public class DBConstant {
+ " create_time TEXT NOT NULL\n"
+ ");";
- public static final String PLUGIN_DATABASE_FILE_PATH = "ext/db/plugin.db";
- public static final String TASK_DATABASE_FILE_PATH = "ext/db/task.db";
+ public static final String PLUGIN_DATABASE_FILE_PATH = "system/database/plugin.db";
+ public static final String TASK_DATABASE_FILE_PATH = "system/database/task.db";
- public static final String PLUGIN_DATABASE_URL = "jdbc:sqlite:" + PLUGIN_DATABASE_FILE_PATH;
- public static final String TASK_DATABASE_URL = "jdbc:sqlite:" + TASK_DATABASE_FILE_PATH;
+ public static final String PLUGIN_DATABASE_URL =
+ "jdbc:sqlite:" + PluginRuntimeOptions.PLUGIN_DATABASE_FILE_PATH.value();
+ public static final String TASK_DATABASE_URL =
+ "jdbc:sqlite:" + TaskRuntimeOptions.TASK_DATABASE_FILE_PATH.value();
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/Persistence.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/Persistence.java
index b2abcb8..72d5670 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/Persistence.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/Persistence.java
@@ -29,6 +29,7 @@ public abstract class Persistence {
public Persistence(final String databaseUrl) {
this.databaseUrl = databaseUrl;
+
initDatabaseFileIfPossible();
initTableIfPossible();
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/PluginPersistence.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/PluginPersistence.java
index d1d2b20..f575fd9 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/PluginPersistence.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/PluginPersistence.java
@@ -19,8 +19,9 @@
package org.apache.iotdb.collector.persistence;
-import org.apache.iotdb.collector.runtime.plugin.utils.PluginFileUtils;
+import org.apache.iotdb.collector.config.PluginRuntimeOptions;
import org.apache.iotdb.collector.service.RuntimeService;
+import org.apache.iotdb.collector.utils.PluginFileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,7 +50,8 @@ public class PluginPersistence extends Persistence {
@Override
protected void initDatabaseFileIfPossible() {
try {
- final Path pluginDatabaseFilePath = Paths.get(DBConstant.PLUGIN_DATABASE_FILE_PATH);
+ final Path pluginDatabaseFilePath =
+ Paths.get(PluginRuntimeOptions.PLUGIN_DATABASE_FILE_PATH.value());
if (!Files.exists(pluginDatabaseFilePath)) {
Files.createFile(pluginDatabaseFilePath);
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/TaskPersistence.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/TaskPersistence.java
index bf62519..b911fe1 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/TaskPersistence.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/TaskPersistence.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.collector.persistence;
+import org.apache.iotdb.collector.config.TaskRuntimeOptions;
import org.apache.iotdb.collector.runtime.task.TaskStateEnum;
import org.apache.iotdb.collector.service.RuntimeService;
import org.apache.iotdb.tsfile.utils.PublicBAOS;
@@ -55,7 +56,8 @@ public class TaskPersistence extends Persistence {
@Override
protected void initDatabaseFileIfPossible() {
try {
- final Path taskDatabaseFilePath = Paths.get(DBConstant.TASK_DATABASE_FILE_PATH);
+ final Path taskDatabaseFilePath =
+ Paths.get(TaskRuntimeOptions.TASK_DATABASE_FILE_PATH.value());
if (!Files.exists(taskDatabaseFilePath)) {
Files.createFile(taskDatabaseFilePath);
}
@@ -119,6 +121,7 @@ public class TaskPersistence extends Persistence {
if (Objects.isNull(response) || response.getStatus() != Response.Status.OK.getStatusCode()) {
LOGGER.warn("Failed to recover task persistence message, because {}", response);
+ tryDeleteTask(taskId);
}
}
@@ -160,6 +163,8 @@ public class TaskPersistence extends Persistence {
statement.setBytes(5, sinkAttributeBuffer);
statement.setString(6, String.valueOf(new Timestamp(System.currentTimeMillis())));
statement.executeUpdate();
+
+ LOGGER.info("successfully persisted task {} info", taskId);
} catch (final SQLException | IOException e) {
LOGGER.warn("Failed to persistence task message, because {}", e.getMessage());
}
@@ -186,6 +191,8 @@ public class TaskPersistence extends Persistence {
final PreparedStatement statement = connection.prepareStatement(deleteSQL);
statement.setString(1, taskId);
statement.executeUpdate();
+
+ LOGGER.info("successfully deleted task {}", taskId);
} catch (final SQLException e) {
LOGGER.warn("Failed to delete task persistence message, because {}", e.getMessage());
}
@@ -199,6 +206,8 @@ public class TaskPersistence extends Persistence {
statement.setInt(1, taskState.getTaskState());
statement.setString(2, taskId);
statement.executeUpdate();
+
+ LOGGER.info("successfully altered task {}", taskId);
} catch (SQLException e) {
LOGGER.warn("Failed to alter task persistence message, because {}", e.getMessage());
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
index e15a493..47313ee 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
@@ -20,5 +20,15 @@
package org.apache.iotdb.collector.plugin.api;
import org.apache.iotdb.pipe.api.PipeSource;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
-public abstract class PullSource implements PipeSource {}
+public abstract class PullSource implements PipeSource {
+
+ @Override
+ public final void customize(
+ PipeParameters pipeParameters,
+ PipeExtractorRuntimeConfiguration pipeExtractorRuntimeConfiguration) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PushSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PushSource.java
index e8eebf4..7663c09 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PushSource.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PushSource.java
@@ -21,6 +21,8 @@ package org.apache.iotdb.collector.plugin.api;
import org.apache.iotdb.pipe.api.PipeSource;
import org.apache.iotdb.pipe.api.collector.EventCollector;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.apache.iotdb.pipe.api.event.Event;
public abstract class PushSource implements PipeSource {
@@ -31,6 +33,14 @@ public abstract class PushSource implements PipeSource {
this.collector = null;
}
+ @Override
+ public final void customize(
+ PipeParameters pipeParameters,
+ PipeExtractorRuntimeConfiguration pipeExtractorRuntimeConfiguration)
+ throws Exception {
+ throw new UnsupportedOperationException();
+ }
+
public final void setCollector(final EventCollector collector) {
this.collector = collector;
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java
index 8ace81c..07c60a2 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/customizer/CollectorRuntimeEnvironment.java
@@ -49,6 +49,11 @@ public class CollectorRuntimeEnvironment implements PipeRuntimeEnvironment {
return creationTime;
}
+ @Override
+ public int getRegionId() {
+ return getInstanceIndex();
+ }
+
public int getParallelism() {
return parallelism;
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/event/PeriodicalEvent.java
similarity index 84%
copy from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
copy to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/event/PeriodicalEvent.java
index e15a493..6d85c9c 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/event/PeriodicalEvent.java
@@ -17,8 +17,8 @@
* under the License.
*/
-package org.apache.iotdb.collector.plugin.api;
+package org.apache.iotdb.collector.plugin.api.event;
-import org.apache.iotdb.pipe.api.PipeSource;
+import org.apache.iotdb.pipe.api.event.Event;
-public abstract class PullSource implements PipeSource {}
+public class PeriodicalEvent implements Event {}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/BuiltinPlugin.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/BuiltinPlugin.java
index 203570c..3ae111f 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/BuiltinPlugin.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/BuiltinPlugin.java
@@ -20,9 +20,12 @@
package org.apache.iotdb.collector.plugin.builtin;
import org.apache.iotdb.collector.plugin.builtin.processor.DoNothingProcessor;
+import org.apache.iotdb.collector.plugin.builtin.processor.SubscriptionProcessor;
import org.apache.iotdb.collector.plugin.builtin.sink.DemoSink;
+import org.apache.iotdb.collector.plugin.builtin.sink.protocol.IoTDBDataRegionSyncConnector;
import org.apache.iotdb.collector.plugin.builtin.source.HttpPullSource;
import org.apache.iotdb.collector.plugin.builtin.source.HttpPushSource;
+import org.apache.iotdb.collector.plugin.builtin.source.IoTDBPushSource;
public enum BuiltinPlugin {
@@ -31,12 +34,15 @@ public enum BuiltinPlugin {
// Pull Sources
HTTP_PULL_SOURCE("http-pull-source", HttpPullSource.class),
+ SUBSCRIPTION_SOURCE("subscription-source", IoTDBPushSource.class),
// Processors
DO_NOTHING_PROCESSOR("do-nothing-processor", DoNothingProcessor.class),
+ SUBSCRIPTION_PROCESSOR("subscription-processor", SubscriptionProcessor.class),
// Sinks
- IOTDB_THRIFT_SINK("iotdb-thrift-sink", DemoSink.class);
+ IOTDB_DEMO_SINK("iotdb-demo-sink", DemoSink.class),
+ IOTDB_SYNC_SINK("iotdb-sync-sink", IoTDBDataRegionSyncConnector.class);
private final String collectorPluginName;
private final Class<?> collectorPluginClass;
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/processor/SubscriptionProcessor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/processor/SubscriptionProcessor.java
new file mode 100644
index 0000000..fdf2c9e
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/processor/SubscriptionProcessor.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.processor;
+
+import org.apache.iotdb.collector.plugin.builtin.source.event.SubDemoEvent;
+import org.apache.iotdb.pipe.api.PipeProcessor;
+import org.apache.iotdb.pipe.api.collector.EventCollector;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeProcessorRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+
+public class SubscriptionProcessor implements PipeProcessor {
+ @Override
+ public void validate(PipeParameterValidator pipeParameterValidator) throws Exception {}
+
+ @Override
+ public void customize(
+ PipeParameters pipeParameters,
+ PipeProcessorRuntimeConfiguration pipeProcessorRuntimeConfiguration)
+ throws Exception {}
+
+ @Override
+ public void process(TabletInsertionEvent tabletInsertionEvent, EventCollector eventCollector)
+ throws Exception {
+ eventCollector.collect(tabletInsertionEvent);
+ }
+
+ @Override
+ public void process(Event event, EventCollector eventCollector) throws Exception {
+ if (event instanceof SubDemoEvent) {
+ process((SubDemoEvent) event, eventCollector);
+ }
+ }
+
+ @Override
+ public void close() throws Exception {}
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBClientManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBClientManager.java
new file mode 100644
index 0000000..521debe
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBClientManager.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.client;
+
+import org.apache.iotdb.collector.config.PipeRuntimeOptions;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.SocketTimeoutException;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public abstract class IoTDBClientManager {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBClientManager.class);
+
+ protected final List<TEndPoint> endPointList;
+ protected long currentClientIndex = 0;
+
+ protected final boolean useLeaderCache;
+
+ protected final String username;
+ protected final String password;
+
+ protected final boolean validateTsFile;
+
+ protected final boolean shouldReceiverConvertOnTypeMismatch;
+ protected final String loadTsFileStrategy;
+
+ protected final boolean shouldMarkAsPipeRequest;
+
+ // This flag indicates whether the receiver supports mods transferring if
+ // it is a DataNode receiver. The flag is useless for configNode receiver.
+ protected boolean supportModsIfIsDataNodeReceiver = true;
+
+ private static final int MAX_CONNECTION_TIMEOUT_MS = 24 * 60 * 60 * 1000; // 1 day
+ private static final int FIRST_ADJUSTMENT_TIMEOUT_MS = 6 * 60 * 60 * 1000; // 6 hours
+ protected static final AtomicInteger CONNECTION_TIMEOUT_MS =
+ new AtomicInteger(PipeRuntimeOptions.PIPE_CONNECTOR_TRANSFER_TIMEOUT_MS.value());
+
+ protected IoTDBClientManager(
+ final List<TEndPoint> endPointList,
+ /* The following parameters are used locally. */
+ final boolean useLeaderCache,
+ /* The following parameters are used to handshake with the receiver. */
+ final String username,
+ final String password,
+ final boolean shouldReceiverConvertOnTypeMismatch,
+ final String loadTsFileStrategy,
+ final boolean validateTsFile,
+ final boolean shouldMarkAsPipeRequest) {
+ this.endPointList = endPointList;
+
+ this.useLeaderCache = useLeaderCache;
+
+ this.username = username;
+ this.password = password;
+ this.shouldReceiverConvertOnTypeMismatch = shouldReceiverConvertOnTypeMismatch;
+ this.loadTsFileStrategy = loadTsFileStrategy;
+ this.validateTsFile = validateTsFile;
+ this.shouldMarkAsPipeRequest = shouldMarkAsPipeRequest;
+ }
+
+ public boolean supportModsIfIsDataNodeReceiver() {
+ return supportModsIfIsDataNodeReceiver;
+ }
+
+ public void adjustTimeoutIfNecessary(Throwable e) {
+ do {
+ if (e instanceof SocketTimeoutException || e instanceof TimeoutException) {
+ int newConnectionTimeout;
+ try {
+ newConnectionTimeout =
+ Math.min(
+ Math.max(
+ FIRST_ADJUSTMENT_TIMEOUT_MS,
+ Math.toIntExact(CONNECTION_TIMEOUT_MS.get() * 2L)),
+ MAX_CONNECTION_TIMEOUT_MS);
+ } catch (ArithmeticException arithmeticException) {
+ newConnectionTimeout = MAX_CONNECTION_TIMEOUT_MS;
+ }
+
+ if (newConnectionTimeout != CONNECTION_TIMEOUT_MS.get()) {
+ CONNECTION_TIMEOUT_MS.set(newConnectionTimeout);
+ LOGGER.info(
+ "Pipe connection timeout is adjusted to {} ms ({} mins)",
+ newConnectionTimeout,
+ newConnectionTimeout / 60000.0);
+ }
+ return;
+ }
+ } while ((e = e.getCause()) != null);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeCacheLeaderClientManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeCacheLeaderClientManager.java
new file mode 100644
index 0000000..dff5f2d
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeCacheLeaderClientManager.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.client;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.Weigher;
+import com.google.common.util.concurrent.AtomicDouble;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+public interface IoTDBDataNodeCacheLeaderClientManager {
+
+ LeaderCacheManager LEADER_CACHE_MANAGER = new LeaderCacheManager();
+
+ class LeaderCacheManager {
+
+ private final AtomicDouble memoryUsageCheatFactor = new AtomicDouble(1);
+
+ // leader cache built by LRU
+ private final Cache<String, TEndPoint> device2endpoint;
+ // a hashmap to reuse the created endpoint
+ private final ConcurrentHashMap<TEndPoint, TEndPoint> endPoints = new ConcurrentHashMap<>();
+
+ public LeaderCacheManager() {
+ device2endpoint =
+ Caffeine.newBuilder()
+ .weigher(
+ (Weigher<String, TEndPoint>)
+ (device, endPoint) -> {
+ final long weightInLong =
+ (long) (device.getBytes().length * memoryUsageCheatFactor.get());
+ if (weightInLong <= 0) {
+ return Integer.MAX_VALUE;
+ }
+ final int weightInInt = (int) weightInLong;
+ return weightInInt != weightInLong ? Integer.MAX_VALUE : weightInInt;
+ })
+ .recordStats()
+ .build();
+ }
+
+ public TEndPoint getLeaderEndPoint(final String deviceId) {
+ return deviceId == null ? null : device2endpoint.getIfPresent(deviceId);
+ }
+
+ public void updateLeaderEndPoint(final String deviceId, final TEndPoint endPoint) {
+ if (deviceId == null || endPoint == null) {
+ return;
+ }
+
+ final TEndPoint endPointFromMap = endPoints.putIfAbsent(endPoint, endPoint);
+ if (endPointFromMap != null) {
+ device2endpoint.put(deviceId, endPointFromMap);
+ } else {
+ device2endpoint.put(deviceId, endPoint);
+ }
+ }
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeSyncClientManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeSyncClientManager.java
new file mode 100644
index 0000000..8992f8e
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBDataNodeSyncClientManager.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.client;
+
+import org.apache.iotdb.collector.config.PipeRuntimeOptions;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferDataNodeHandshakeV1Req;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferDataNodeHandshakeV2Req;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferHandshakeV2Req;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+
+import org.apache.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class IoTDBDataNodeSyncClientManager extends IoTDBSyncClientManager
+ implements IoTDBDataNodeCacheLeaderClientManager {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(IoTDBDataNodeSyncClientManager.class);
+
+ public IoTDBDataNodeSyncClientManager(
+ final List<TEndPoint> endPoints,
+ final boolean useSSL,
+ final String trustStorePath,
+ final String trustStorePwd,
+ /* The following parameters are used locally. */
+ final boolean useLeaderCache,
+ final String loadBalanceStrategy,
+ /* The following parameters are used to handshake with the receiver. */
+ final String username,
+ final String password,
+ final boolean shouldReceiverConvertOnTypeMismatch,
+ final String loadTsFileStrategy,
+ final boolean validateTsFile,
+ final boolean shouldMarkAsPipeRequest) {
+ super(
+ endPoints,
+ useSSL,
+ trustStorePath,
+ trustStorePwd,
+ useLeaderCache,
+ loadBalanceStrategy,
+ username,
+ password,
+ shouldReceiverConvertOnTypeMismatch,
+ loadTsFileStrategy,
+ validateTsFile,
+ shouldMarkAsPipeRequest);
+ }
+
+ @Override
+ protected PipeTransferDataNodeHandshakeV1Req buildHandshakeV1Req() throws IOException {
+ return PipeTransferDataNodeHandshakeV1Req.toTPipeTransferReq(
+ PipeRuntimeOptions.TIMESTAMP_PRECISION.value());
+ }
+
+ @Override
+ protected PipeTransferHandshakeV2Req buildHandshakeV2Req(final Map<String, String> params)
+ throws IOException {
+ return PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params);
+ }
+
+ @Override
+ protected String getClusterId() {
+ return PipeRuntimeOptions.CLUSTER_ID.value();
+ }
+
+ public Pair<IoTDBSyncClient, Boolean> getClient(final String deviceId) {
+ final TEndPoint endPoint = LEADER_CACHE_MANAGER.getLeaderEndPoint(deviceId);
+ return useLeaderCache
+ && endPoint != null
+ && endPoint2ClientAndStatus.containsKey(endPoint)
+ && Boolean.TRUE.equals(endPoint2ClientAndStatus.get(endPoint).getRight())
+ ? endPoint2ClientAndStatus.get(endPoint)
+ : getClient();
+ }
+
+ public Pair<IoTDBSyncClient, Boolean> getClient(final TEndPoint endPoint) {
+ return useLeaderCache
+ && endPoint != null
+ && endPoint2ClientAndStatus.containsKey(endPoint)
+ && Boolean.TRUE.equals(endPoint2ClientAndStatus.get(endPoint).getRight())
+ ? endPoint2ClientAndStatus.get(endPoint)
+ : getClient();
+ }
+
+ public void updateLeaderCache(final String deviceId, final TEndPoint endPoint) {
+ if (!useLeaderCache || deviceId == null || endPoint == null) {
+ return;
+ }
+
+ try {
+ if (!endPoint2ClientAndStatus.containsKey(endPoint)) {
+ endPointList.add(endPoint);
+ endPoint2ClientAndStatus.put(endPoint, new Pair<>(null, false));
+ reconstructClient(endPoint);
+ }
+
+ LEADER_CACHE_MANAGER.updateLeaderEndPoint(deviceId, endPoint);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to update leader cache for device {} with endpoint {}:{}.",
+ deviceId,
+ endPoint.getIp(),
+ endPoint.getPort(),
+ e);
+ }
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClient.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClient.java
new file mode 100644
index 0000000..acb7be3
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClient.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.client;
+
+import org.apache.iotdb.collector.config.PipeRuntimeOptions;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.IoTDBConnectorRequestVersion;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferSliceReq;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.pipe.api.exception.PipeConnectionException;
+import org.apache.iotdb.rpc.DeepCopyRpcTransportFactory;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.rpc.TimeoutChangeableTransport;
+import org.apache.iotdb.service.rpc.thrift.IClientRPCService;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
+
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class IoTDBSyncClient extends IClientRPCService.Client
+ implements ThriftClient, AutoCloseable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSyncClient.class);
+
+ private static final AtomicInteger SLICE_ORDER_ID_GENERATOR = new AtomicInteger(0);
+
+ private final String ipAddress;
+ private final int port;
+ private final TEndPoint endPoint;
+
+ public IoTDBSyncClient(
+ ThriftClientProperty property,
+ String ipAddress,
+ int port,
+ boolean useSSL,
+ String trustStore,
+ String trustStorePwd)
+ throws TTransportException {
+ super(
+ property
+ .getProtocolFactory()
+ .getProtocol(
+ useSSL
+ ? DeepCopyRpcTransportFactory.INSTANCE.getTransport(
+ ipAddress,
+ port,
+ property.getConnectionTimeoutMs(),
+ trustStore,
+ trustStorePwd)
+ : DeepCopyRpcTransportFactory.INSTANCE.getTransport(
+ ipAddress, port, property.getConnectionTimeoutMs())));
+ this.ipAddress = ipAddress;
+ this.port = port;
+ this.endPoint = new TEndPoint(ipAddress, port);
+ final TTransport transport = getInputProtocol().getTransport();
+ if (!transport.isOpen()) {
+ transport.open();
+ }
+ }
+
+ public String getIpAddress() {
+ return ipAddress;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public TEndPoint getEndPoint() {
+ return endPoint;
+ }
+
+ public void setTimeout(int timeout) {
+ ((TimeoutChangeableTransport) (getInputProtocol().getTransport())).setTimeout(timeout);
+ }
+
+ @Override
+ public TPipeTransferResp pipeTransfer(final TPipeTransferReq req) throws TException {
+ final int bodySizeLimit = (int) (PipeRuntimeOptions.THRIFT_FRAME_MAX_SIZE.value() * 0.8);
+ if (req.getVersion() != IoTDBConnectorRequestVersion.VERSION_1.getVersion()
+ || req.body.limit() < bodySizeLimit) {
+ return super.pipeTransfer(req);
+ }
+
+ LOGGER.warn(
+ "The body size of the request is too large. The request will be sliced. Origin req: {}-{}. "
+ + "Request body size: {}, threshold: {}",
+ req.getVersion(),
+ req.getType(),
+ req.body.limit(),
+ bodySizeLimit);
+
+ try {
+ final int sliceOrderId = SLICE_ORDER_ID_GENERATOR.getAndIncrement();
+ // Slice the buffer to avoid the buffer being too large
+ final int sliceCount =
+ req.body.limit() / bodySizeLimit + (req.body.limit() % bodySizeLimit == 0 ? 0 : 1);
+ for (int i = 0; i < sliceCount; ++i) {
+ final int startIndexInBody = i * bodySizeLimit;
+ final int endIndexInBody = Math.min((i + 1) * bodySizeLimit, req.body.limit());
+ final TPipeTransferResp sliceResp =
+ super.pipeTransfer(
+ PipeTransferSliceReq.toTPipeTransferReq(
+ sliceOrderId,
+ req.getType(),
+ i,
+ sliceCount,
+ req.body.duplicate(),
+ startIndexInBody,
+ endIndexInBody));
+
+ if (i == sliceCount - 1) {
+ return sliceResp;
+ }
+
+ if (sliceResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new PipeConnectionException(
+ String.format(
+ "Failed to transfer slice. Origin req: %s-%s, slice index: %d, slice count: %d. Reason: %s",
+ req.getVersion(), req.getType(), i, sliceCount, sliceResp.getStatus()));
+ }
+ }
+
+ // Should not reach here
+ return super.pipeTransfer(req);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to transfer slice. Origin req: {}-{}. Retry the whole transfer.",
+ req.getVersion(),
+ req.getType(),
+ e);
+ // Fall back to the original behavior
+ return super.pipeTransfer(req);
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ invalidate();
+ }
+
+ @Override
+ public void invalidate() {
+ if (getInputProtocol().getTransport().isOpen()) {
+ getInputProtocol().getTransport().close();
+ }
+ }
+
+ @Override
+ public void invalidateAll() {
+ invalidate();
+ }
+
+ @Override
+ public boolean printLogWhenEncounterException() {
+ return true;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClientManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClientManager.java
new file mode 100644
index 0000000..fadb2d2
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/IoTDBSyncClientManager.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.client;
+
+import org.apache.iotdb.collector.config.PipeRuntimeOptions;
+import org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeTransferHandshakeConstant;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferHandshakeV1Req;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferHandshakeV2Req;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.pipe.api.exception.PipeConnectionException;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
+
+import org.apache.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY;
+
+public abstract class IoTDBSyncClientManager extends IoTDBClientManager implements Closeable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSyncClientManager.class);
+
+ private final boolean useSSL;
+ private final String trustStorePath;
+ private final String trustStorePwd;
+
+ protected final Map<TEndPoint, Pair<IoTDBSyncClient, Boolean>> endPoint2ClientAndStatus =
+ new ConcurrentHashMap<>();
+ private final Map<TEndPoint, String> endPoint2HandshakeErrorMessage = new ConcurrentHashMap<>();
+
+ private final LoadBalancer loadBalancer;
+
+ protected IoTDBSyncClientManager(
+ List<TEndPoint> endPoints,
+ boolean useSSL,
+ String trustStorePath,
+ String trustStorePwd,
+ /* The following parameters are used locally. */
+ boolean useLeaderCache,
+ String loadBalanceStrategy,
+ /* The following parameters are used to handshake with the receiver. */
+ String username,
+ String password,
+ boolean shouldReceiverConvertOnTypeMismatch,
+ String loadTsFileStrategy,
+ boolean validateTsFile,
+ boolean shouldMarkAsPipeRequest) {
+ super(
+ endPoints,
+ useLeaderCache,
+ username,
+ password,
+ shouldReceiverConvertOnTypeMismatch,
+ loadTsFileStrategy,
+ validateTsFile,
+ shouldMarkAsPipeRequest);
+
+ this.useSSL = useSSL;
+ this.trustStorePath = trustStorePath;
+ this.trustStorePwd = trustStorePwd;
+
+ for (final TEndPoint endPoint : endPoints) {
+ endPoint2ClientAndStatus.put(endPoint, new Pair<>(null, false));
+ }
+
+ switch (loadBalanceStrategy) {
+ case CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY:
+ loadBalancer = new RoundRobinLoadBalancer();
+ break;
+ case CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY:
+ loadBalancer = new RandomLoadBalancer();
+ break;
+ case CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY:
+ loadBalancer = new PriorityLoadBalancer();
+ break;
+ default:
+ LOGGER.warn(
+ "Unknown load balance strategy: {}, use round-robin strategy instead.",
+ loadBalanceStrategy);
+ loadBalancer = new RoundRobinLoadBalancer();
+ }
+ }
+
+ public void checkClientStatusAndTryReconstructIfNecessary() {
+ // Reconstruct all dead clients
+ for (final Map.Entry<TEndPoint, Pair<IoTDBSyncClient, Boolean>> entry :
+ endPoint2ClientAndStatus.entrySet()) {
+ if (Boolean.TRUE.equals(entry.getValue().getRight())) {
+ continue;
+ }
+
+ reconstructClient(entry.getKey());
+ }
+
+ // Check whether any clients are available
+ for (final Pair<IoTDBSyncClient, Boolean> clientAndStatus : endPoint2ClientAndStatus.values()) {
+ if (Boolean.TRUE.equals(clientAndStatus.getRight())) {
+ return;
+ }
+ }
+
+ // If all clients are not available, throw an exception
+ final StringBuilder errorMessage =
+ new StringBuilder(
+ String.format(
+ "All target servers %s are not available.", endPoint2ClientAndStatus.keySet()));
+ for (final Map.Entry<TEndPoint, String> entry : endPoint2HandshakeErrorMessage.entrySet()) {
+ errorMessage
+ .append(" (")
+ .append("host: ")
+ .append(entry.getKey().getIp())
+ .append(", port: ")
+ .append(entry.getKey().getPort())
+ .append(", because: ")
+ .append(entry.getValue())
+ .append(")");
+ }
+ throw new PipeConnectionException(errorMessage.toString());
+ }
+
+ protected void reconstructClient(TEndPoint endPoint) {
+ endPoint2HandshakeErrorMessage.remove(endPoint);
+
+ final Pair<IoTDBSyncClient, Boolean> clientAndStatus = endPoint2ClientAndStatus.get(endPoint);
+
+ if (clientAndStatus.getLeft() != null) {
+ try {
+ clientAndStatus.getLeft().close();
+ } catch (Exception e) {
+ LOGGER.warn(
+ "Failed to close client with target server ip: {}, port: {}, because: {}. Ignore it.",
+ endPoint.getIp(),
+ endPoint.getPort(),
+ e.getMessage());
+ }
+ }
+
+ // It is necessary to ensure that the client is initialized successfully and not null. If false
+ // is returned, it means that the initialization is not successful and the handshake operation
+ // is not performed.
+ if (initClientAndStatus(clientAndStatus, endPoint)) {
+ sendHandshakeReq(clientAndStatus);
+ }
+ }
+
+ private boolean initClientAndStatus(
+ final Pair<IoTDBSyncClient, Boolean> clientAndStatus, final TEndPoint endPoint) {
+ try {
+ clientAndStatus.setLeft(
+ new IoTDBSyncClient(
+ new ThriftClientProperty.Builder()
+ .setConnectionTimeoutMs(
+ PipeRuntimeOptions.PIPE_CONNECTOR_TRANSFER_TIMEOUT_MS.value())
+ .setRpcThriftCompressionEnabled(
+ PipeRuntimeOptions.IS_PIPE_CONNECTOR_RPC_THRIFT_COMPRESSION_ENABLED.value())
+ .build(),
+ endPoint.getIp(),
+ endPoint.getPort(),
+ useSSL,
+ trustStorePath,
+ trustStorePwd));
+ return true;
+ } catch (Exception e) {
+ endPoint2HandshakeErrorMessage.put(endPoint, e.getMessage());
+ LOGGER.warn(
+ "Failed to initialize client with target server ip: {}, port: {}, because {}",
+ endPoint.getIp(),
+ endPoint.getPort(),
+ e.getMessage(),
+ e);
+ return false;
+ }
+ }
+
+ public void sendHandshakeReq(final Pair<IoTDBSyncClient, Boolean> clientAndStatus) {
+ final IoTDBSyncClient client = clientAndStatus.getLeft();
+ try {
+ final HashMap<String, String> params = new HashMap<>();
+ params.put(
+ PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION,
+ PipeRuntimeOptions.TIMESTAMP_PRECISION.value());
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId());
+ params.put(
+ PipeTransferHandshakeConstant.HANDSHAKE_KEY_CONVERT_ON_TYPE_MISMATCH,
+ Boolean.toString(shouldReceiverConvertOnTypeMismatch));
+ params.put(
+ PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, loadTsFileStrategy);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, username);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password);
+ params.put(
+ PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE,
+ Boolean.toString(validateTsFile));
+ params.put(
+ PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST,
+ Boolean.toString(shouldMarkAsPipeRequest));
+
+ // Try to handshake by PipeTransferHandshakeV2Req.
+ TPipeTransferResp resp = client.pipeTransfer(buildHandshakeV2Req(params));
+ // Receiver may be an old version, so we need to retry to handshake by
+ // PipeTransferHandshakeV1Req.
+ if (resp.getStatus().getCode() == TSStatusCode.PIPE_TYPE_ERROR.getStatusCode()) {
+ LOGGER.warn(
+ "Handshake error with target server ip: {}, port: {}, because: {}. "
+ + "Retry to handshake by PipeTransferHandshakeV1Req.",
+ client.getIpAddress(),
+ client.getPort(),
+ resp.getStatus());
+ supportModsIfIsDataNodeReceiver = false;
+ resp = client.pipeTransfer(buildHandshakeV1Req());
+ }
+
+ if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ LOGGER.warn(
+ "Handshake error with target server ip: {}, port: {}, because: {}.",
+ client.getIpAddress(),
+ client.getPort(),
+ resp.getStatus());
+ endPoint2HandshakeErrorMessage.put(client.getEndPoint(), resp.getStatus().getMessage());
+ } else {
+ clientAndStatus.setRight(true);
+ client.setTimeout(CONNECTION_TIMEOUT_MS.get());
+ LOGGER.info(
+ "Handshake success. Target server ip: {}, port: {}",
+ client.getIpAddress(),
+ client.getPort());
+ }
+ } catch (Exception e) {
+ LOGGER.warn(
+ "Handshake error with target server ip: {}, port: {}, because: {}.",
+ client.getIpAddress(),
+ client.getPort(),
+ e.getMessage(),
+ e);
+ endPoint2HandshakeErrorMessage.put(client.getEndPoint(), e.getMessage());
+ }
+ }
+
+ protected abstract PipeTransferHandshakeV1Req buildHandshakeV1Req() throws IOException;
+
+ protected abstract PipeTransferHandshakeV2Req buildHandshakeV2Req(Map<String, String> params)
+ throws IOException;
+
+ protected abstract String getClusterId();
+
+ public Pair<IoTDBSyncClient, Boolean> getClient() {
+ return loadBalancer.getClient();
+ }
+
+ @Override
+ public void close() {
+ for (final Map.Entry<TEndPoint, Pair<IoTDBSyncClient, Boolean>> entry :
+ endPoint2ClientAndStatus.entrySet()) {
+ final TEndPoint endPoint = entry.getKey();
+ final Pair<IoTDBSyncClient, Boolean> clientAndStatus = entry.getValue();
+
+ if (clientAndStatus == null) {
+ continue;
+ }
+
+ try {
+ if (clientAndStatus.getLeft() != null) {
+ clientAndStatus.getLeft().close();
+ clientAndStatus.setLeft(null);
+ }
+ LOGGER.info("Client {}:{} closed.", endPoint.getIp(), endPoint.getPort());
+ } catch (Exception e) {
+ LOGGER.warn(
+ "Failed to close client {}:{}, because: {}.",
+ endPoint.getIp(),
+ endPoint.getPort(),
+ e.getMessage(),
+ e);
+ } finally {
+ clientAndStatus.setRight(false);
+ }
+ }
+ }
+
+ /////////////////////// Strategies for load balance //////////////////////////
+
+ private interface LoadBalancer {
+ Pair<IoTDBSyncClient, Boolean> getClient();
+ }
+
+ private class RoundRobinLoadBalancer implements LoadBalancer {
+ @Override
+ public Pair<IoTDBSyncClient, Boolean> getClient() {
+ final int clientSize = endPointList.size();
+ // Round-robin, find the next alive client
+ for (int tryCount = 0; tryCount < clientSize; ++tryCount) {
+ final int clientIndex = (int) (currentClientIndex++ % clientSize);
+ final Pair<IoTDBSyncClient, Boolean> clientAndStatus =
+ endPoint2ClientAndStatus.get(endPointList.get(clientIndex));
+ if (Boolean.TRUE.equals(clientAndStatus.getRight())) {
+ return clientAndStatus;
+ }
+ }
+
+ throw new PipeConnectionException(
+ "All clients are dead, please check the connection to the receiver.");
+ }
+ }
+
+ private class RandomLoadBalancer implements LoadBalancer {
+ @Override
+ public Pair<IoTDBSyncClient, Boolean> getClient() {
+ final int clientSize = endPointList.size();
+ final int clientIndex = (int) (Math.random() * clientSize);
+ final Pair<IoTDBSyncClient, Boolean> clientAndStatus =
+ endPoint2ClientAndStatus.get(endPointList.get(clientIndex));
+ if (Boolean.TRUE.equals(clientAndStatus.getRight())) {
+ return clientAndStatus;
+ }
+
+ // Random, find the next alive client
+ for (int tryCount = 0; tryCount < clientSize - 1; ++tryCount) {
+ final int nextClientIndex = (clientIndex + tryCount + 1) % clientSize;
+ final Pair<IoTDBSyncClient, Boolean> nextClientAndStatus =
+ endPoint2ClientAndStatus.get(endPointList.get(nextClientIndex));
+ if (Boolean.TRUE.equals(nextClientAndStatus.getRight())) {
+ return nextClientAndStatus;
+ }
+ }
+
+ throw new PipeConnectionException(
+ "All clients are dead, please check the connection to the receiver.");
+ }
+ }
+
+ private class PriorityLoadBalancer implements LoadBalancer {
+ @Override
+ public Pair<IoTDBSyncClient, Boolean> getClient() {
+ // Priority, find the first alive client
+ for (final TEndPoint endPoint : endPointList) {
+ final Pair<IoTDBSyncClient, Boolean> clientAndStatus =
+ endPoint2ClientAndStatus.get(endPoint);
+ if (Boolean.TRUE.equals(clientAndStatus.getRight())) {
+ return clientAndStatus;
+ }
+ }
+
+ throw new PipeConnectionException(
+ "All clients are dead, please check the connection to the receiver.");
+ }
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClient.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClient.java
new file mode 100644
index 0000000..20061ef
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClient.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.client;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.util.Optional;
+
+/**
+ * This class defines the failed interfaces that thrift client needs to support so that the Thrift
+ * Client can clean up the clientManager when it receives the corresponding exception.
+ */
+public interface ThriftClient {
+
+ Logger logger = LoggerFactory.getLogger(ThriftClient.class);
+
+ /** Close this connection. */
+ void invalidate();
+
+ /** Removing all pooled instances corresponding to current instance's endpoint. */
+ void invalidateAll();
+
+ /**
+ * Whether to print logs when exceptions are encountered.
+ *
+ * @return result
+ */
+ boolean printLogWhenEncounterException();
+
+ /**
+ * Perform corresponding operations on ThriftClient o based on the Throwable t.
+ *
+ * @param t Throwable
+ * @param o ThriftClient
+ */
+ static void resolveException(Throwable t, ThriftClient o) {
+ Throwable origin = t;
+ if (t instanceof InvocationTargetException) {
+ origin = ((InvocationTargetException) t).getTargetException();
+ }
+ Throwable cur = origin;
+ if (cur instanceof TException) {
+ int level = 0;
+ while (cur != null) {
+ logger.debug(
+ "level-{} Exception class {}, message {}",
+ level,
+ cur.getClass().getName(),
+ cur.getMessage());
+ cur = cur.getCause();
+ level++;
+ }
+ o.invalidate();
+ }
+
+ Throwable rootCause = ExceptionUtils.getRootCause(origin);
+ if (rootCause != null) {
+ // if the exception is SocketException and its error message is Broken pipe, it means that
+ // the remote node may restart and all the connection we cached before should be cleared.
+ logger.debug(
+ "root cause message {}, LocalizedMessage {}, ",
+ rootCause.getMessage(),
+ rootCause.getLocalizedMessage(),
+ rootCause);
+ if (isConnectionBroken(rootCause)) {
+ if (o.printLogWhenEncounterException()) {
+ logger.info(
+ "Broken pipe error happened in sending RPC,"
+ + " we need to clear all previous cached connection, error msg is {}",
+ rootCause.toString());
+ }
+ o.invalidateAll();
+ }
+ }
+ }
+
+ /**
+ * Determine whether the target node has gone offline once based on the cause.
+ *
+ * @param cause Throwable
+ * @return true/false
+ */
+ static boolean isConnectionBroken(Throwable cause) {
+ return (cause instanceof SocketException && cause.getMessage().contains("Broken pipe"))
+ || (cause instanceof TTransportException
+ && (hasExpectedMessage(cause, "Socket is closed by peer")
+ || hasExpectedMessage(cause, "Read call frame size failed")))
+ || (cause instanceof IOException
+ && (hasExpectedMessage(cause, "Connection reset by peer")
+ || hasExpectedMessage(cause, "Broken pipe")))
+ || (cause instanceof ConnectException && hasExpectedMessage(cause, "Connection refused"));
+ }
+
+ static boolean hasExpectedMessage(Throwable cause, String expectedMessage) {
+ return Optional.ofNullable(cause.getMessage())
+ .map(m -> m.contains(expectedMessage))
+ .orElse(false);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClientProperty.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClientProperty.java
new file mode 100644
index 0000000..b85c273
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/client/ThriftClientProperty.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.client;
+
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.protocol.TProtocolFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/** This class defines the configurations commonly used by the Thrift Client. */
+public class ThriftClientProperty {
+
+ private final TProtocolFactory protocolFactory;
+ private final int connectionTimeoutMs;
+ private final int selectorNumOfAsyncClientPool;
+ private final boolean printLogWhenEncounterException;
+
+ private ThriftClientProperty(
+ TProtocolFactory protocolFactory,
+ int connectionTimeoutMs,
+ int selectorNumOfAsyncClientPool,
+ boolean printLogWhenEncounterException) {
+ this.protocolFactory = protocolFactory;
+ this.connectionTimeoutMs = connectionTimeoutMs;
+ this.selectorNumOfAsyncClientPool = selectorNumOfAsyncClientPool;
+ this.printLogWhenEncounterException = printLogWhenEncounterException;
+ }
+
+ public TProtocolFactory getProtocolFactory() {
+ return protocolFactory;
+ }
+
+ public int getConnectionTimeoutMs() {
+ return connectionTimeoutMs;
+ }
+
+ public int getSelectorNumOfAsyncClientPool() {
+ return selectorNumOfAsyncClientPool;
+ }
+
+ public boolean isPrintLogWhenEncounterException() {
+ return printLogWhenEncounterException;
+ }
+
+ public static class Builder {
+
+ /** whether to use thrift compression. */
+ private boolean rpcThriftCompressionEnabled = DefaultProperty.RPC_THRIFT_COMPRESSED_ENABLED;
+
+ /** socket timeout for thrift client. */
+ private int connectionTimeoutMs = DefaultProperty.CONNECTION_TIMEOUT_MS;
+
+ /** number of selector threads for asynchronous thrift client in a clientManager. */
+ private int selectorNumOfAsyncClientManager =
+ DefaultProperty.SELECTOR_NUM_OF_ASYNC_CLIENT_MANAGER;
+
+ /**
+ * Whether to print logs when the client encounters exceptions. For example, logs are not
+ * printed in the heartbeat client.
+ */
+ private boolean printLogWhenEncounterException =
+ DefaultProperty.PRINT_LOG_WHEN_ENCOUNTER_EXCEPTION;
+
+ public Builder setRpcThriftCompressionEnabled(boolean rpcThriftCompressionEnabled) {
+ this.rpcThriftCompressionEnabled = rpcThriftCompressionEnabled;
+ return this;
+ }
+
+ public Builder setConnectionTimeoutMs(int connectionTimeoutMs) {
+ this.connectionTimeoutMs = connectionTimeoutMs;
+ return this;
+ }
+
+ public Builder setSelectorNumOfAsyncClientManager(int selectorNumOfAsyncClientManager) {
+ this.selectorNumOfAsyncClientManager = selectorNumOfAsyncClientManager;
+ return this;
+ }
+
+ public Builder setPrintLogWhenEncounterException(boolean printLogWhenEncounterException) {
+ this.printLogWhenEncounterException = printLogWhenEncounterException;
+ return this;
+ }
+
+ public ThriftClientProperty build() {
+ return new ThriftClientProperty(
+ rpcThriftCompressionEnabled
+ ? new TCompactProtocol.Factory()
+ : new TBinaryProtocol.Factory(),
+ connectionTimeoutMs,
+ selectorNumOfAsyncClientManager,
+ printLogWhenEncounterException);
+ }
+ }
+
+ public static class DefaultProperty {
+
+ private DefaultProperty() {}
+
+ public static final boolean RPC_THRIFT_COMPRESSED_ENABLED = false;
+ public static final int CONNECTION_TIMEOUT_MS = (int) TimeUnit.SECONDS.toMillis(20);
+ public static final int CONNECTION_NEVER_TIMEOUT_MS = 0;
+ public static final int SELECTOR_NUM_OF_ASYNC_CLIENT_MANAGER = 1;
+ public static final boolean PRINT_LOG_WHEN_ENCOUNTER_EXCEPTION = true;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressor.java
new file mode 100644
index 0000000..dc3f1e7
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressor.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import java.io.IOException;
+
+public abstract class PipeCompressor {
+
+ public enum PipeCompressionType {
+ SNAPPY((byte) 0),
+ GZIP((byte) 1),
+ LZ4((byte) 2),
+ ZSTD((byte) 3),
+ LZMA2((byte) 4);
+
+ final byte index;
+
+ PipeCompressionType(byte index) {
+ this.index = index;
+ }
+
+ public byte getIndex() {
+ return index;
+ }
+ }
+
+ private final PipeCompressionType compressionType;
+
+ protected PipeCompressor(PipeCompressionType compressionType) {
+ this.compressionType = compressionType;
+ }
+
+ public abstract byte[] compress(byte[] data) throws IOException;
+
+ /**
+ * Decompress the byte array to a byte array. NOTE: the length of the decompressed byte array is
+ * not provided in this method, and some decompressors (LZ4) may construct large byte arrays,
+ * leading to potential OOM.
+ *
+ * @param byteArray the byte array to be decompressed
+ * @return the decompressed byte array
+ * @throws IOException
+ */
+ public abstract byte[] decompress(byte[] byteArray) throws IOException;
+
+ /**
+ * Decompress the byte array to a byte array with a known length.
+ *
+ * @param byteArray the byte array to be decompressed
+ * @param decompressedLength the length of the decompressed byte array
+ * @return the decompressed byte array
+ * @throws IOException
+ */
+ public abstract byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException;
+
+ public byte serialize() {
+ return compressionType.getIndex();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorConfig.java
similarity index 60%
copy from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
copy to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorConfig.java
index e15a493..fc54f3a 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorConfig.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -17,8 +17,23 @@
* under the License.
*/
-package org.apache.iotdb.collector.plugin.api;
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
-import org.apache.iotdb.pipe.api.PipeSource;
+public class PipeCompressorConfig {
-public abstract class PullSource implements PipeSource {}
+ private final String name;
+ private final int zstdCompressionLevel;
+
+ public PipeCompressorConfig(String name, int zstdCompressionLevel) {
+ this.name = name;
+ this.zstdCompressionLevel = zstdCompressionLevel;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getZstdCompressionLevel() {
+ return zstdCompressionLevel;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorFactory.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorFactory.java
new file mode 100644
index 0000000..e2da42f
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeCompressorFactory.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_GZIP;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_LZ4;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_LZMA2;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_SNAPPY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE;
+
+public class PipeCompressorFactory {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeCompressorFactory.class);
+
+ private static final Map<String, PipeCompressor> COMPRESSOR_NAME_TO_INSTANCE =
+ new ConcurrentHashMap<>();
+
+ static {
+ COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_SNAPPY, new PipeSnappyCompressor());
+ COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_GZIP, new PipeGZIPCompressor());
+ COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_LZ4, new PipeLZ4Compressor());
+ COMPRESSOR_NAME_TO_INSTANCE.put(
+ CONNECTOR_COMPRESSOR_ZSTD,
+ new PipeZSTDCompressor(CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE));
+ COMPRESSOR_NAME_TO_INSTANCE.put(CONNECTOR_COMPRESSOR_LZMA2, new PipeLZMA2Compressor());
+ }
+
+ public static PipeCompressor getCompressor(PipeCompressorConfig config) {
+ if (config == null) {
+ throw new IllegalArgumentException("PipeCompressorConfig is null");
+ }
+ if (config.getName() == null) {
+ throw new IllegalArgumentException("PipeCompressorConfig.getName() is null");
+ }
+
+ final String compressorName = config.getName();
+
+ // For ZSTD compressor, we need to consider the compression level
+ if (compressorName.equals(CONNECTOR_COMPRESSOR_ZSTD)) {
+ final int zstdCompressionLevel = config.getZstdCompressionLevel();
+ return COMPRESSOR_NAME_TO_INSTANCE.computeIfAbsent(
+ CONNECTOR_COMPRESSOR_ZSTD + "_" + zstdCompressionLevel,
+ key -> {
+ LOGGER.info("Create new PipeZSTDCompressor with level: {}", zstdCompressionLevel);
+ return new PipeZSTDCompressor(zstdCompressionLevel);
+ });
+ }
+
+ // For other compressors, we can directly get the instance by name
+ final PipeCompressor compressor = COMPRESSOR_NAME_TO_INSTANCE.get(compressorName);
+ if (compressor != null) {
+ return compressor;
+ }
+
+ throw new UnsupportedOperationException("PipeCompressor not found for name: " + compressorName);
+ }
+
+ private static Map<Byte, PipeCompressor> COMPRESSOR_INDEX_TO_INSTANCE = new HashMap<>();
+
+ static {
+ COMPRESSOR_INDEX_TO_INSTANCE.put(
+ PipeCompressor.PipeCompressionType.SNAPPY.getIndex(),
+ COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_SNAPPY));
+ COMPRESSOR_INDEX_TO_INSTANCE.put(
+ PipeCompressor.PipeCompressionType.GZIP.getIndex(),
+ COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_GZIP));
+ COMPRESSOR_INDEX_TO_INSTANCE.put(
+ PipeCompressor.PipeCompressionType.LZ4.getIndex(),
+ COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_LZ4));
+ COMPRESSOR_INDEX_TO_INSTANCE.put(
+ PipeCompressor.PipeCompressionType.ZSTD.getIndex(),
+ COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_ZSTD));
+ COMPRESSOR_INDEX_TO_INSTANCE.put(
+ PipeCompressor.PipeCompressionType.LZMA2.getIndex(),
+ COMPRESSOR_NAME_TO_INSTANCE.get(CONNECTOR_COMPRESSOR_LZMA2));
+ COMPRESSOR_INDEX_TO_INSTANCE = Collections.unmodifiableMap(COMPRESSOR_INDEX_TO_INSTANCE);
+ }
+
+ public static PipeCompressor getCompressor(byte index) {
+ final PipeCompressor compressor = COMPRESSOR_INDEX_TO_INSTANCE.get(index);
+ if (compressor == null) {
+ throw new UnsupportedOperationException("PipeCompressor not found for index: " + index);
+ }
+ return compressor;
+ }
+
+ private PipeCompressorFactory() {
+ // Empty constructor
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeGZIPCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeGZIPCompressor.java
new file mode 100644
index 0000000..7fbfc34
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeGZIPCompressor.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+
+import java.io.IOException;
+
+public class PipeGZIPCompressor extends PipeCompressor {
+
+ private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.GZIP);
+ private static final IUnCompressor DECOMPRESSOR =
+ IUnCompressor.getUnCompressor(CompressionType.GZIP);
+
+ public PipeGZIPCompressor() {
+ super(PipeCompressionType.GZIP);
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ return COMPRESSOR.compress(data);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray) throws IOException {
+ return DECOMPRESSOR.uncompress(byteArray);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException {
+ byte[] uncompressed = new byte[decompressedLength];
+ DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0);
+ return uncompressed;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZ4Compressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZ4Compressor.java
new file mode 100644
index 0000000..725b15e
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZ4Compressor.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+
+import java.io.IOException;
+
+public class PipeLZ4Compressor extends PipeCompressor {
+
+ private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.LZ4);
+ private static final IUnCompressor DECOMPRESSOR =
+ IUnCompressor.getUnCompressor(CompressionType.LZ4);
+
+ public PipeLZ4Compressor() {
+ super(PipeCompressionType.LZ4);
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ return COMPRESSOR.compress(data);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray) throws IOException {
+ return DECOMPRESSOR.uncompress(byteArray);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException {
+ byte[] uncompressed = new byte[decompressedLength];
+ DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0);
+ return uncompressed;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZMA2Compressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZMA2Compressor.java
new file mode 100644
index 0000000..2fd459c
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeLZMA2Compressor.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+
+import java.io.IOException;
+
+public class PipeLZMA2Compressor extends PipeCompressor {
+
+ private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.LZMA2);
+ private static final IUnCompressor DECOMPRESSOR =
+ IUnCompressor.getUnCompressor(CompressionType.LZMA2);
+
+ public PipeLZMA2Compressor() {
+ super(PipeCompressionType.LZMA2);
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ return COMPRESSOR.compress(data);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray) throws IOException {
+ return DECOMPRESSOR.uncompress(byteArray);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException {
+ byte[] uncompressed = new byte[decompressedLength];
+ DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0);
+ return uncompressed;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeSnappyCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeSnappyCompressor.java
new file mode 100644
index 0000000..101b445
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeSnappyCompressor.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+
+import java.io.IOException;
+
+public class PipeSnappyCompressor extends PipeCompressor {
+
+ private static final ICompressor COMPRESSOR = ICompressor.getCompressor(CompressionType.SNAPPY);
+ private static final IUnCompressor DECOMPRESSOR =
+ IUnCompressor.getUnCompressor(CompressionType.SNAPPY);
+
+ public PipeSnappyCompressor() {
+ super(PipeCompressionType.SNAPPY);
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ return COMPRESSOR.compress(data);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray) throws IOException {
+ return DECOMPRESSOR.uncompress(byteArray);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray, int decompressedLength) throws IOException {
+ byte[] uncompressed = new byte[decompressedLength];
+ DECOMPRESSOR.uncompress(byteArray, 0, byteArray.length, uncompressed, 0);
+ return uncompressed;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeZSTDCompressor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeZSTDCompressor.java
new file mode 100644
index 0000000..67e40ac
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/compressor/PipeZSTDCompressor.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.compressor;
+
+import com.github.luben.zstd.Zstd;
+
+import java.io.IOException;
+
+public class PipeZSTDCompressor extends PipeCompressor {
+
+ private final int compressionLevel;
+
+ public PipeZSTDCompressor(int compressionLevel) {
+ super(PipeCompressionType.ZSTD);
+ this.compressionLevel = compressionLevel;
+ }
+
+ @Override
+ public byte[] compress(byte[] data) throws IOException {
+ return Zstd.compress(data, compressionLevel);
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray) {
+ return Zstd.decompress(byteArray, (int) Zstd.decompressedSize(byteArray, 0, byteArray.length));
+ }
+
+ @Override
+ public byte[] decompress(byte[] byteArray, int decompressedLength) {
+ return Zstd.decompress(byteArray, decompressedLength);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/ColumnHeaderConstant.java
similarity index 67%
copy from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
copy to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/ColumnHeaderConstant.java
index e15a493..5d4db67 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/ColumnHeaderConstant.java
@@ -17,8 +17,19 @@
* under the License.
*/
-package org.apache.iotdb.collector.plugin.api;
+package org.apache.iotdb.collector.plugin.builtin.sink.constant;
-import org.apache.iotdb.pipe.api.PipeSource;
+public class ColumnHeaderConstant {
-public abstract class PullSource implements PipeSource {}
+ private ColumnHeaderConstant() {
+ // forbidding instantiation
+ }
+
+ public static final String DATABASE = "Database";
+
+ public static final String PATH_PATTERN = "PathPattern";
+
+ public static final String TYPE = "Type";
+
+ public static final String TABLE_NAME = "TableName";
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeConnectorConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeConnectorConstant.java
new file mode 100644
index 0000000..2c65761
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeConnectorConstant.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.constant;
+
+import com.github.luben.zstd.Zstd;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+public class PipeConnectorConstant {
+
+ public static final long MB = 1L << 20;
+
+ public static final String CONNECTOR_KEY = "connector";
+ public static final String SINK_KEY = "sink";
+
+ public static final String CONNECTOR_IOTDB_IP_KEY = "connector.ip";
+ public static final String SINK_IOTDB_IP_KEY = "sink.ip";
+ public static final String CONNECTOR_IOTDB_HOST_KEY = "connector.host";
+ public static final String SINK_IOTDB_HOST_KEY = "sink.host";
+ public static final String CONNECTOR_IOTDB_PORT_KEY = "connector.port";
+ public static final String SINK_IOTDB_PORT_KEY = "sink.port";
+ public static final String CONNECTOR_IOTDB_NODE_URLS_KEY = "connector.node-urls";
+ public static final String SINK_IOTDB_NODE_URLS_KEY = "sink.node-urls";
+
+ public static final String SINK_IOTDB_SSL_ENABLE_KEY = "sink.ssl.enable";
+ public static final String SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY = "sink.ssl.trust-store-path";
+ public static final String SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY = "sink.ssl.trust-store-pwd";
+
+ public static final String CONNECTOR_IOTDB_PARALLEL_TASKS_KEY = "connector.parallel.tasks";
+ public static final String SINK_IOTDB_PARALLEL_TASKS_KEY = "sink.parallel.tasks";
+
+ public static final String CONNECTOR_REALTIME_FIRST_KEY = "connector.realtime-first";
+ public static final String SINK_REALTIME_FIRST_KEY = "sink.realtime-first";
+ public static final boolean CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY = "connector.batch.enable";
+ public static final String SINK_IOTDB_BATCH_MODE_ENABLE_KEY = "sink.batch.enable";
+ public static final boolean CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_IOTDB_BATCH_DELAY_KEY = "connector.batch.max-delay-seconds";
+ public static final String SINK_IOTDB_BATCH_DELAY_KEY = "sink.batch.max-delay-seconds";
+ public static final int CONNECTOR_IOTDB_PLAIN_BATCH_DELAY_DEFAULT_VALUE = 1;
+ public static final int CONNECTOR_IOTDB_TS_FILE_BATCH_DELAY_DEFAULT_VALUE = 5;
+
+ public static final String CONNECTOR_IOTDB_BATCH_SIZE_KEY = "connector.batch.size-bytes";
+ public static final String SINK_IOTDB_BATCH_SIZE_KEY = "sink.batch.size-bytes";
+ public static final long CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE = 16 * MB;
+ public static final long CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE = 80 * MB;
+
+ public static final String CONNECTOR_IOTDB_USER_KEY = "connector.user";
+ public static final String SINK_IOTDB_USER_KEY = "sink.user";
+ public static final String CONNECTOR_IOTDB_USERNAME_KEY = "connector.username";
+ public static final String SINK_IOTDB_USERNAME_KEY = "sink.username";
+ public static final String CONNECTOR_IOTDB_USER_DEFAULT_VALUE = "root";
+
+ public static final String CONNECTOR_IOTDB_PASSWORD_KEY = "connector.password";
+ public static final String SINK_IOTDB_PASSWORD_KEY = "sink.password";
+ public static final String CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE = "root";
+
+ public static final String CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY =
+ "connector.exception.data.convert-on-type-mismatch";
+ public static final String SINK_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY =
+ "sink.exception.data.convert-on-type-mismatch";
+ public static final boolean CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE =
+ true;
+
+ public static final String CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY =
+ "connector.exception.conflict.resolve-strategy";
+ public static final String SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY =
+ "sink.exception.conflict.resolve-strategy";
+ public static final String CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE = "retry";
+
+ public static final String CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_KEY =
+ "connector.exception.conflict.retry-max-time-seconds";
+ public static final String SINK_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_KEY =
+ "sink.exception.conflict.retry-max-time-seconds";
+ public static final long CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE = 60;
+
+ public static final String CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_KEY =
+ "connector.exception.conflict.record-ignored-data";
+ public static final String SINK_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_KEY =
+ "sink.exception.conflict.record-ignored-data";
+ public static final boolean CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_KEY =
+ "connector.exception.others.retry-max-time-seconds";
+ public static final String SINK_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_KEY =
+ "sink.exception.others.retry-max-time-seconds";
+ public static final long CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE = -1;
+
+ public static final String CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_KEY =
+ "connector.exception.others.record-ignored-data";
+ public static final String SINK_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_KEY =
+ "sink.exception.others.record-ignored-data";
+ public static final boolean CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_AIR_GAP_E_LANGUAGE_ENABLE_KEY =
+ "connector.air-gap.e-language.enable";
+ public static final String SINK_AIR_GAP_E_LANGUAGE_ENABLE_KEY = "sink.air-gap.e-language.enable";
+ public static final boolean CONNECTOR_AIR_GAP_E_LANGUAGE_ENABLE_DEFAULT_VALUE = false;
+
+ public static final String CONNECTOR_AIR_GAP_HANDSHAKE_TIMEOUT_MS_KEY =
+ "connector.air-gap.handshake-timeout-ms";
+ public static final String SINK_AIR_GAP_HANDSHAKE_TIMEOUT_MS_KEY =
+ "sink.air-gap.handshake-timeout-ms";
+ public static final int CONNECTOR_AIR_GAP_HANDSHAKE_TIMEOUT_MS_DEFAULT_VALUE = 5000;
+
+ public static final String CONNECTOR_IOTDB_SYNC_CONNECTOR_VERSION_KEY = "connector.version";
+ public static final String SINK_IOTDB_SYNC_CONNECTOR_VERSION_KEY = "sink.version";
+ public static final String CONNECTOR_IOTDB_SYNC_CONNECTOR_VERSION_DEFAULT_VALUE = "1.1";
+
+ public static final String CONNECTOR_WEBSOCKET_PORT_KEY = "connector.websocket.port";
+ public static final String SINK_WEBSOCKET_PORT_KEY = "sink.websocket.port";
+ public static final int CONNECTOR_WEBSOCKET_PORT_DEFAULT_VALUE = 8080;
+
+ public static final String CONNECTOR_OPC_UA_MODEL_KEY = "connector.opcua.model";
+ public static final String SINK_OPC_UA_MODEL_KEY = "sink.opcua.model";
+ public static final String CONNECTOR_OPC_UA_MODEL_CLIENT_SERVER_VALUE = "client-server";
+ public static final String CONNECTOR_OPC_UA_MODEL_PUB_SUB_VALUE = "pub-sub";
+ public static final String CONNECTOR_OPC_UA_MODEL_DEFAULT_VALUE =
+ CONNECTOR_OPC_UA_MODEL_CLIENT_SERVER_VALUE;
+
+ public static final String CONNECTOR_OPC_UA_TCP_BIND_PORT_KEY = "connector.opcua.tcp.port";
+ public static final String SINK_OPC_UA_TCP_BIND_PORT_KEY = "sink.opcua.tcp.port";
+ public static final int CONNECTOR_OPC_UA_TCP_BIND_PORT_DEFAULT_VALUE = 12686;
+
+ public static final String CONNECTOR_OPC_UA_HTTPS_BIND_PORT_KEY = "connector.opcua.https.port";
+ public static final String SINK_OPC_UA_HTTPS_BIND_PORT_KEY = "sink.opcua.https.port";
+ public static final int CONNECTOR_OPC_UA_HTTPS_BIND_PORT_DEFAULT_VALUE = 8443;
+
+ public static final String CONNECTOR_OPC_UA_SECURITY_DIR_KEY = "connector.opcua.security.dir";
+ public static final String SINK_OPC_UA_SECURITY_DIR_KEY = "sink.opcua.security.dir";
+ // public static final String CONNECTOR_OPC_UA_SECURITY_DIR_DEFAULT_VALUE =
+ // CommonDescriptor.getInstance().getConfDir() != null
+ // ? CommonDescriptor.getInstance().getConfDir() + File.separatorChar + "opc_security"
+ // : System.getProperty("user.home") + File.separatorChar + "iotdb_opc_security";
+
+ public static final String CONNECTOR_OPC_UA_ENABLE_ANONYMOUS_ACCESS_KEY =
+ "connector.opcua.enable-anonymous-access";
+ public static final String SINK_OPC_UA_ENABLE_ANONYMOUS_ACCESS_KEY =
+ "sink.opcua.enable-anonymous-access";
+ public static final boolean CONNECTOR_OPC_UA_ENABLE_ANONYMOUS_ACCESS_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_OPC_UA_PLACEHOLDER_KEY = "connector.opcua.placeholder";
+ public static final String SINK_OPC_UA_PLACEHOLDER_KEY = "sink.opcua.placeholder";
+ public static final String CONNECTOR_OPC_UA_PLACEHOLDER_DEFAULT_VALUE = "null";
+
+ public static final String CONNECTOR_LEADER_CACHE_ENABLE_KEY = "connector.leader-cache.enable";
+ public static final String SINK_LEADER_CACHE_ENABLE_KEY = "sink.leader-cache.enable";
+ public static final boolean CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_LOAD_BALANCE_STRATEGY_KEY =
+ "connector.load-balance-strategy";
+ public static final String SINK_LOAD_BALANCE_STRATEGY_KEY = "sink.load-balance-strategy";
+ public static final String CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY = "round-robin";
+ public static final String CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY = "random";
+ public static final String CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY = "priority";
+ public static final Set<String> CONNECTOR_LOAD_BALANCE_STRATEGY_SET =
+ Collections.unmodifiableSet(
+ new HashSet<>(
+ Arrays.asList(
+ CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY,
+ CONNECTOR_LOAD_BALANCE_RANDOM_STRATEGY,
+ CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY)));
+
+ public static final String CONNECTOR_COMPRESSOR_KEY = "connector.compressor";
+ public static final String SINK_COMPRESSOR_KEY = "sink.compressor";
+ public static final String CONNECTOR_COMPRESSOR_DEFAULT_VALUE = "";
+ public static final String CONNECTOR_COMPRESSOR_SNAPPY = "snappy";
+ public static final String CONNECTOR_COMPRESSOR_GZIP = "gzip";
+ public static final String CONNECTOR_COMPRESSOR_LZ4 = "lz4";
+ public static final String CONNECTOR_COMPRESSOR_ZSTD = "zstd";
+ public static final String CONNECTOR_COMPRESSOR_LZMA2 = "lzma2";
+ public static final Set<String> CONNECTOR_COMPRESSOR_SET =
+ Collections.unmodifiableSet(
+ new HashSet<>(
+ Arrays.asList(
+ CONNECTOR_COMPRESSOR_SNAPPY,
+ CONNECTOR_COMPRESSOR_GZIP,
+ CONNECTOR_COMPRESSOR_LZ4,
+ CONNECTOR_COMPRESSOR_ZSTD,
+ CONNECTOR_COMPRESSOR_LZMA2)));
+
+ public static final String CONNECTOR_COMPRESSOR_ZSTD_LEVEL_KEY =
+ "connector.compressor.zstd.level";
+ public static final String SINK_COMPRESSOR_ZSTD_LEVEL_KEY = "sink.compressor.zstd.level";
+ public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE =
+ Zstd.defaultCompressionLevel();
+ public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE = Zstd.minCompressionLevel();
+ public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE = Zstd.maxCompressionLevel();
+
+ public static final String CONNECTOR_RATE_LIMIT_KEY = "connector.rate-limit-bytes-per-second";
+ public static final String SINK_RATE_LIMIT_KEY = "sink.rate-limit-bytes-per-second";
+ public static final double CONNECTOR_RATE_LIMIT_DEFAULT_VALUE = -1;
+
+ public static final String CONNECTOR_FORMAT_KEY = "connector.format";
+ public static final String SINK_FORMAT_KEY = "sink.format";
+ public static final String CONNECTOR_FORMAT_TABLET_VALUE = "tablet";
+ public static final String CONNECTOR_FORMAT_TS_FILE_VALUE = "tsfile";
+ public static final String CONNECTOR_FORMAT_HYBRID_VALUE = "hybrid";
+
+ public static final String SINK_TOPIC_KEY = "sink.topic";
+ public static final String SINK_CONSUMER_GROUP_KEY = "sink.consumer-group";
+
+ public static final String CONNECTOR_CONSENSUS_GROUP_ID_KEY = "connector.consensus.group-id";
+ public static final String CONNECTOR_CONSENSUS_PIPE_NAME = "connector.consensus.pipe-name";
+
+ public static final String CONNECTOR_LOAD_TSFILE_STRATEGY_KEY = "connector.load-tsfile-strategy";
+ public static final String SINK_LOAD_TSFILE_STRATEGY_KEY = "sink.load-tsfile-strategy";
+ public static final String CONNECTOR_LOAD_TSFILE_STRATEGY_ASYNC_VALUE = "async";
+ public static final String CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE = "sync";
+ public static final Set<String> CONNECTOR_LOAD_TSFILE_STRATEGY_SET =
+ Collections.unmodifiableSet(
+ new HashSet<>(
+ Arrays.asList(
+ CONNECTOR_LOAD_TSFILE_STRATEGY_ASYNC_VALUE,
+ CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE)));
+
+ public static final String CONNECTOR_LOAD_TSFILE_VALIDATION_KEY =
+ "connector.load-tsfile-validation";
+ public static final String SINK_LOAD_TSFILE_VALIDATION_KEY = "sink.load-tsfile-validation";
+ public static final boolean CONNECTOR_LOAD_TSFILE_VALIDATION_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_MARK_AS_PIPE_REQUEST_KEY = "connector.mark-as-pipe-request";
+ public static final String SINK_MARK_AS_PIPE_REQUEST_KEY = "sink.mark-as-pipe-request";
+ public static final boolean CONNECTOR_MARK_AS_PIPE_REQUEST_DEFAULT_VALUE = true;
+
+ public static final String CONNECTOR_SKIP_IF_KEY = "connector.skipif";
+ public static final String SINK_SKIP_IF_KEY = "sink.skipif";
+ public static final String CONNECTOR_IOTDB_SKIP_IF_NO_PRIVILEGES = "no-privileges";
+
+ public static final String CONNECTOR_OPC_DA_CLSID_KEY = "connector.opcda.clsid";
+ public static final String SINK_OPC_DA_CLSID_KEY = "sink.opcda.clsid";
+
+ public static final String CONNECTOR_OPC_DA_PROGID_KEY = "connector.opcda.progid";
+ public static final String SINK_OPC_DA_PROGID_KEY = "sink.opcda.progid";
+
+ private PipeConnectorConstant() {
+ throw new IllegalStateException("Utility class");
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PushSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeTransferHandshakeConstant.java
similarity index 50%
copy from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PushSource.java
copy to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeTransferHandshakeConstant.java
index e8eebf4..1adcfab 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PushSource.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/constant/PipeTransferHandshakeConstant.java
@@ -17,30 +17,20 @@
* under the License.
*/
-package org.apache.iotdb.collector.plugin.api;
+package org.apache.iotdb.collector.plugin.builtin.sink.constant;
-import org.apache.iotdb.pipe.api.PipeSource;
-import org.apache.iotdb.pipe.api.collector.EventCollector;
-import org.apache.iotdb.pipe.api.event.Event;
+public class PipeTransferHandshakeConstant {
-public abstract class PushSource implements PipeSource {
+ public static final String HANDSHAKE_KEY_TIME_PRECISION = "timestampPrecision";
+ public static final String HANDSHAKE_KEY_CLUSTER_ID = "clusterID";
+ public static final String HANDSHAKE_KEY_CONVERT_ON_TYPE_MISMATCH = "convertOnTypeMismatch";
+ public static final String HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY = "loadTsFileStrategy";
+ public static final String HANDSHAKE_KEY_USERNAME = "username";
+ public static final String HANDSHAKE_KEY_PASSWORD = "password";
+ public static final String HANDSHAKE_KEY_VALIDATE_TSFILE = "validateTsFile";
+ public static final String HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST = "markAsPipeRequest";
- protected EventCollector collector;
-
- public PushSource() {
- this.collector = null;
- }
-
- public final void setCollector(final EventCollector collector) {
- this.collector = collector;
- }
-
- @Override
- public final Event supply() {
- throw new UnsupportedOperationException();
- }
-
- public final void supply(final Event event) throws Exception {
- collector.collect(event);
+ private PipeTransferHandshakeConstant() {
+ // Utility class
}
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeInsertionEvent.java
similarity index 63%
copy from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
copy to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeInsertionEvent.java
index e15a493..1027ece 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeInsertionEvent.java
@@ -17,8 +17,24 @@
* under the License.
*/
-package org.apache.iotdb.collector.plugin.api;
+package org.apache.iotdb.collector.plugin.builtin.sink.event;
-import org.apache.iotdb.pipe.api.PipeSource;
+public abstract class PipeInsertionEvent {
-public abstract class PullSource implements PipeSource {}
+ protected Boolean isTableModelEvent;
+
+ protected String treeModelDatabaseName;
+ protected String tableModelDatabaseName;
+
+ public boolean isTableModelEvent() {
+ return isTableModelEvent;
+ }
+
+ public String getTreeModelDatabaseName() {
+ return treeModelDatabaseName;
+ }
+
+ public String getTableModelDatabaseName() {
+ return tableModelDatabaseName;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeRawTabletInsertionEvent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeRawTabletInsertionEvent.java
new file mode 100644
index 0000000..bee813a
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeRawTabletInsertionEvent.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.event;
+
+import org.apache.iotdb.pipe.api.access.Row;
+import org.apache.iotdb.pipe.api.collector.RowCollector;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+
+import org.apache.tsfile.write.record.Tablet;
+
+import java.util.function.BiConsumer;
+
+public class PipeRawTabletInsertionEvent extends PipeInsertionEvent
+ implements TabletInsertionEvent, AutoCloseable {
+
+ public PipeRawTabletInsertionEvent(Tablet tablet, String deviceId) {
+ this.deviceId = deviceId;
+ this.tablet = tablet;
+ this.tablet.setDeviceId(deviceId);
+ }
+
+ public String deviceId;
+ protected Tablet tablet;
+ private boolean isAligned;
+
+ @Override
+ public Iterable<TabletInsertionEvent> processRowByRow(
+ final BiConsumer<Row, RowCollector> consumer) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterable<TabletInsertionEvent> processTablet(
+ final BiConsumer<Tablet, RowCollector> consumer) {
+ throw new UnsupportedOperationException();
+ }
+
+ public Tablet getTablet() {
+ return tablet;
+ }
+
+ public boolean isAligned() {
+ return isAligned;
+ }
+
+ public Tablet convertToTablet() {
+ return tablet;
+ }
+
+ public boolean isTableModelEvent() {
+ return false;
+ }
+
+ public String getDeviceId() {
+ return deviceId;
+ }
+
+ public String getTableModelDatabaseName() {
+ return null;
+ }
+
+ @Override
+ public void close() {}
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeTsFileInsertionEvent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeTsFileInsertionEvent.java
new file mode 100644
index 0000000..2fc5e9a
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/event/PipeTsFileInsertionEvent.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.event;
+
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+public class PipeTsFileInsertionEvent extends PipeInsertionEvent implements TsFileInsertionEvent {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileInsertionEvent.class);
+
+ private File tsFile;
+
+ @Override
+ public boolean isTableModelEvent() {
+ throw new PipeException("");
+ }
+
+ public File getTsFile() {
+ return tsFile;
+ }
+
+ @Override
+ public Iterable<TabletInsertionEvent> toTabletInsertionEvents() throws PipeException {
+ return toTabletInsertionEvents(Long.MAX_VALUE);
+ }
+
+ public Iterable<TabletInsertionEvent> toTabletInsertionEvents(final long timeoutMs)
+ throws PipeException {
+ return null;
+ }
+
+ @Override
+ public void close() throws Exception {}
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeConnectorCriticalException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeConnectorCriticalException.java
new file mode 100644
index 0000000..041c7a3
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeConnectorCriticalException.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeRuntimeConnectorCriticalException extends PipeRuntimeCriticalException {
+
+ public PipeRuntimeConnectorCriticalException(final String message) {
+ super(message);
+ }
+
+ public PipeRuntimeConnectorCriticalException(final String message, final long timeStamp) {
+ super(message, timeStamp);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeRuntimeConnectorCriticalException
+ && Objects.equals(getMessage(), ((PipeRuntimeConnectorCriticalException) obj).getMessage())
+ && Objects.equals(getTimeStamp(), ((PipeRuntimeException) obj).getTimeStamp());
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public void serialize(final ByteBuffer byteBuffer) {
+ PipeRuntimeExceptionType.CONNECTOR_CRITICAL_EXCEPTION.serialize(byteBuffer);
+ ReadWriteIOUtils.write(getMessage(), byteBuffer);
+ ReadWriteIOUtils.write(getTimeStamp(), byteBuffer);
+ }
+
+ @Override
+ public void serialize(final OutputStream stream) throws IOException {
+ PipeRuntimeExceptionType.CONNECTOR_CRITICAL_EXCEPTION.serialize(stream);
+ ReadWriteIOUtils.write(getMessage(), stream);
+ ReadWriteIOUtils.write(getTimeStamp(), stream);
+ }
+
+ public static PipeRuntimeConnectorCriticalException deserializeFrom(
+ final PipeRuntimeMetaVersion version, final ByteBuffer byteBuffer) {
+ final String message = ReadWriteIOUtils.readString(byteBuffer);
+ switch (version) {
+ case VERSION_1:
+ return new PipeRuntimeConnectorCriticalException(message);
+ case VERSION_2:
+ return new PipeRuntimeConnectorCriticalException(
+ message, ReadWriteIOUtils.readLong(byteBuffer));
+ default:
+ throw new UnsupportedOperationException(String.format("Unsupported version %s", version));
+ }
+ }
+
+ public static PipeRuntimeConnectorCriticalException deserializeFrom(
+ final PipeRuntimeMetaVersion version, final InputStream stream) throws IOException {
+ final String message = ReadWriteIOUtils.readString(stream);
+ switch (version) {
+ case VERSION_1:
+ return new PipeRuntimeConnectorCriticalException(message);
+ case VERSION_2:
+ return new PipeRuntimeConnectorCriticalException(
+ message, ReadWriteIOUtils.readLong(stream));
+ default:
+ throw new UnsupportedOperationException(String.format("Unsupported version %s", version));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "PipeRuntimeConnectorCriticalException{"
+ + "message='"
+ + getMessage()
+ + "', timeStamp="
+ + getTimeStamp()
+ + "}";
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeCriticalException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeCriticalException.java
new file mode 100644
index 0000000..4ade0b2
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeCriticalException.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeRuntimeCriticalException extends PipeRuntimeException {
+
+ public PipeRuntimeCriticalException(final String message) {
+ super(message);
+ }
+
+ public PipeRuntimeCriticalException(final String message, final long timeStamp) {
+ super(message, timeStamp);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeRuntimeCriticalException
+ && Objects.equals(getMessage(), ((PipeRuntimeCriticalException) obj).getMessage())
+ && Objects.equals(getTimeStamp(), ((PipeRuntimeException) obj).getTimeStamp());
+ }
+
+ @Override
+ public void serialize(final ByteBuffer byteBuffer) {
+ PipeRuntimeExceptionType.CRITICAL_EXCEPTION.serialize(byteBuffer);
+ ReadWriteIOUtils.write(getMessage(), byteBuffer);
+ ReadWriteIOUtils.write(getTimeStamp(), byteBuffer);
+ }
+
+ @Override
+ public void serialize(final OutputStream stream) throws IOException {
+ PipeRuntimeExceptionType.CRITICAL_EXCEPTION.serialize(stream);
+ ReadWriteIOUtils.write(getMessage(), stream);
+ ReadWriteIOUtils.write(getTimeStamp(), stream);
+ }
+
+ public static PipeRuntimeCriticalException deserializeFrom(
+ final PipeRuntimeMetaVersion version, final ByteBuffer byteBuffer) {
+ final String message = ReadWriteIOUtils.readString(byteBuffer);
+ switch (version) {
+ case VERSION_1:
+ return new PipeRuntimeCriticalException(message);
+ case VERSION_2:
+ return new PipeRuntimeCriticalException(message, ReadWriteIOUtils.readLong(byteBuffer));
+ default:
+ throw new UnsupportedOperationException(String.format("Unsupported version %s", version));
+ }
+ }
+
+ public static PipeRuntimeCriticalException deserializeFrom(
+ final PipeRuntimeMetaVersion version, final InputStream stream) throws IOException {
+ final String message = ReadWriteIOUtils.readString(stream);
+ switch (version) {
+ case VERSION_1:
+ return new PipeRuntimeCriticalException(message);
+ case VERSION_2:
+ return new PipeRuntimeCriticalException(message, ReadWriteIOUtils.readLong(stream));
+ default:
+ throw new UnsupportedOperationException(String.format("Unsupported version %s", version));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "PipeRuntimeCriticalException{"
+ + "message='"
+ + getMessage()
+ + "', timeStamp="
+ + getTimeStamp()
+ + "}";
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeException.java
new file mode 100644
index 0000000..65652b4
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeException.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception;
+
+import org.apache.iotdb.pipe.api.exception.PipeException;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public abstract class PipeRuntimeException extends PipeException {
+
+ protected PipeRuntimeException(final String message) {
+ super(message);
+ }
+
+ protected PipeRuntimeException(final String message, final long timeStamp) {
+ super(message, timeStamp);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeRuntimeException
+ && Objects.equals(getMessage(), ((PipeRuntimeException) obj).getMessage())
+ && Objects.equals(getTimeStamp(), ((PipeRuntimeException) obj).getTimeStamp());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getMessage(), getTimeStamp());
+ }
+
+ public abstract void serialize(final ByteBuffer byteBuffer);
+
+ public abstract void serialize(final OutputStream stream) throws IOException;
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeExceptionType.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeExceptionType.java
new file mode 100644
index 0000000..bd80d74
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeExceptionType.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+public enum PipeRuntimeExceptionType {
+ NON_CRITICAL_EXCEPTION((short) 1),
+ CRITICAL_EXCEPTION((short) 2),
+ CONNECTOR_CRITICAL_EXCEPTION((short) 3),
+ OUT_OF_MEMORY_CRITICAL_EXCEPTION((short) 4),
+ ;
+
+ private final short type;
+
+ PipeRuntimeExceptionType(short type) {
+ this.type = type;
+ }
+
+ public short getType() {
+ return type;
+ }
+
+ public void serialize(ByteBuffer byteBuffer) {
+ ReadWriteIOUtils.write(type, byteBuffer);
+ }
+
+ public void serialize(OutputStream stream) throws IOException {
+ ReadWriteIOUtils.write(type, stream);
+ }
+
+ public static PipeRuntimeException deserializeFrom(
+ PipeRuntimeMetaVersion version, ByteBuffer byteBuffer) {
+ final short type = ReadWriteIOUtils.readShort(byteBuffer);
+ switch (type) {
+ case 1:
+ return PipeRuntimeNonCriticalException.deserializeFrom(version, byteBuffer);
+ case 2:
+ return PipeRuntimeCriticalException.deserializeFrom(version, byteBuffer);
+ case 3:
+ return PipeRuntimeConnectorCriticalException.deserializeFrom(version, byteBuffer);
+ case 4:
+ return PipeRuntimeOutOfMemoryCriticalException.deserializeFrom(version, byteBuffer);
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Unsupported PipeRuntimeException type %s.", type));
+ }
+ }
+
+ public static PipeRuntimeException deserializeFrom(
+ PipeRuntimeMetaVersion version, InputStream stream) throws IOException {
+ final short type = ReadWriteIOUtils.readShort(stream);
+ switch (type) {
+ case 1:
+ return PipeRuntimeNonCriticalException.deserializeFrom(version, stream);
+ case 2:
+ return PipeRuntimeCriticalException.deserializeFrom(version, stream);
+ case 3:
+ return PipeRuntimeConnectorCriticalException.deserializeFrom(version, stream);
+ case 4:
+ return PipeRuntimeOutOfMemoryCriticalException.deserializeFrom(version, stream);
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Unsupported PipeRuntimeException type %s.", type));
+ }
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeMetaVersion.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeMetaVersion.java
new file mode 100644
index 0000000..9986df9
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeMetaVersion.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+public enum PipeRuntimeMetaVersion {
+
+ // for compatibility use
+ VERSION_1(PipeStatus.RUNNING.getType()),
+
+ VERSION_2(Byte.MAX_VALUE),
+ ;
+
+ private static final Map<Byte, PipeRuntimeMetaVersion> VERSION_MAP = new HashMap<>();
+
+ static {
+ // for compatibility use
+ for (final PipeStatus status : PipeStatus.values()) {
+ VERSION_MAP.put(status.getType(), VERSION_1);
+ }
+
+ for (final PipeRuntimeMetaVersion version : PipeRuntimeMetaVersion.values()) {
+ VERSION_MAP.put(version.getVersion(), version);
+ }
+ }
+
+ private final byte version;
+
+ PipeRuntimeMetaVersion(byte version) {
+ this.version = version;
+ }
+
+ public byte getVersion() {
+ return version;
+ }
+
+ public void serialize(OutputStream outputStream) throws IOException {
+ ReadWriteIOUtils.write(version, outputStream);
+ }
+
+ public static PipeRuntimeMetaVersion deserialize(InputStream inputStream) throws IOException {
+ return deserialize(ReadWriteIOUtils.readByte(inputStream));
+ }
+
+ public static PipeRuntimeMetaVersion deserialize(ByteBuffer byteBuffer) {
+ return deserialize(ReadWriteIOUtils.readByte(byteBuffer));
+ }
+
+ public static PipeRuntimeMetaVersion deserialize(byte version) {
+ return VERSION_MAP.get(version);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeNonCriticalException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeNonCriticalException.java
new file mode 100644
index 0000000..ef41889
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeNonCriticalException.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeRuntimeNonCriticalException extends PipeRuntimeException {
+
+ public PipeRuntimeNonCriticalException(String message) {
+ super(message);
+ }
+
+ public PipeRuntimeNonCriticalException(String message, long timeStamp) {
+ super(message, timeStamp);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeRuntimeNonCriticalException
+ && Objects.equals(getMessage(), ((PipeRuntimeNonCriticalException) obj).getMessage())
+ && Objects.equals(getTimeStamp(), ((PipeRuntimeException) obj).getTimeStamp());
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public void serialize(ByteBuffer byteBuffer) {
+ PipeRuntimeExceptionType.NON_CRITICAL_EXCEPTION.serialize(byteBuffer);
+ ReadWriteIOUtils.write(getMessage(), byteBuffer);
+ ReadWriteIOUtils.write(getTimeStamp(), byteBuffer);
+ }
+
+ @Override
+ public void serialize(OutputStream stream) throws IOException {
+ PipeRuntimeExceptionType.NON_CRITICAL_EXCEPTION.serialize(stream);
+ ReadWriteIOUtils.write(getMessage(), stream);
+ ReadWriteIOUtils.write(getTimeStamp(), stream);
+ }
+
+ public static PipeRuntimeNonCriticalException deserializeFrom(
+ PipeRuntimeMetaVersion version, ByteBuffer byteBuffer) {
+ final String message = ReadWriteIOUtils.readString(byteBuffer);
+ switch (version) {
+ case VERSION_1:
+ return new PipeRuntimeNonCriticalException(message);
+ case VERSION_2:
+ return new PipeRuntimeNonCriticalException(message, ReadWriteIOUtils.readLong(byteBuffer));
+ default:
+ throw new UnsupportedOperationException(String.format("Unsupported version %s", version));
+ }
+ }
+
+ public static PipeRuntimeNonCriticalException deserializeFrom(
+ PipeRuntimeMetaVersion version, InputStream stream) throws IOException {
+ final String message = ReadWriteIOUtils.readString(stream);
+ switch (version) {
+ case VERSION_1:
+ return new PipeRuntimeNonCriticalException(message);
+ case VERSION_2:
+ return new PipeRuntimeNonCriticalException(message, ReadWriteIOUtils.readLong(stream));
+ default:
+ throw new UnsupportedOperationException(String.format("Unsupported version %s", version));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "PipeRuntimeNonCriticalException{"
+ + "message='"
+ + getMessage()
+ + "', timeStamp="
+ + getTimeStamp()
+ + "}";
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeOutOfMemoryCriticalException.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeOutOfMemoryCriticalException.java
new file mode 100644
index 0000000..5106dcf
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeRuntimeOutOfMemoryCriticalException.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.exception;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeRuntimeOutOfMemoryCriticalException extends PipeRuntimeCriticalException {
+
+ public PipeRuntimeOutOfMemoryCriticalException(String message) {
+ super(message);
+ }
+
+ public PipeRuntimeOutOfMemoryCriticalException(String message, long timeStamp) {
+ super(message, timeStamp);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeRuntimeOutOfMemoryCriticalException
+ && Objects.equals(
+ getMessage(), ((PipeRuntimeOutOfMemoryCriticalException) obj).getMessage())
+ && Objects.equals(getTimeStamp(), ((PipeRuntimeException) obj).getTimeStamp());
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public void serialize(ByteBuffer byteBuffer) {
+ PipeRuntimeExceptionType.OUT_OF_MEMORY_CRITICAL_EXCEPTION.serialize(byteBuffer);
+ ReadWriteIOUtils.write(getMessage(), byteBuffer);
+ ReadWriteIOUtils.write(getTimeStamp(), byteBuffer);
+ }
+
+ @Override
+ public void serialize(OutputStream stream) throws IOException {
+ PipeRuntimeExceptionType.OUT_OF_MEMORY_CRITICAL_EXCEPTION.serialize(stream);
+ ReadWriteIOUtils.write(getMessage(), stream);
+ ReadWriteIOUtils.write(getTimeStamp(), stream);
+ }
+
+ public static PipeRuntimeOutOfMemoryCriticalException deserializeFrom(
+ PipeRuntimeMetaVersion version, ByteBuffer byteBuffer) {
+ final String message = ReadWriteIOUtils.readString(byteBuffer);
+ switch (version) {
+ case VERSION_1:
+ return new PipeRuntimeOutOfMemoryCriticalException(message);
+ case VERSION_2:
+ return new PipeRuntimeOutOfMemoryCriticalException(
+ message, ReadWriteIOUtils.readLong(byteBuffer));
+ default:
+ throw new UnsupportedOperationException(String.format("Unsupported version %s", version));
+ }
+ }
+
+ public static PipeRuntimeOutOfMemoryCriticalException deserializeFrom(
+ PipeRuntimeMetaVersion version, InputStream stream) throws IOException {
+ final String message = ReadWriteIOUtils.readString(stream);
+ switch (version) {
+ case VERSION_1:
+ return new PipeRuntimeOutOfMemoryCriticalException(message);
+ case VERSION_2:
+ return new PipeRuntimeOutOfMemoryCriticalException(
+ message, ReadWriteIOUtils.readLong(stream));
+ default:
+ throw new UnsupportedOperationException(String.format("Unsupported version %s", version));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "PipeRuntimeOutOfMemoryException{"
+ + "message='"
+ + getMessage()
+ + "', timeStamp="
+ + getTimeStamp()
+ + "}";
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/Persistence.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeStatus.java
similarity index 57%
copy from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/Persistence.java
copy to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeStatus.java
index b2abcb8..5ce28d1 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/persistence/Persistence.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/exception/PipeStatus.java
@@ -17,29 +17,34 @@
* under the License.
*/
-package org.apache.iotdb.collector.persistence;
+package org.apache.iotdb.collector.plugin.builtin.sink.exception;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
+public enum PipeStatus {
+ RUNNING((byte) 0),
+ STOPPED((byte) 1),
+ DROPPED((byte) 2),
+ ;
-public abstract class Persistence {
+ private final byte type;
- private final String databaseUrl;
-
- public Persistence(final String databaseUrl) {
- this.databaseUrl = databaseUrl;
- initDatabaseFileIfPossible();
- initTableIfPossible();
+ PipeStatus(byte type) {
+ this.type = type;
}
- protected abstract void initDatabaseFileIfPossible();
-
- protected abstract void initTableIfPossible();
-
- public abstract void tryResume();
+ public byte getType() {
+ return type;
+ }
- protected Connection getConnection() throws SQLException {
- return DriverManager.getConnection(databaseUrl);
+ public static PipeStatus getPipeStatus(byte type) {
+ switch (type) {
+ case 0:
+ return PipeStatus.RUNNING;
+ case 1:
+ return PipeStatus.STOPPED;
+ case 2:
+ return PipeStatus.DROPPED;
+ default:
+ throw new IllegalArgumentException("Invalid input: " + type);
+ }
}
}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventBatch.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventBatch.java
new file mode 100644
index 0000000..b26a065
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventBatch.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public abstract class PipeTabletEventBatch implements AutoCloseable {
+
+ protected final List<PipeRawTabletInsertionEvent> events = new ArrayList<>();
+
+ private final int maxDelayInMs;
+ private long firstEventProcessingTime = Long.MIN_VALUE;
+
+ protected long totalBufferSize = 0;
+
+ protected volatile boolean isClosed = false;
+
+ protected PipeTabletEventBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) {
+ this.maxDelayInMs = maxDelayInMs;
+ }
+
+ /**
+ * Try offer {@link Event} into batch if the given {@link Event} is not duplicated.
+ *
+ * @param event the given {@link Event}
+ * @return {@code true} if the batch can be transferred
+ */
+ public synchronized boolean onEvent(final TabletInsertionEvent event) throws IOException {
+ if (isClosed || !(event instanceof PipeRawTabletInsertionEvent)) {
+ return false;
+ }
+
+ // The deduplication logic here is to avoid the accumulation of
+ // the same event in a batch when retrying.
+ if (events.isEmpty() || !Objects.equals(events.get(events.size() - 1), event)) {
+ if (constructBatch(event)) {
+ events.add((PipeRawTabletInsertionEvent) event);
+ }
+
+ if (firstEventProcessingTime == Long.MIN_VALUE) {
+ firstEventProcessingTime = System.currentTimeMillis();
+ }
+ }
+
+ return shouldEmit();
+ }
+
+ /**
+ * Added an {@link TabletInsertionEvent} into batch.
+ *
+ * @param event the {@link TabletInsertionEvent} in batch
+ * @return {@code true} if the event is calculated into batch, {@code false} if the event is
+ * cached and not emitted in this batch. If there are failure encountered, just throw
+ * exceptions and do not return {@code false} here.
+ */
+ protected abstract boolean constructBatch(final TabletInsertionEvent event) throws IOException;
+
+ public boolean shouldEmit() {
+ return totalBufferSize >= getMaxBatchSizeInBytes()
+ || System.currentTimeMillis() - firstEventProcessingTime >= maxDelayInMs;
+ }
+
+ private long getMaxBatchSizeInBytes() {
+ // return allocatedMemoryBlock.getMemoryUsageInBytes();
+ return 16777216;
+ }
+
+ public synchronized void onSuccess() {
+ events.clear();
+
+ totalBufferSize = 0;
+
+ firstEventProcessingTime = Long.MIN_VALUE;
+ }
+
+ @Override
+ public synchronized void close() {
+ isClosed = true;
+
+ events.clear();
+ }
+
+ public List<PipeRawTabletInsertionEvent> deepCopyEvents() {
+ return new ArrayList<>(events);
+ }
+
+ public boolean isEmpty() {
+ return events.isEmpty();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java
new file mode 100644
index 0000000..e147788
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferTabletBatchReqV2;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+
+import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class PipeTabletEventPlainBatch extends PipeTabletEventBatch {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventPlainBatch.class);
+
+ private final List<ByteBuffer> binaryBuffers = new ArrayList<>();
+ private final List<ByteBuffer> insertNodeBuffers = new ArrayList<>();
+ private final List<ByteBuffer> tabletBuffers = new ArrayList<>();
+
+ private static final String TREE_MODEL_DATABASE_PLACEHOLDER = null;
+ private final List<String> binaryDataBases = new ArrayList<>();
+ private final List<String> insertNodeDataBases = new ArrayList<>();
+ private final List<String> tabletDataBases = new ArrayList<>();
+
+ // Used to rate limit when transferring data
+ private final Map<Pair<String, Long>, Long> pipe2BytesAccumulated = new HashMap<>();
+
+ PipeTabletEventPlainBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) {
+ super(maxDelayInMs, requestMaxBatchSizeInBytes);
+ }
+
+ @Override
+ protected boolean constructBatch(final TabletInsertionEvent event) throws IOException {
+ final int bufferSize = buildTabletInsertionBuffer(event);
+ totalBufferSize += bufferSize;
+ pipe2BytesAccumulated.compute(
+ new Pair<>("", 0L),
+ (pipeName, bytesAccumulated) ->
+ bytesAccumulated == null ? bufferSize : bytesAccumulated + bufferSize);
+ return true;
+ }
+
+ @Override
+ public synchronized void onSuccess() {
+ super.onSuccess();
+
+ binaryBuffers.clear();
+ insertNodeBuffers.clear();
+ tabletBuffers.clear();
+
+ binaryDataBases.clear();
+ insertNodeDataBases.clear();
+ tabletDataBases.clear();
+
+ pipe2BytesAccumulated.clear();
+ }
+
+ public PipeTransferTabletBatchReqV2 toTPipeTransferReq() throws IOException {
+ return PipeTransferTabletBatchReqV2.toTPipeTransferReq(
+ binaryBuffers,
+ insertNodeBuffers,
+ tabletBuffers,
+ binaryDataBases,
+ insertNodeDataBases,
+ tabletDataBases);
+ }
+
+ private int buildTabletInsertionBuffer(final TabletInsertionEvent event) throws IOException {
+ int databaseEstimateSize;
+ final ByteBuffer buffer;
+
+ final PipeRawTabletInsertionEvent pipeRawTabletInsertionEvent =
+ (PipeRawTabletInsertionEvent) event;
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ pipeRawTabletInsertionEvent.convertToTablet().serialize(outputStream);
+ ReadWriteIOUtils.write(pipeRawTabletInsertionEvent.isAligned(), outputStream);
+ buffer = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+ tabletBuffers.add(buffer);
+ if (pipeRawTabletInsertionEvent.isTableModelEvent()) {
+ databaseEstimateSize = pipeRawTabletInsertionEvent.getTableModelDatabaseName().length();
+ tabletDataBases.add(pipeRawTabletInsertionEvent.getTableModelDatabaseName());
+ } else {
+ databaseEstimateSize = 4;
+ tabletDataBases.add(TREE_MODEL_DATABASE_PLACEHOLDER);
+ }
+
+ return buffer.limit() + databaseEstimateSize;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
new file mode 100644
index 0000000..93e9d37
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.collector.utils.builder.PipeTableModeTsFileBuilder;
+import org.apache.iotdb.collector.utils.builder.PipeTreeModelTsFileBuilder;
+import org.apache.iotdb.collector.utils.builder.PipeTsFileBuilder;
+import org.apache.iotdb.collector.utils.sorter.PipeTableModelTabletEventSorter;
+import org.apache.iotdb.collector.utils.sorter.PipeTreeModelTabletEventSorter;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.write.record.Tablet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class PipeTabletEventTsFileBatch extends PipeTabletEventBatch {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventTsFileBatch.class);
+
+ private static final AtomicLong BATCH_ID_GENERATOR = new AtomicLong(0);
+ private final AtomicLong currentBatchId = new AtomicLong(BATCH_ID_GENERATOR.incrementAndGet());
+
+ private final PipeTsFileBuilder treeModeTsFileBuilder;
+ private final PipeTsFileBuilder tableModeTsFileBuilder;
+
+ private final Map<Pair<String, Long>, Double> pipeName2WeightMap = new HashMap<>();
+
+ public PipeTabletEventTsFileBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) {
+ super(maxDelayInMs, requestMaxBatchSizeInBytes);
+
+ final AtomicLong tsFileIdGenerator = new AtomicLong(0);
+ treeModeTsFileBuilder = new PipeTreeModelTsFileBuilder(currentBatchId, tsFileIdGenerator);
+ tableModeTsFileBuilder = new PipeTableModeTsFileBuilder(currentBatchId, tsFileIdGenerator);
+ }
+
+ @Override
+ protected boolean constructBatch(final TabletInsertionEvent event) {
+ if (event instanceof PipeRawTabletInsertionEvent) {
+ final PipeRawTabletInsertionEvent rawTabletInsertionEvent =
+ (PipeRawTabletInsertionEvent) event;
+ final Tablet tablet = rawTabletInsertionEvent.convertToTablet();
+ if (tablet.getRowSize() == 0) {
+ return true;
+ }
+ if (rawTabletInsertionEvent.isTableModelEvent()) {
+ // table Model
+ bufferTableModelTablet("", 0L, tablet, rawTabletInsertionEvent.getTableModelDatabaseName());
+ } else {
+ // tree Model
+ bufferTreeModelTablet("", 0L, tablet, rawTabletInsertionEvent.isAligned());
+ }
+ } else {
+ LOGGER.warn(
+ "Batch id = {}: Unsupported event {} type {} when constructing tsfile batch",
+ currentBatchId.get(),
+ event,
+ event.getClass());
+ }
+ return true;
+ }
+
+ private void bufferTreeModelTablet(
+ final String pipeName,
+ final long creationTime,
+ final Tablet tablet,
+ final boolean isAligned) {
+ new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary();
+
+ pipeName2WeightMap.compute(
+ new Pair<>(pipeName, creationTime),
+ (pipe, weight) -> Objects.nonNull(weight) ? ++weight : 1);
+
+ treeModeTsFileBuilder.bufferTreeModelTablet(tablet, isAligned);
+ }
+
+ private void bufferTableModelTablet(
+ final String pipeName, final long creationTime, final Tablet tablet, final String dataBase) {
+ new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByDevIdTimestamp();
+
+ // totalBufferSize += PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet);
+
+ pipeName2WeightMap.compute(
+ new Pair<>(pipeName, creationTime),
+ (pipe, weight) -> Objects.nonNull(weight) ? ++weight : 1);
+
+ tableModeTsFileBuilder.bufferTableModelTablet(dataBase, tablet);
+ }
+
+ public Map<Pair<String, Long>, Double> deepCopyPipe2WeightMap() {
+ final double sum = pipeName2WeightMap.values().stream().reduce(Double::sum).orElse(0.0);
+ if (sum == 0.0) {
+ return Collections.emptyMap();
+ }
+ pipeName2WeightMap.entrySet().forEach(entry -> entry.setValue(entry.getValue() / sum));
+ return new HashMap<>(pipeName2WeightMap);
+ }
+
+ /**
+ * Converts a Tablet to a TSFile and returns the generated TSFile along with its corresponding
+ * database name.
+ *
+ * @return a list of pairs containing the database name and the generated TSFile
+ * @throws IOException if an I/O error occurs during the conversion process
+ * @throws WriteProcessException if an error occurs during the write process
+ */
+ public synchronized List<Pair<String, File>> sealTsFiles()
+ throws IOException, WriteProcessException {
+ if (isClosed) {
+ return Collections.emptyList();
+ }
+
+ final List<Pair<String, File>> list = new ArrayList<>();
+ if (!treeModeTsFileBuilder.isEmpty()) {
+ list.addAll(treeModeTsFileBuilder.convertTabletToTsFileWithDBInfo());
+ }
+ if (!tableModeTsFileBuilder.isEmpty()) {
+ list.addAll(tableModeTsFileBuilder.convertTabletToTsFileWithDBInfo());
+ }
+ return list;
+ }
+
+ @Override
+ public synchronized void onSuccess() {
+ super.onSuccess();
+
+ pipeName2WeightMap.clear();
+ tableModeTsFileBuilder.onSuccess();
+ treeModeTsFileBuilder.onSuccess();
+ }
+
+ @Override
+ public synchronized void close() {
+ super.close();
+
+ pipeName2WeightMap.clear();
+
+ tableModeTsFileBuilder.close();
+ treeModeTsFileBuilder.close();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java
new file mode 100644
index 0000000..5cd29e4
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch;
+
+import org.apache.iotdb.collector.config.PipeRuntimeOptions;
+import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+
+import org.apache.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_DELAY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_SIZE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_DELAY_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_TS_FILE_BATCH_DELAY_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_FORMAT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_DELAY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_SIZE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LEADER_CACHE_ENABLE_KEY;
+
+public class PipeTransferBatchReqBuilder implements AutoCloseable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTransferBatchReqBuilder.class);
+
+ private final boolean useLeaderCache;
+
+ private final int requestMaxDelayInMs;
+ private final long requestMaxBatchSizeInBytes;
+
+ // If the leader cache is disabled (or unable to find the endpoint of event in the leader cache),
+ // the event will be stored in the default batch.
+ private final PipeTabletEventBatch defaultBatch;
+ // If the leader cache is enabled, the batch will be divided by the leader endpoint,
+ // each endpoint has a batch.
+ // This is only used in plain batch since tsfile does not return redirection info.
+ private final Map<TEndPoint, PipeTabletEventPlainBatch> endPointToBatch = new HashMap<>();
+
+ public PipeTransferBatchReqBuilder(final PipeParameters parameters) {
+ final boolean usingTsFileBatch =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(CONNECTOR_FORMAT_KEY, SINK_FORMAT_KEY), CONNECTOR_FORMAT_HYBRID_VALUE)
+ .equals(CONNECTOR_FORMAT_TS_FILE_VALUE);
+
+ useLeaderCache =
+ !usingTsFileBatch
+ && parameters.getBooleanOrDefault(
+ Arrays.asList(SINK_LEADER_CACHE_ENABLE_KEY, CONNECTOR_LEADER_CACHE_ENABLE_KEY),
+ CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE);
+
+ final int requestMaxDelayInSeconds;
+ if (usingTsFileBatch) {
+ requestMaxDelayInSeconds =
+ parameters.getIntOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_DELAY_KEY, SINK_IOTDB_BATCH_DELAY_KEY),
+ CONNECTOR_IOTDB_TS_FILE_BATCH_DELAY_DEFAULT_VALUE);
+ requestMaxDelayInMs =
+ requestMaxDelayInSeconds < 0 ? Integer.MAX_VALUE : requestMaxDelayInSeconds * 1000;
+ requestMaxBatchSizeInBytes =
+ parameters.getLongOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY),
+ CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE);
+ this.defaultBatch =
+ new PipeTabletEventTsFileBatch(requestMaxDelayInMs, requestMaxBatchSizeInBytes);
+ } else {
+ requestMaxDelayInSeconds =
+ parameters.getIntOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_DELAY_KEY, SINK_IOTDB_BATCH_DELAY_KEY),
+ CONNECTOR_IOTDB_PLAIN_BATCH_DELAY_DEFAULT_VALUE);
+ requestMaxDelayInMs =
+ requestMaxDelayInSeconds < 0 ? Integer.MAX_VALUE : requestMaxDelayInSeconds * 1000;
+ requestMaxBatchSizeInBytes =
+ parameters.getLongOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY),
+ CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE);
+ this.defaultBatch =
+ new PipeTabletEventPlainBatch(requestMaxDelayInMs, requestMaxBatchSizeInBytes);
+ }
+ }
+
+ /**
+ * Try offer {@link Event} into the corresponding batch if the given {@link Event} is not
+ * duplicated.
+ *
+ * @param event the given {@link Event}
+ * @return {@link Pair}<{@link TEndPoint}, {@link PipeTabletEventPlainBatch}> not null means this
+ * {@link PipeTabletEventPlainBatch} can be transferred. the first element is the leader
+ * endpoint to transfer to (might be null), the second element is the batch to be transferred.
+ */
+ public synchronized Pair<TEndPoint, PipeTabletEventBatch> onEvent(
+ final TabletInsertionEvent event) throws IOException {
+ if (!(event instanceof PipeRawTabletInsertionEvent)) {
+ LOGGER.warn(
+ "Unsupported event {} type {} when building transfer request", event, event.getClass());
+ return null;
+ }
+
+ if (!useLeaderCache) {
+ return defaultBatch.onEvent(event) ? new Pair<>(null, defaultBatch) : null;
+ }
+
+ String deviceId = ((PipeRawTabletInsertionEvent) event).getDeviceId();
+
+ if (Objects.isNull(deviceId)) {
+ return defaultBatch.onEvent(event) ? new Pair<>(null, defaultBatch) : null;
+ }
+
+ final TEndPoint endPoint =
+ new TEndPoint(PipeRuntimeOptions.RPC_ADDRESS.value(), PipeRuntimeOptions.RPC_PORT.value());
+
+ final PipeTabletEventPlainBatch batch =
+ endPointToBatch.computeIfAbsent(
+ endPoint,
+ k -> new PipeTabletEventPlainBatch(requestMaxDelayInMs, requestMaxBatchSizeInBytes));
+ return batch.onEvent(event) ? new Pair<>(endPoint, batch) : null;
+ }
+
+ /** Get all batches that have at least 1 event. */
+ public synchronized List<Pair<TEndPoint, PipeTabletEventBatch>> getAllNonEmptyBatches() {
+ final List<Pair<TEndPoint, PipeTabletEventBatch>> nonEmptyBatches = new ArrayList<>();
+ if (!defaultBatch.isEmpty()) {
+ nonEmptyBatches.add(new Pair<>(null, defaultBatch));
+ }
+ endPointToBatch.forEach(
+ (endPoint, batch) -> {
+ if (!batch.isEmpty()) {
+ nonEmptyBatches.add(new Pair<>(endPoint, batch));
+ }
+ });
+ return nonEmptyBatches;
+ }
+
+ public boolean isEmpty() {
+ return defaultBatch.isEmpty()
+ && endPointToBatch.values().stream().allMatch(PipeTabletEventPlainBatch::isEmpty);
+ }
+
+ @Override
+ public synchronized void close() {
+ defaultBatch.close();
+ endPointToBatch.values().forEach(PipeTabletEventPlainBatch::close);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/IoTDBConnectorRequestVersion.java
similarity index 71%
copy from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
copy to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/IoTDBConnectorRequestVersion.java
index e15a493..54f10f2 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/api/PullSource.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/IoTDBConnectorRequestVersion.java
@@ -17,8 +17,20 @@
* under the License.
*/
-package org.apache.iotdb.collector.plugin.api;
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
-import org.apache.iotdb.pipe.api.PipeSource;
+public enum IoTDBConnectorRequestVersion {
+ VERSION_1((byte) 1),
+ VERSION_2((byte) 2),
+ ;
-public abstract class PullSource implements PipeSource {}
+ private final byte version;
+
+ IoTDBConnectorRequestVersion(byte type) {
+ this.version = type;
+ }
+
+ public byte getVersion() {
+ return version;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeRequestType.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeRequestType.java
new file mode 100644
index 0000000..aab7926
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeRequestType.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public enum PipeRequestType {
+
+ // Handshake
+ HANDSHAKE_CONFIGNODE_V1((short) 0),
+ HANDSHAKE_DATANODE_V1((short) 1),
+ HANDSHAKE_CONFIGNODE_V2((short) 50),
+ HANDSHAKE_DATANODE_V2((short) 51),
+
+ // Data region
+ TRANSFER_TABLET_INSERT_NODE((short) 2),
+ TRANSFER_TABLET_RAW((short) 3),
+ TRANSFER_TS_FILE_PIECE((short) 4),
+ TRANSFER_TS_FILE_SEAL((short) 5),
+ TRANSFER_TABLET_BATCH((short) 6),
+ TRANSFER_TABLET_BINARY((short) 7),
+ TRANSFER_TS_FILE_PIECE_WITH_MOD((short) 8),
+ TRANSFER_TS_FILE_SEAL_WITH_MOD((short) 9),
+
+ TRANSFER_TABLET_INSERT_NODE_V2((short) 10),
+ TRANSFER_TABLET_RAW_V2((short) 11),
+ TRANSFER_TABLET_BINARY_V2((short) 12),
+ TRANSFER_TABLET_BATCH_V2((short) 13),
+
+ // Schema region / Delete Data
+ TRANSFER_PLAN_NODE((short) 100),
+ TRANSFER_SCHEMA_SNAPSHOT_PIECE((short) 101),
+ TRANSFER_SCHEMA_SNAPSHOT_SEAL((short) 102),
+
+ // Config region
+ TRANSFER_CONFIG_PLAN((short) 200),
+ TRANSFER_CONFIG_SNAPSHOT_PIECE((short) 201),
+ TRANSFER_CONFIG_SNAPSHOT_SEAL((short) 202),
+
+ // RPC Compression
+ TRANSFER_COMPRESSED((short) 300),
+
+ // Fallback Handling
+ TRANSFER_SLICE((short) 400),
+ ;
+
+ private final short type;
+
+ PipeRequestType(short type) {
+ this.type = type;
+ }
+
+ public short getType() {
+ return type;
+ }
+
+ private static final Map<Short, PipeRequestType> TYPE_MAP =
+ Arrays.stream(PipeRequestType.values())
+ .collect(
+ HashMap::new,
+ (typeMap, pipeRequestType) -> typeMap.put(pipeRequestType.getType(), pipeRequestType),
+ HashMap::putAll);
+
+ public static boolean isValidatedRequestType(short type) {
+ return TYPE_MAP.containsKey(type);
+ }
+
+ public static PipeRequestType valueOf(short type) {
+ return TYPE_MAP.get(type);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferCompressedReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferCompressedReq.java
new file mode 100644
index 0000000..7f53932
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferCompressedReq.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressor;
+import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressorFactory;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.BytesUtils;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class PipeTransferCompressedReq extends TPipeTransferReq {
+
+ /** Generate a compressed req with provided compressors. */
+ public static TPipeTransferReq toTPipeTransferReq(
+ final TPipeTransferReq originalReq, final List<PipeCompressor> compressors)
+ throws IOException {
+ // The generated PipeTransferCompressedReq consists of:
+ // version
+ // type: TRANSFER_COMPRESSED
+ // body:
+ // (byte) count of compressors (n)
+ // (n*3 bytes) for each compressor:
+ // (byte) compressor type
+ // (int) length of uncompressed bytes
+ // compressed req:
+ // (byte) version
+ // (2 bytes) type
+ // (bytes) body
+ final PipeTransferCompressedReq compressedReq = new PipeTransferCompressedReq();
+ compressedReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ compressedReq.type = PipeRequestType.TRANSFER_COMPRESSED.getType();
+
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ byte[] body =
+ BytesUtils.concatByteArrayList(
+ Arrays.asList(
+ new byte[] {originalReq.version},
+ BytesUtils.shortToBytes(originalReq.type),
+ originalReq.getBody()));
+
+ ReadWriteIOUtils.write((byte) compressors.size(), outputStream);
+ for (final PipeCompressor compressor : compressors) {
+ ReadWriteIOUtils.write(compressor.serialize(), outputStream);
+ ReadWriteIOUtils.write(body.length, outputStream);
+ body = compressor.compress(body);
+ }
+ outputStream.write(body);
+
+ compressedReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+ return compressedReq;
+ }
+
+ /** Get the original req from a compressed req. */
+ public static TPipeTransferReq fromTPipeTransferReq(final TPipeTransferReq transferReq)
+ throws IOException {
+ final ByteBuffer compressedBuffer = transferReq.body;
+
+ final List<PipeCompressor> compressors = new ArrayList<>();
+ final List<Integer> uncompressedLengths = new ArrayList<>();
+ final int compressorsSize = ReadWriteIOUtils.readByte(compressedBuffer);
+ for (int i = 0; i < compressorsSize; ++i) {
+ compressors.add(
+ PipeCompressorFactory.getCompressor(ReadWriteIOUtils.readByte(compressedBuffer)));
+ uncompressedLengths.add(ReadWriteIOUtils.readInt(compressedBuffer));
+ }
+
+ byte[] body = new byte[compressedBuffer.remaining()];
+ compressedBuffer.get(body);
+
+ for (int i = compressors.size() - 1; i >= 0; --i) {
+ body = compressors.get(i).decompress(body, uncompressedLengths.get(i));
+ }
+
+ final ByteBuffer decompressedBuffer = ByteBuffer.wrap(body);
+
+ final TPipeTransferReq decompressedReq = new TPipeTransferReq();
+ decompressedReq.version = ReadWriteIOUtils.readByte(decompressedBuffer);
+ decompressedReq.type = ReadWriteIOUtils.readShort(decompressedBuffer);
+ decompressedReq.body = decompressedBuffer.slice();
+
+ return decompressedReq;
+ }
+
+ /**
+ * For air-gap connectors. Generate the bytes of a compressed req from the bytes of original req.
+ */
+ public static byte[] toTPipeTransferReqBytes(
+ final byte[] rawReqInBytes, final List<PipeCompressor> compressors) throws IOException {
+ // The generated bytes consists of:
+ // (byte) version
+ // (2 bytes) type: TRANSFER_COMPRESSED
+ // (byte) count of compressors (n)
+ // (n*3 bytes) for each compressor:
+ // (byte) compressor type
+ // (int) length of uncompressed bytes
+ // compressed req:
+ // (byte) version
+ // (2 bytes) type
+ // (bytes) body
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ byte[] body = rawReqInBytes;
+
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(PipeRequestType.TRANSFER_COMPRESSED.getType(), outputStream);
+ ReadWriteIOUtils.write((byte) compressors.size(), outputStream);
+ for (final PipeCompressor compressor : compressors) {
+ ReadWriteIOUtils.write(compressor.serialize(), outputStream);
+ ReadWriteIOUtils.write(body.length, outputStream);
+ body = compressor.compress(body);
+ }
+ outputStream.write(body);
+
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ private PipeTransferCompressedReq() {
+ // Empty constructor
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferDataNodeHandshakeV1Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferDataNodeHandshakeV1Req.java
new file mode 100644
index 0000000..f4cd65b
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferDataNodeHandshakeV1Req.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import java.io.IOException;
+
+public class PipeTransferDataNodeHandshakeV1Req extends PipeTransferHandshakeV1Req {
+
+ private PipeTransferDataNodeHandshakeV1Req() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.HANDSHAKE_DATANODE_V1;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferDataNodeHandshakeV1Req toTPipeTransferReq(
+ final String timestampPrecision) throws IOException {
+ return (PipeTransferDataNodeHandshakeV1Req)
+ new PipeTransferDataNodeHandshakeV1Req().convertToTPipeTransferReq(timestampPrecision);
+ }
+
+ public static PipeTransferDataNodeHandshakeV1Req fromTPipeTransferReq(
+ final TPipeTransferReq transferReq) {
+ return (PipeTransferDataNodeHandshakeV1Req)
+ new PipeTransferDataNodeHandshakeV1Req().translateFromTPipeTransferReq(transferReq);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(final String timestampPrecision) throws IOException {
+ return new PipeTransferDataNodeHandshakeV1Req()
+ .convertToTransferHandshakeBytes(timestampPrecision);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeTransferDataNodeHandshakeV1Req && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferDataNodeHandshakeV2Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferDataNodeHandshakeV2Req.java
new file mode 100644
index 0000000..3fa37bf
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferDataNodeHandshakeV2Req.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class PipeTransferDataNodeHandshakeV2Req extends PipeTransferHandshakeV2Req {
+
+ private PipeTransferDataNodeHandshakeV2Req() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.HANDSHAKE_DATANODE_V2;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferDataNodeHandshakeV2Req toTPipeTransferReq(Map<String, String> params)
+ throws IOException {
+ return (PipeTransferDataNodeHandshakeV2Req)
+ new PipeTransferDataNodeHandshakeV2Req().convertToTPipeTransferReq(params);
+ }
+
+ public static PipeTransferDataNodeHandshakeV2Req fromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+ return (PipeTransferDataNodeHandshakeV2Req)
+ new PipeTransferDataNodeHandshakeV2Req().translateFromTPipeTransferReq(transferReq);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(Map<String, String> params) throws IOException {
+ return new PipeTransferDataNodeHandshakeV2Req().convertToTransferHandshakeBytes(params);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeTransferDataNodeHandshakeV2Req && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFilePieceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFilePieceReq.java
new file mode 100644
index 0000000..05ed843
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFilePieceReq.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Objects;
+
+public abstract class PipeTransferFilePieceReq extends TPipeTransferReq {
+
+ private transient String fileName;
+ private transient long startWritingOffset;
+ private transient byte[] filePiece;
+
+ protected abstract PipeRequestType getPlanType();
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ protected final PipeTransferFilePieceReq convertToTPipeTransferReq(
+ String snapshotName, long startWritingOffset, byte[] snapshotPiece) throws IOException {
+
+ this.fileName = snapshotName;
+ this.startWritingOffset = startWritingOffset;
+ this.filePiece = snapshotPiece;
+
+ this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ this.type = getPlanType().getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(snapshotName, outputStream);
+ ReadWriteIOUtils.write(startWritingOffset, outputStream);
+ ReadWriteIOUtils.write(new Binary(snapshotPiece), outputStream);
+ body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return this;
+ }
+
+ protected final PipeTransferFilePieceReq translateFromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+
+ fileName = ReadWriteIOUtils.readString(transferReq.body);
+ startWritingOffset = ReadWriteIOUtils.readLong(transferReq.body);
+ filePiece = ReadWriteIOUtils.readBinary(transferReq.body).getValues();
+
+ version = transferReq.version;
+ type = transferReq.type;
+ body = transferReq.body;
+
+ return this;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ protected final byte[] convertToTPipeTransferBytes(
+ String snapshotName, long startWritingOffset, byte[] snapshotPiece) throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(getPlanType().getType(), outputStream);
+ ReadWriteIOUtils.write(snapshotName, outputStream);
+ ReadWriteIOUtils.write(startWritingOffset, outputStream);
+ ReadWriteIOUtils.write(new Binary(snapshotPiece), outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferFilePieceReq that = (PipeTransferFilePieceReq) obj;
+ return fileName.equals(that.fileName)
+ && startWritingOffset == that.startWritingOffset
+ && Arrays.equals(filePiece, that.filePiece)
+ && version == that.version
+ && type == that.type
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ fileName, startWritingOffset, Arrays.hashCode(filePiece), version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV1.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV1.java
new file mode 100644
index 0000000..30f583d
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV1.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public abstract class PipeTransferFileSealReqV1 extends TPipeTransferReq {
+
+ private transient String fileName;
+ private transient long fileLength;
+
+ protected abstract PipeRequestType getPlanType();
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ protected PipeTransferFileSealReqV1 convertToTPipeTransferReq(String fileName, long fileLength)
+ throws IOException {
+
+ this.fileName = fileName;
+ this.fileLength = fileLength;
+
+ this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ this.type = getPlanType().getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(fileName, outputStream);
+ ReadWriteIOUtils.write(fileLength, outputStream);
+ this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return this;
+ }
+
+ public PipeTransferFileSealReqV1 translateFromTPipeTransferReq(TPipeTransferReq req) {
+
+ fileName = ReadWriteIOUtils.readString(req.body);
+ fileLength = ReadWriteIOUtils.readLong(req.body);
+
+ version = req.version;
+ type = req.type;
+ body = req.body;
+
+ return this;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public byte[] convertToTPipeTransferSnapshotSealBytes(String fileName, long fileLength)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(getPlanType().getType(), outputStream);
+ ReadWriteIOUtils.write(fileName, outputStream);
+ ReadWriteIOUtils.write(fileLength, outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferFileSealReqV1 that = (PipeTransferFileSealReqV1) obj;
+ return fileName.equals(that.fileName)
+ && fileLength == that.fileLength
+ && version == that.version
+ && type == that.type
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(fileName, fileLength, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV2.java
new file mode 100644
index 0000000..a380300
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferFileSealReqV2.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public abstract class PipeTransferFileSealReqV2 extends TPipeTransferReq {
+
+ public static final String DATABASE_PATTERN = "database_pattern";
+ public static final String TREE = "tree";
+ public static final String TABLE = "table";
+ protected transient List<String> fileNames;
+ protected transient List<Long> fileLengths;
+ protected transient Map<String, String> parameters;
+
+ public final Map<String, String> getParameters() {
+ return parameters;
+ }
+
+ protected abstract PipeRequestType getPlanType();
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ protected PipeTransferFileSealReqV2 convertToTPipeTransferReq(
+ final List<String> fileNames,
+ final List<Long> fileLengths,
+ final Map<String, String> parameters)
+ throws IOException {
+
+ this.fileNames = fileNames;
+ this.fileLengths = fileLengths;
+ this.parameters = parameters;
+
+ this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ this.type = getPlanType().getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(fileNames.size(), outputStream);
+ for (final String fileName : fileNames) {
+ ReadWriteIOUtils.write(fileName, outputStream);
+ }
+ ReadWriteIOUtils.write(fileLengths.size(), outputStream);
+ for (final Long fileLength : fileLengths) {
+ ReadWriteIOUtils.write(fileLength, outputStream);
+ }
+ ReadWriteIOUtils.write(parameters.size(), outputStream);
+ for (final Map.Entry<String, String> entry : parameters.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+ this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return this;
+ }
+
+ public PipeTransferFileSealReqV2 translateFromTPipeTransferReq(final TPipeTransferReq req) {
+ fileNames = new ArrayList<>();
+ int size = ReadWriteIOUtils.readInt(req.body);
+ for (int i = 0; i < size; ++i) {
+ fileNames.add(ReadWriteIOUtils.readString(req.body));
+ }
+
+ fileLengths = new ArrayList<>();
+ size = ReadWriteIOUtils.readInt(req.body);
+ for (int i = 0; i < size; ++i) {
+ fileLengths.add(ReadWriteIOUtils.readLong(req.body));
+ }
+
+ parameters = new HashMap<>();
+ size = ReadWriteIOUtils.readInt(req.body);
+ for (int i = 0; i < size; ++i) {
+ final String key = ReadWriteIOUtils.readString(req.body);
+ final String value = ReadWriteIOUtils.readString(req.body);
+ parameters.put(key, value);
+ }
+
+ version = req.version;
+ type = req.type;
+ body = req.body;
+
+ return this;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public byte[] convertToTPipeTransferSnapshotSealBytes(
+ List<String> fileNames, List<Long> fileLengths, Map<String, String> parameters)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(getPlanType().getType(), outputStream);
+ ReadWriteIOUtils.write(fileNames.size(), outputStream);
+ for (String fileName : fileNames) {
+ ReadWriteIOUtils.write(fileName, outputStream);
+ }
+ ReadWriteIOUtils.write(fileLengths.size(), outputStream);
+ for (Long fileLength : fileLengths) {
+ ReadWriteIOUtils.write(fileLength, outputStream);
+ }
+ ReadWriteIOUtils.write(parameters.size(), outputStream);
+ for (final Map.Entry<String, String> entry : parameters.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferFileSealReqV2 that = (PipeTransferFileSealReqV2) obj;
+ return Objects.equals(fileNames, that.fileNames)
+ && Objects.equals(fileLengths, that.fileLengths)
+ && Objects.equals(parameters, that.parameters)
+ && Objects.equals(version, that.version)
+ && Objects.equals(type, that.type)
+ && Objects.equals(body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(fileNames, fileLengths, parameters, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV1Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV1Req.java
new file mode 100644
index 0000000..a95bb55
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV1Req.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public abstract class PipeTransferHandshakeV1Req extends TPipeTransferReq {
+
+ private transient String timestampPrecision;
+
+ protected abstract PipeRequestType getPlanType();
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public final PipeTransferHandshakeV1Req convertToTPipeTransferReq(String timestampPrecision)
+ throws IOException {
+ this.timestampPrecision = timestampPrecision;
+
+ this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ this.type = getPlanType().getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(timestampPrecision, outputStream);
+ this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return this;
+ }
+
+ protected final PipeTransferHandshakeV1Req translateFromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+ timestampPrecision = ReadWriteIOUtils.readString(transferReq.body);
+
+ version = transferReq.version;
+ type = transferReq.type;
+ body = transferReq.body;
+
+ return this;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ protected final byte[] convertToTransferHandshakeBytes(String timestampPrecision)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(getPlanType().getType(), outputStream);
+ ReadWriteIOUtils.write(timestampPrecision, outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferHandshakeV1Req that = (PipeTransferHandshakeV1Req) obj;
+ return timestampPrecision.equals(that.timestampPrecision)
+ && version == that.version
+ && type == that.type
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(timestampPrecision, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java
new file mode 100644
index 0000000..02c1310
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public abstract class PipeTransferHandshakeV2Req extends TPipeTransferReq {
+
+ private transient Map<String, String> params;
+
+ protected abstract PipeRequestType getPlanType();
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map<String, String> params)
+ throws IOException {
+ this.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ this.type = getPlanType().getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(params.size(), outputStream);
+ for (final Map.Entry<String, String> entry : params.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+ this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ this.params = params;
+ return this;
+ }
+
+ protected final PipeTransferHandshakeV2Req translateFromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+ Map<String, String> params = new HashMap<>();
+ final int size = ReadWriteIOUtils.readInt(transferReq.body);
+ for (int i = 0; i < size; ++i) {
+ final String key = ReadWriteIOUtils.readString(transferReq.body);
+ final String value = ReadWriteIOUtils.readString(transferReq.body);
+ params.put(key, value);
+ }
+ this.params = params;
+
+ version = transferReq.version;
+ type = transferReq.type;
+ body = transferReq.body;
+
+ return this;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public final byte[] convertToTransferHandshakeBytes(Map<String, String> params)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(getPlanType().getType(), outputStream);
+ ReadWriteIOUtils.write(params.size(), outputStream);
+ for (final Map.Entry<String, String> entry : params.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferHandshakeV2Req that = (PipeTransferHandshakeV2Req) obj;
+ return Objects.equals(params, that.params)
+ && version == that.version
+ && type == that.type
+ && Objects.equals(body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(params, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSchemaSnapshotPieceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSchemaSnapshotPieceReq.java
new file mode 100644
index 0000000..518fd73
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSchemaSnapshotPieceReq.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import java.io.IOException;
+
+public class PipeTransferSchemaSnapshotPieceReq extends PipeTransferFilePieceReq {
+
+ private PipeTransferSchemaSnapshotPieceReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferSchemaSnapshotPieceReq toTPipeTransferReq(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return (PipeTransferSchemaSnapshotPieceReq)
+ new PipeTransferSchemaSnapshotPieceReq()
+ .convertToTPipeTransferReq(fileName, startWritingOffset, filePiece);
+ }
+
+ public static PipeTransferSchemaSnapshotPieceReq fromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+ return (PipeTransferSchemaSnapshotPieceReq)
+ new PipeTransferSchemaSnapshotPieceReq().translateFromTPipeTransferReq(transferReq);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return new PipeTransferSchemaSnapshotPieceReq()
+ .convertToTPipeTransferBytes(fileName, startWritingOffset, filePiece);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeTransferSchemaSnapshotPieceReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSchemaSnapshotSealReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSchemaSnapshotSealReq.java
new file mode 100644
index 0000000..32d8bef
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSchemaSnapshotSealReq.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.constant.ColumnHeaderConstant;
+import org.apache.iotdb.collector.plugin.builtin.sink.protocol.session.IClientSession;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class PipeTransferSchemaSnapshotSealReq extends PipeTransferFileSealReqV2 {
+
+ private PipeTransferSchemaSnapshotSealReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferSchemaSnapshotSealReq toTPipeTransferReq(
+ final String treePattern,
+ final String tablePatternDatabase,
+ final String tablePatternTable,
+ final boolean isTreeCaptured,
+ final boolean isTableCaptured,
+ final String mTreeSnapshotName,
+ final long mTreeSnapshotLength,
+ final String tLogName,
+ final long tLogLength,
+ final String attributeSnapshotName,
+ final long attributeSnapshotLength,
+ final String databaseName,
+ final String typeString)
+ throws IOException {
+ final Map<String, String> parameters = new HashMap<>();
+ parameters.put(ColumnHeaderConstant.PATH_PATTERN, treePattern);
+ parameters.put(DATABASE_PATTERN, tablePatternDatabase);
+ parameters.put(ColumnHeaderConstant.TABLE_NAME, tablePatternTable);
+ if (isTreeCaptured) {
+ parameters.put(IClientSession.SqlDialect.TREE.toString(), "");
+ }
+ if (isTableCaptured) {
+ parameters.put(IClientSession.SqlDialect.TABLE.toString(), "");
+ }
+ parameters.put(ColumnHeaderConstant.DATABASE, databaseName);
+ parameters.put(ColumnHeaderConstant.TYPE, typeString);
+
+ final List<String> fileNameList;
+ final List<Long> fileLengthList;
+
+ // Tree model sync
+ if (Objects.isNull(attributeSnapshotName)) {
+ fileNameList =
+ Objects.nonNull(tLogName)
+ ? Arrays.asList(mTreeSnapshotName, tLogName)
+ : Collections.singletonList(mTreeSnapshotName);
+ fileLengthList =
+ Objects.nonNull(tLogName)
+ ? Arrays.asList(mTreeSnapshotLength, tLogLength)
+ : Collections.singletonList(mTreeSnapshotLength);
+ } else {
+ fileNameList = Arrays.asList(mTreeSnapshotName, tLogName, attributeSnapshotName);
+ fileLengthList = Arrays.asList(mTreeSnapshotLength, tLogLength, attributeSnapshotLength);
+ }
+
+ return (PipeTransferSchemaSnapshotSealReq)
+ new PipeTransferSchemaSnapshotSealReq()
+ .convertToTPipeTransferReq(fileNameList, fileLengthList, parameters);
+ }
+
+ public static PipeTransferSchemaSnapshotSealReq fromTPipeTransferReq(final TPipeTransferReq req) {
+ return (PipeTransferSchemaSnapshotSealReq)
+ new PipeTransferSchemaSnapshotSealReq().translateFromTPipeTransferReq(req);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ final String treePattern,
+ final String tablePatternDatabase,
+ final String tablePatternTable,
+ final boolean isTreeCaptured,
+ final boolean isTableCaptured,
+ final String mTreeSnapshotName,
+ final long mTreeSnapshotLength,
+ final String tLogName,
+ final long tLogLength,
+ final String attributeSnapshotName,
+ final long attributeSnapshotLength,
+ final String databaseName,
+ final String typeString)
+ throws IOException {
+ final Map<String, String> parameters = new HashMap<>();
+ parameters.put(ColumnHeaderConstant.PATH_PATTERN, treePattern);
+ parameters.put(DATABASE_PATTERN, tablePatternDatabase);
+ parameters.put(ColumnHeaderConstant.TABLE_NAME, tablePatternTable);
+ if (isTreeCaptured) {
+ parameters.put(TREE, "");
+ }
+ if (isTableCaptured) {
+ parameters.put(TABLE, "");
+ }
+ parameters.put(ColumnHeaderConstant.DATABASE, databaseName);
+ parameters.put(ColumnHeaderConstant.TYPE, typeString);
+
+ final List<String> fileNameList;
+ final List<Long> fileLengthList;
+
+ // Tree model sync
+ if (Objects.isNull(attributeSnapshotName)) {
+ fileNameList =
+ Objects.nonNull(tLogName)
+ ? Arrays.asList(mTreeSnapshotName, tLogName)
+ : Collections.singletonList(mTreeSnapshotName);
+ fileLengthList =
+ Objects.nonNull(tLogName)
+ ? Arrays.asList(mTreeSnapshotLength, tLogLength)
+ : Collections.singletonList(mTreeSnapshotLength);
+ } else {
+ fileNameList = Arrays.asList(mTreeSnapshotName, tLogName, attributeSnapshotName);
+ fileLengthList = Arrays.asList(mTreeSnapshotLength, tLogLength, attributeSnapshotLength);
+ }
+
+ return new PipeTransferSchemaSnapshotSealReq()
+ .convertToTPipeTransferSnapshotSealBytes(fileNameList, fileLengthList, parameters);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeTransferSchemaSnapshotSealReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSliceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSliceReq.java
new file mode 100644
index 0000000..48cee90
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferSliceReq.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Objects;
+
+public class PipeTransferSliceReq extends TPipeTransferReq {
+
+ private transient int orderId;
+
+ private transient short originReqType;
+ private transient int originBodySize;
+
+ private transient byte[] sliceBody;
+
+ private transient int sliceIndex;
+ private transient int sliceCount;
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferSliceReq toTPipeTransferReq(
+ final int orderId,
+ final short originReqType,
+ final int sliceIndex,
+ final int sliceCount,
+ final ByteBuffer duplicatedOriginBody,
+ final int startIndexInBody,
+ final int endIndexInBody)
+ throws IOException {
+ final PipeTransferSliceReq sliceReq = new PipeTransferSliceReq();
+
+ sliceReq.orderId = orderId;
+
+ sliceReq.originReqType = originReqType;
+ sliceReq.originBodySize = duplicatedOriginBody.limit();
+
+ sliceReq.sliceBody = new byte[endIndexInBody - startIndexInBody];
+ duplicatedOriginBody.position(startIndexInBody);
+ duplicatedOriginBody.get(sliceReq.sliceBody);
+
+ sliceReq.sliceIndex = sliceIndex;
+ sliceReq.sliceCount = sliceCount;
+
+ sliceReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ sliceReq.type = PipeRequestType.TRANSFER_SLICE.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(sliceReq.orderId, outputStream);
+
+ ReadWriteIOUtils.write(sliceReq.originReqType, outputStream);
+ ReadWriteIOUtils.write(sliceReq.originBodySize, outputStream);
+
+ ReadWriteIOUtils.write(new Binary(sliceReq.sliceBody), outputStream);
+
+ ReadWriteIOUtils.write(sliceReq.sliceIndex, outputStream);
+ ReadWriteIOUtils.write(sliceReq.sliceCount, outputStream);
+
+ sliceReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return sliceReq;
+ }
+
+ public static PipeTransferSliceReq fromTPipeTransferReq(final TPipeTransferReq transferReq) {
+ final PipeTransferSliceReq sliceReq = new PipeTransferSliceReq();
+
+ sliceReq.orderId = ReadWriteIOUtils.readInt(transferReq.body);
+
+ sliceReq.originReqType = ReadWriteIOUtils.readShort(transferReq.body);
+ sliceReq.originBodySize = ReadWriteIOUtils.readInt(transferReq.body);
+
+ sliceReq.sliceBody = ReadWriteIOUtils.readBinary(transferReq.body).getValues();
+
+ sliceReq.sliceIndex = ReadWriteIOUtils.readInt(transferReq.body);
+ sliceReq.sliceCount = ReadWriteIOUtils.readInt(transferReq.body);
+
+ sliceReq.version = transferReq.version;
+ sliceReq.type = transferReq.type;
+ sliceReq.body = transferReq.body;
+
+ return sliceReq;
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeTransferSliceReq that = (PipeTransferSliceReq) obj;
+ return Objects.equals(orderId, that.orderId)
+ && Objects.equals(originReqType, that.originReqType)
+ && Objects.equals(originBodySize, that.originBodySize)
+ && Arrays.equals(sliceBody, that.sliceBody)
+ && Objects.equals(sliceIndex, that.sliceIndex)
+ && Objects.equals(sliceCount, that.sliceCount)
+ && Objects.equals(version, that.version)
+ && Objects.equals(type, that.type)
+ && Objects.equals(body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ orderId,
+ originReqType,
+ originBodySize,
+ Arrays.hashCode(sliceBody),
+ sliceIndex,
+ sliceCount,
+ version,
+ type,
+ body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletBatchReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletBatchReqV2.java
new file mode 100644
index 0000000..f59678e
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletBatchReqV2.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class PipeTransferTabletBatchReqV2 extends TPipeTransferReq {
+
+ private final transient List<PipeTransferTabletBinaryReqV2> binaryReqs = new ArrayList<>();
+ private final transient List<PipeTransferTabletRawReqV2> tabletReqs = new ArrayList<>();
+
+ private PipeTransferTabletBatchReqV2() {
+ // Empty constructor
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletBatchReqV2 toTPipeTransferReq(
+ final List<ByteBuffer> binaryBuffers,
+ final List<ByteBuffer> insertNodeBuffers,
+ final List<ByteBuffer> tabletBuffers,
+ final List<String> binaryDataBases,
+ final List<String> insertNodeDataBases,
+ final List<String> tabletDataBases)
+ throws IOException {
+ final PipeTransferTabletBatchReqV2 batchReq = new PipeTransferTabletBatchReqV2();
+
+ batchReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ batchReq.type = PipeRequestType.TRANSFER_TABLET_BATCH_V2.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(binaryBuffers.size(), outputStream);
+ for (int i = 0; i < binaryBuffers.size(); i++) {
+ final ByteBuffer binaryBuffer = binaryBuffers.get(i);
+ ReadWriteIOUtils.write(binaryBuffer.limit(), outputStream);
+ outputStream.write(binaryBuffer.array(), 0, binaryBuffer.limit());
+ ReadWriteIOUtils.write(binaryDataBases.get(i), outputStream);
+ }
+
+ ReadWriteIOUtils.write(insertNodeBuffers.size(), outputStream);
+ for (int i = 0; i < insertNodeBuffers.size(); i++) {
+ final ByteBuffer insertNodeBuffer = insertNodeBuffers.get(i);
+ outputStream.write(insertNodeBuffer.array(), 0, insertNodeBuffer.limit());
+ ReadWriteIOUtils.write(insertNodeDataBases.get(i), outputStream);
+ }
+
+ ReadWriteIOUtils.write(tabletBuffers.size(), outputStream);
+ for (int i = 0; i < tabletBuffers.size(); i++) {
+ final ByteBuffer tabletBuffer = tabletBuffers.get(i);
+ outputStream.write(tabletBuffer.array(), 0, tabletBuffer.limit());
+ ReadWriteIOUtils.write(tabletDataBases.get(i), outputStream);
+ }
+
+ batchReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return batchReq;
+ }
+
+ // public static PipeTransferTabletBatchReqV2 fromTPipeTransferReq(
+ // final TPipeTransferReq transferReq) {
+ // final PipeTransferTabletBatchReqV2 batchReq = new PipeTransferTabletBatchReqV2();
+ //
+ // int size = ReadWriteIOUtils.readInt(transferReq.body);
+ // for (int i = 0; i < size; ++i) {
+ // final int length = ReadWriteIOUtils.readInt(transferReq.body);
+ // final byte[] body = new byte[length];
+ // transferReq.body.get(body);
+ // batchReq.binaryReqs.add(
+ // PipeTransferTabletBinaryReqV2.toTPipeTransferBinaryReq(
+ // ByteBuffer.wrap(body), ReadWriteIOUtils.readString(transferReq.body)));
+ // }
+ //
+ // size = ReadWriteIOUtils.readInt(transferReq.body);
+ // for (int i = 0; i < size; ++i) {
+ // batchReq.insertNodeReqs.add(
+ // PipeTransferTabletInsertNodeReqV2.toTabletInsertNodeReq(
+ // (InsertNode) PlanFragment.deserializeHelper(transferReq.body, null),
+ // ReadWriteIOUtils.readString(transferReq.body)));
+ // }
+ //
+ // size = ReadWriteIOUtils.readInt(transferReq.body);
+ // for (int i = 0; i < size; ++i) {
+ // batchReq.tabletReqs.add(
+ // PipeTransferTabletRawReqV2.toTPipeTransferRawReq(
+ // Tablet.deserialize(transferReq.body),
+ // ReadWriteIOUtils.readBool(transferReq.body),
+ // ReadWriteIOUtils.readString(transferReq.body)));
+ // }
+ //
+ // batchReq.version = transferReq.version;
+ // batchReq.type = transferReq.type;
+ // batchReq.body = transferReq.body;
+ //
+ // return batchReq;
+ // }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeTransferTabletBatchReqV2 that = (PipeTransferTabletBatchReqV2) obj;
+ return Objects.equals(binaryReqs, that.binaryReqs)
+ && Objects.equals(tabletReqs, that.tabletReqs)
+ && version == that.version
+ && type == that.type
+ && Objects.equals(body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(binaryReqs, tabletReqs, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletBinaryReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletBinaryReq.java
new file mode 100644
index 0000000..cc9259d
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletBinaryReq.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.BytesUtils;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeTransferTabletBinaryReq extends TPipeTransferReq {
+
+ protected transient ByteBuffer byteBuffer;
+
+ protected PipeTransferTabletBinaryReq() {
+ // Do nothing
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletBinaryReq toTPipeTransferReq(final ByteBuffer byteBuffer) {
+ final PipeTransferTabletBinaryReq req = new PipeTransferTabletBinaryReq();
+ req.byteBuffer = byteBuffer;
+
+ req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ req.type = PipeRequestType.TRANSFER_TABLET_BINARY.getType();
+ req.body = byteBuffer;
+
+ return req;
+ }
+
+ public static PipeTransferTabletBinaryReq fromTPipeTransferReq(
+ final TPipeTransferReq transferReq) {
+ final PipeTransferTabletBinaryReq binaryReq = new PipeTransferTabletBinaryReq();
+ binaryReq.byteBuffer = transferReq.body;
+
+ binaryReq.version = transferReq.version;
+ binaryReq.type = transferReq.type;
+ binaryReq.body = transferReq.body;
+
+ return binaryReq;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(final ByteBuffer byteBuffer) throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_BINARY.getType(), outputStream);
+ return BytesUtils.concatByteArray(byteArrayOutputStream.toByteArray(), byteBuffer.array());
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeTransferTabletBinaryReq that = (PipeTransferTabletBinaryReq) obj;
+ return byteBuffer.equals(that.byteBuffer)
+ && version == that.version
+ && type == that.type
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(byteBuffer, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletBinaryReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletBinaryReqV2.java
new file mode 100644
index 0000000..621b282
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletBinaryReqV2.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeTransferTabletBinaryReqV2 extends PipeTransferTabletBinaryReq {
+
+ private transient String dataBaseName;
+
+ protected PipeTransferTabletBinaryReqV2() {
+ // Do nothing
+ }
+
+ /////////////////////////////// Batch ///////////////////////////////
+
+ public static PipeTransferTabletBinaryReqV2 toTPipeTransferBinaryReq(
+ final ByteBuffer byteBuffer, final String dataBaseName) {
+ final PipeTransferTabletBinaryReqV2 req = new PipeTransferTabletBinaryReqV2();
+
+ req.byteBuffer = byteBuffer;
+ req.dataBaseName = dataBaseName;
+ req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ req.type = PipeRequestType.TRANSFER_TABLET_BINARY_V2.getType();
+
+ return req;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletBinaryReqV2 toTPipeTransferReq(
+ final ByteBuffer byteBuffer, final String dataBaseName) throws IOException {
+ final PipeTransferTabletBinaryReqV2 req = new PipeTransferTabletBinaryReqV2();
+ req.byteBuffer = byteBuffer;
+ req.dataBaseName = dataBaseName;
+
+ req.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ req.type = PipeRequestType.TRANSFER_TABLET_BINARY_V2.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(byteBuffer.limit(), outputStream);
+ outputStream.write(byteBuffer.array(), 0, byteBuffer.limit());
+ ReadWriteIOUtils.write(dataBaseName, outputStream);
+ req.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return req;
+ }
+
+ public static PipeTransferTabletBinaryReqV2 fromTPipeTransferReq(
+ final org.apache.iotdb.service.rpc.thrift.TPipeTransferReq transferReq) {
+ final PipeTransferTabletBinaryReqV2 binaryReq = new PipeTransferTabletBinaryReqV2();
+
+ final int length = ReadWriteIOUtils.readInt(transferReq.body);
+ final byte[] body = new byte[length];
+ transferReq.body.get(body);
+ binaryReq.byteBuffer = ByteBuffer.wrap(body);
+ binaryReq.dataBaseName = ReadWriteIOUtils.readString(transferReq.body);
+
+ binaryReq.version = transferReq.version;
+ binaryReq.type = transferReq.type;
+ binaryReq.body = transferReq.body;
+
+ return binaryReq;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(final ByteBuffer byteBuffer, final String dataBaseName)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_BINARY_V2.getType(), outputStream);
+ ReadWriteIOUtils.write(byteBuffer.limit(), outputStream);
+ outputStream.write(byteBuffer.array(), 0, byteBuffer.limit());
+ ReadWriteIOUtils.write(dataBaseName, outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ final PipeTransferTabletBinaryReqV2 that = (PipeTransferTabletBinaryReqV2) o;
+ return Objects.equals(dataBaseName, that.dataBaseName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), dataBaseName);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletRawReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletRawReq.java
new file mode 100644
index 0000000..b1b3fa9
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletRawReq.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.apache.tsfile.write.record.Tablet;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeTransferTabletRawReq extends TPipeTransferReq {
+
+ protected transient Tablet tablet;
+ protected transient boolean isAligned;
+
+ public Tablet getTablet() {
+ return tablet;
+ }
+
+ public boolean getIsAligned() {
+ return isAligned;
+ }
+
+ /////////////////////////////// WriteBack & Batch ///////////////////////////////
+
+ public static PipeTransferTabletRawReq toTPipeTransferRawReq(
+ final Tablet tablet, final boolean isAligned) {
+ final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq();
+
+ tabletReq.tablet = tablet;
+ tabletReq.isAligned = isAligned;
+
+ return tabletReq;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletRawReq toTPipeTransferReq(
+ final Tablet tablet, final boolean isAligned) throws IOException {
+ PipeTransferTabletRawReq tabletReq = toTPipeTransferRawReq(tablet, isAligned);
+
+ tabletReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ tabletReq.type = PipeRequestType.TRANSFER_TABLET_RAW.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ tablet.serialize(outputStream);
+ ReadWriteIOUtils.write(isAligned, outputStream);
+ tabletReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return tabletReq;
+ }
+
+ public static PipeTransferTabletRawReq fromTPipeTransferReq(final TPipeTransferReq transferReq) {
+ final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq();
+
+ tabletReq.tablet = Tablet.deserialize(transferReq.body);
+ tabletReq.isAligned = ReadWriteIOUtils.readBool(transferReq.body);
+
+ tabletReq.version = transferReq.version;
+ tabletReq.type = transferReq.type;
+ tabletReq.body = transferReq.body;
+
+ return tabletReq;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(final Tablet tablet, final boolean isAligned)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), outputStream);
+ tablet.serialize(outputStream);
+ ReadWriteIOUtils.write(isAligned, outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeTransferTabletRawReq that = (PipeTransferTabletRawReq) obj;
+ return Objects.equals(tablet, that.tablet)
+ && isAligned == that.isAligned
+ && version == that.version
+ && type == that.type
+ && Objects.equals(body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tablet, isAligned, version, type, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletRawReqV2.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletRawReqV2.java
new file mode 100644
index 0000000..45f51df
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTabletRawReqV2.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.apache.tsfile.write.record.Tablet;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeTransferTabletRawReqV2 extends PipeTransferTabletRawReq {
+
+ protected transient String dataBaseName;
+
+ /////////////////////////////// WriteBack & Batch ///////////////////////////////
+
+ public static PipeTransferTabletRawReqV2 toTPipeTransferRawReq(
+ final Tablet tablet, final boolean isAligned, final String dataBaseName) {
+ final PipeTransferTabletRawReqV2 tabletReq = new PipeTransferTabletRawReqV2();
+
+ tabletReq.tablet = tablet;
+ tabletReq.isAligned = isAligned;
+ tabletReq.dataBaseName = dataBaseName;
+ tabletReq.version = IoTDBConnectorRequestVersion.VERSION_1.getVersion();
+ tabletReq.type = PipeRequestType.TRANSFER_TABLET_RAW_V2.getType();
+
+ return tabletReq;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTabletRawReqV2 toTPipeTransferReq(
+ final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException {
+ PipeTransferTabletRawReqV2 tabletReq = toTPipeTransferRawReq(tablet, isAligned, dataBaseName);
+
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ tablet.serialize(outputStream);
+ ReadWriteIOUtils.write(isAligned, outputStream);
+ ReadWriteIOUtils.write(dataBaseName, outputStream);
+ tabletReq.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return tabletReq;
+ }
+
+ public static PipeTransferTabletRawReqV2 fromTPipeTransferReq(
+ final TPipeTransferReq transferReq) {
+ final PipeTransferTabletRawReqV2 tabletReq = new PipeTransferTabletRawReqV2();
+
+ tabletReq.tablet = Tablet.deserialize(transferReq.body);
+ tabletReq.isAligned = ReadWriteIOUtils.readBool(transferReq.body);
+ tabletReq.dataBaseName = ReadWriteIOUtils.readString(transferReq.body);
+
+ tabletReq.version = transferReq.version;
+ tabletReq.type = transferReq.type;
+ tabletReq.body = transferReq.body;
+
+ return tabletReq;
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
+ ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), outputStream);
+ tablet.serialize(outputStream);
+ ReadWriteIOUtils.write(isAligned, outputStream);
+ ReadWriteIOUtils.write(dataBaseName, outputStream);
+ return byteArrayOutputStream.toByteArray();
+ }
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ final PipeTransferTabletRawReqV2 that = (PipeTransferTabletRawReqV2) o;
+ return Objects.equals(dataBaseName, that.dataBaseName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), dataBaseName);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTsFilePieceReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTsFilePieceReq.java
new file mode 100644
index 0000000..60afaec
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTsFilePieceReq.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import java.io.IOException;
+
+public class PipeTransferTsFilePieceReq extends PipeTransferFilePieceReq {
+
+ private PipeTransferTsFilePieceReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_TS_FILE_PIECE;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTsFilePieceReq toTPipeTransferReq(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return (PipeTransferTsFilePieceReq)
+ new PipeTransferTsFilePieceReq()
+ .convertToTPipeTransferReq(fileName, startWritingOffset, filePiece);
+ }
+
+ public static PipeTransferTsFilePieceReq fromTPipeTransferReq(TPipeTransferReq transferReq) {
+ return (PipeTransferTsFilePieceReq)
+ new PipeTransferTsFilePieceReq().translateFromTPipeTransferReq(transferReq);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return new PipeTransferTsFilePieceReq()
+ .convertToTPipeTransferBytes(fileName, startWritingOffset, filePiece);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeTransferTsFilePieceReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTsFilePieceWithModReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTsFilePieceWithModReq.java
new file mode 100644
index 0000000..3cb7996
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTsFilePieceWithModReq.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import java.io.IOException;
+
+public class PipeTransferTsFilePieceWithModReq extends PipeTransferFilePieceReq {
+
+ private PipeTransferTsFilePieceWithModReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_TS_FILE_PIECE_WITH_MOD;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTsFilePieceWithModReq toTPipeTransferReq(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return (PipeTransferTsFilePieceWithModReq)
+ new PipeTransferTsFilePieceWithModReq()
+ .convertToTPipeTransferReq(fileName, startWritingOffset, filePiece);
+ }
+
+ public static PipeTransferTsFilePieceWithModReq fromTPipeTransferReq(
+ TPipeTransferReq transferReq) {
+ return (PipeTransferTsFilePieceWithModReq)
+ new PipeTransferTsFilePieceWithModReq().translateFromTPipeTransferReq(transferReq);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ String fileName, long startWritingOffset, byte[] filePiece) throws IOException {
+ return new PipeTransferTsFilePieceWithModReq()
+ .convertToTPipeTransferBytes(fileName, startWritingOffset, filePiece);
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof PipeTransferTsFilePieceWithModReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTsFileSealWithModReq.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTsFileSealWithModReq.java
new file mode 100644
index 0000000..b6e15dc
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/request/PipeTransferTsFileSealWithModReq.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+
+public class PipeTransferTsFileSealWithModReq extends PipeTransferFileSealReqV2 {
+
+ private PipeTransferTsFileSealWithModReq() {
+ // Empty constructor
+ }
+
+ @Override
+ protected PipeRequestType getPlanType() {
+ return PipeRequestType.TRANSFER_TS_FILE_SEAL_WITH_MOD;
+ }
+
+ protected static final String DATABASE_NAME_KEY_PREFIX = "DATABASE_NAME_";
+
+ protected static String generateDatabaseNameWithFileNameKey(final String fileName) {
+ return DATABASE_NAME_KEY_PREFIX + fileName;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferTsFileSealWithModReq toTPipeTransferReq(
+ final String modFileName,
+ final long modFileLength,
+ final String tsFileName,
+ final long tsFileLength)
+ throws IOException {
+ return toTPipeTransferReq(modFileName, modFileLength, tsFileName, tsFileLength, null);
+ }
+
+ public static PipeTransferTsFileSealWithModReq toTPipeTransferReq(
+ final String modFileName,
+ final long modFileLength,
+ final String tsFileName,
+ final long tsFileLength,
+ final String dataBaseName)
+ throws IOException {
+ return (PipeTransferTsFileSealWithModReq)
+ new PipeTransferTsFileSealWithModReq()
+ .convertToTPipeTransferReq(
+ Arrays.asList(modFileName, tsFileName),
+ Arrays.asList(modFileLength, tsFileLength),
+ Collections.singletonMap(
+ generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName));
+ }
+
+ public static PipeTransferTsFileSealWithModReq toTPipeTransferReq(
+ final String tsFileName, final long tsFileLength, final String dataBaseName)
+ throws IOException {
+ return (PipeTransferTsFileSealWithModReq)
+ new PipeTransferTsFileSealWithModReq()
+ .convertToTPipeTransferReq(
+ Collections.singletonList(tsFileName),
+ Collections.singletonList(tsFileLength),
+ Collections.singletonMap(
+ generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName));
+ }
+
+ public static PipeTransferTsFileSealWithModReq fromTPipeTransferReq(final TPipeTransferReq req) {
+ return (PipeTransferTsFileSealWithModReq)
+ new PipeTransferTsFileSealWithModReq().translateFromTPipeTransferReq(req);
+ }
+
+ /////////////////////////////// Air Gap ///////////////////////////////
+
+ public static byte[] toTPipeTransferBytes(
+ final String modFileName,
+ final long modFileLength,
+ final String tsFileName,
+ final long tsFileLength)
+ throws IOException {
+ return new PipeTransferTsFileSealWithModReq()
+ .convertToTPipeTransferSnapshotSealBytes(
+ Arrays.asList(modFileName, tsFileName),
+ Arrays.asList(modFileLength, tsFileLength),
+ new HashMap<>());
+ }
+
+ public static byte[] toTPipeTransferBytes(
+ final String modFileName,
+ final long modFileLength,
+ final String tsFileName,
+ final long tsFileLength,
+ final String dataBaseName)
+ throws IOException {
+ return new PipeTransferTsFileSealWithModReq()
+ .convertToTPipeTransferSnapshotSealBytes(
+ Arrays.asList(modFileName, tsFileName),
+ Arrays.asList(modFileLength, tsFileLength),
+ Collections.singletonMap(
+ generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName));
+ }
+
+ public static byte[] toTPipeTransferBytes(
+ final String tsFileName, final long tsFileLength, final String dataBaseName)
+ throws IOException {
+ return new PipeTransferTsFileSealWithModReq()
+ .convertToTPipeTransferSnapshotSealBytes(
+ Collections.singletonList(tsFileName),
+ Collections.singletonList(tsFileLength),
+ Collections.singletonMap(
+ generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName));
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ return obj instanceof PipeTransferTsFileSealWithModReq && super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/response/PipeTransferFilePieceResp.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/response/PipeTransferFilePieceResp.java
new file mode 100644
index 0000000..d2b6a9e
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/payload/thrift/response/PipeTransferFilePieceResp.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.response;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.util.Objects;
+
+public class PipeTransferFilePieceResp extends TPipeTransferResp {
+
+ private long endWritingOffset;
+
+ private PipeTransferFilePieceResp() {
+ // Empty constructor
+ }
+
+ public long getEndWritingOffset() {
+ return endWritingOffset;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ public static PipeTransferFilePieceResp fromTPipeTransferResp(TPipeTransferResp transferResp) {
+ final PipeTransferFilePieceResp filePieceResp = new PipeTransferFilePieceResp();
+
+ filePieceResp.status = transferResp.status;
+
+ if (transferResp.isSetBody()) {
+ filePieceResp.endWritingOffset = ReadWriteIOUtils.readLong(transferResp.body);
+ filePieceResp.body = transferResp.body;
+ }
+
+ return filePieceResp;
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ PipeTransferFilePieceResp that = (PipeTransferFilePieceResp) obj;
+ return endWritingOffset == that.endWritingOffset
+ && status.equals(that.status)
+ && body.equals(that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(endWritingOffset, status, body);
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBConnector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBConnector.java
new file mode 100644
index 0000000..d358ae8
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBConnector.java
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.protocol;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressor;
+import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressorConfig;
+import org.apache.iotdb.collector.plugin.builtin.sink.compressor.PipeCompressorFactory;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferCompressedReq;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.pipe.api.PipeSink;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeSinkRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
+import org.apache.iotdb.rpc.UrlUtils;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_SET;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_TABLET_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_SIZE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_HOST_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_IP_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_NODE_URLS_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PASSWORD_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PORT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_USERNAME_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_USER_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_IOTDB_USER_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_STRATEGY_SET;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_STRATEGY_SET;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_VALIDATION_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LOAD_TSFILE_VALIDATION_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_MARK_AS_PIPE_REQUEST_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_MARK_AS_PIPE_REQUEST_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_COMPRESSOR_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_COMPRESSOR_ZSTD_LEVEL_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_FORMAT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_MODE_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_SIZE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_HOST_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_IP_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_NODE_URLS_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_PASSWORD_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_PORT_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_USERNAME_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_USER_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LOAD_BALANCE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LOAD_TSFILE_STRATEGY_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LOAD_TSFILE_VALIDATION_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_MARK_AS_PIPE_REQUEST_KEY;
+
+public abstract class IoTDBConnector implements PipeSink {
+
+ private static final String PARSE_URL_ERROR_FORMATTER =
+ "Exception occurred while parsing node urls from target servers: {}";
+ private static final String PARSE_URL_ERROR_MESSAGE =
+ "Error occurred while parsing node urls from target servers, please check the specified 'host':'port' or 'node-urls'";
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConnector.class);
+
+ protected final List<TEndPoint> nodeUrls = new ArrayList<>();
+
+ protected String username = CONNECTOR_IOTDB_USER_DEFAULT_VALUE;
+ protected String password = CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE;
+
+ protected String loadBalanceStrategy;
+
+ protected String loadTsFileStrategy;
+ protected boolean loadTsFileValidation;
+
+ protected boolean shouldMarkAsPipeRequest;
+
+ private boolean isRpcCompressionEnabled;
+ private final List<PipeCompressor> compressors = new ArrayList<>();
+
+ protected boolean isTabletBatchModeEnabled = true;
+
+ protected PipeReceiverStatusHandler receiverStatusHandler;
+ protected boolean shouldReceiverConvertOnTypeMismatch =
+ CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE;
+
+ @Override
+ public void validate(final PipeParameterValidator validator) throws Exception {
+ final PipeParameters parameters = validator.getParameters();
+
+ validator.validate(
+ args ->
+ (boolean) args[0]
+ || (((boolean) args[1] || (boolean) args[2]) && (boolean) args[3])
+ || (boolean) args[4]
+ || (((boolean) args[5] || (boolean) args[6]) && (boolean) args[7]),
+ String.format(
+ "One of %s, %s:%s, %s, %s:%s must be specified",
+ CONNECTOR_IOTDB_NODE_URLS_KEY,
+ CONNECTOR_IOTDB_HOST_KEY,
+ CONNECTOR_IOTDB_PORT_KEY,
+ SINK_IOTDB_NODE_URLS_KEY,
+ SINK_IOTDB_HOST_KEY,
+ SINK_IOTDB_PORT_KEY),
+ parameters.hasAttribute(CONNECTOR_IOTDB_NODE_URLS_KEY),
+ parameters.hasAttribute(CONNECTOR_IOTDB_IP_KEY),
+ parameters.hasAttribute(CONNECTOR_IOTDB_HOST_KEY),
+ parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY),
+ parameters.hasAttribute(SINK_IOTDB_NODE_URLS_KEY),
+ parameters.hasAttribute(SINK_IOTDB_IP_KEY),
+ parameters.hasAttribute(SINK_IOTDB_HOST_KEY),
+ parameters.hasAttribute(SINK_IOTDB_PORT_KEY));
+
+ validator.validate(
+ requestMaxBatchSizeInBytes -> (long) requestMaxBatchSizeInBytes > 0,
+ String.format(
+ "%s must be > 0, but got %s",
+ SINK_IOTDB_BATCH_SIZE_KEY,
+ parameters.getLongOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY),
+ CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE)),
+ parameters.getLongOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY, SINK_IOTDB_BATCH_SIZE_KEY),
+ CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE));
+
+ // Check coexistence of user and username
+ validator.validateSynonymAttributes(
+ Arrays.asList(CONNECTOR_IOTDB_USER_KEY, SINK_IOTDB_USER_KEY),
+ Arrays.asList(CONNECTOR_IOTDB_USERNAME_KEY, SINK_IOTDB_USERNAME_KEY),
+ false);
+
+ username =
+ parameters.getStringOrDefault(
+ Arrays.asList(
+ CONNECTOR_IOTDB_USER_KEY,
+ SINK_IOTDB_USER_KEY,
+ CONNECTOR_IOTDB_USERNAME_KEY,
+ SINK_IOTDB_USERNAME_KEY),
+ CONNECTOR_IOTDB_USER_DEFAULT_VALUE);
+ password =
+ parameters.getStringOrDefault(
+ Arrays.asList(CONNECTOR_IOTDB_PASSWORD_KEY, SINK_IOTDB_PASSWORD_KEY),
+ CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE);
+
+ loadBalanceStrategy =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(CONNECTOR_LOAD_BALANCE_STRATEGY_KEY, SINK_LOAD_BALANCE_STRATEGY_KEY),
+ CONNECTOR_LOAD_BALANCE_ROUND_ROBIN_STRATEGY)
+ .trim()
+ .toLowerCase();
+ validator.validate(
+ arg -> CONNECTOR_LOAD_BALANCE_STRATEGY_SET.contains(loadBalanceStrategy),
+ String.format(
+ "Load balance strategy should be one of %s, but got %s.",
+ CONNECTOR_LOAD_BALANCE_STRATEGY_SET, loadBalanceStrategy),
+ loadBalanceStrategy);
+
+ loadTsFileStrategy =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(CONNECTOR_LOAD_TSFILE_STRATEGY_KEY, SINK_LOAD_TSFILE_STRATEGY_KEY),
+ CONNECTOR_LOAD_TSFILE_STRATEGY_SYNC_VALUE)
+ .trim()
+ .toLowerCase();
+ validator.validate(
+ arg -> CONNECTOR_LOAD_TSFILE_STRATEGY_SET.contains(loadTsFileStrategy),
+ String.format(
+ "Load tsfile strategy should be one of %s, but got %s.",
+ CONNECTOR_LOAD_TSFILE_STRATEGY_SET, loadTsFileStrategy),
+ loadTsFileStrategy);
+ loadTsFileValidation =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(CONNECTOR_LOAD_TSFILE_VALIDATION_KEY, SINK_LOAD_TSFILE_VALIDATION_KEY),
+ CONNECTOR_LOAD_TSFILE_VALIDATION_DEFAULT_VALUE);
+
+ final int zstdCompressionLevel =
+ parameters.getIntOrDefault(
+ Arrays.asList(CONNECTOR_COMPRESSOR_ZSTD_LEVEL_KEY, SINK_COMPRESSOR_ZSTD_LEVEL_KEY),
+ CONNECTOR_COMPRESSOR_ZSTD_LEVEL_DEFAULT_VALUE);
+ validator.validate(
+ arg ->
+ (int) arg >= CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE
+ && (int) arg <= CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE,
+ String.format(
+ "Zstd compression level should be in the range [%d, %d], but got %d.",
+ CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE,
+ CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE,
+ zstdCompressionLevel),
+ zstdCompressionLevel);
+
+ final String compressionTypes =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(CONNECTOR_COMPRESSOR_KEY, SINK_COMPRESSOR_KEY),
+ CONNECTOR_COMPRESSOR_DEFAULT_VALUE)
+ .toLowerCase();
+ if (!compressionTypes.isEmpty()) {
+ for (final String compressionType : compressionTypes.split(",")) {
+ final String trimmedCompressionType = compressionType.trim();
+ if (trimmedCompressionType.isEmpty()) {
+ continue;
+ }
+
+ validator.validate(
+ arg -> CONNECTOR_COMPRESSOR_SET.contains(trimmedCompressionType),
+ String.format(
+ "Compressor should be one of %s, but got %s.",
+ CONNECTOR_COMPRESSOR_SET, trimmedCompressionType),
+ trimmedCompressionType);
+ compressors.add(
+ PipeCompressorFactory.getCompressor(
+ new PipeCompressorConfig(trimmedCompressionType, zstdCompressionLevel)));
+ }
+ }
+ validator.validate(
+ arg -> compressors.size() <= Byte.MAX_VALUE,
+ String.format(
+ "The number of compressors should be less than or equal to %d, but got %d.",
+ Byte.MAX_VALUE, compressors.size()),
+ compressors.size());
+ isRpcCompressionEnabled = !compressors.isEmpty();
+
+ validator.validate(
+ arg -> arg.equals("retry") || arg.equals("ignore"),
+ String.format(
+ "The value of key %s or %s must be either 'retry' or 'ignore'.",
+ CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY,
+ SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY),
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY,
+ SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY),
+ CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE)
+ .trim()
+ .toLowerCase());
+
+ validator.validateAttributeValueRange(
+ validator.getParameters().hasAttribute(CONNECTOR_FORMAT_KEY)
+ ? CONNECTOR_FORMAT_KEY
+ : SINK_FORMAT_KEY,
+ true,
+ CONNECTOR_FORMAT_TABLET_VALUE,
+ CONNECTOR_FORMAT_HYBRID_VALUE,
+ CONNECTOR_FORMAT_TS_FILE_VALUE);
+ }
+
+ @Override
+ public void customize(
+ final PipeParameters parameters, final PipeSinkRuntimeConfiguration configuration)
+ throws Exception {
+ nodeUrls.clear();
+ nodeUrls.addAll(parseNodeUrls(parameters));
+ LOGGER.info("IoTDBConnector nodeUrls: {}", nodeUrls);
+
+ isTabletBatchModeEnabled =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(
+ CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY, SINK_IOTDB_BATCH_MODE_ENABLE_KEY),
+ CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE)
+ || parameters
+ .getStringOrDefault(
+ Arrays.asList(CONNECTOR_FORMAT_KEY, SINK_FORMAT_KEY),
+ CONNECTOR_FORMAT_HYBRID_VALUE)
+ .equals(CONNECTOR_FORMAT_TS_FILE_VALUE);
+ LOGGER.info("IoTDBConnector isTabletBatchModeEnabled: {}", isTabletBatchModeEnabled);
+
+ shouldMarkAsPipeRequest =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(CONNECTOR_MARK_AS_PIPE_REQUEST_KEY, SINK_MARK_AS_PIPE_REQUEST_KEY),
+ CONNECTOR_MARK_AS_PIPE_REQUEST_DEFAULT_VALUE);
+ LOGGER.info("IoTDBConnector shouldMarkAsPipeRequest: {}", shouldMarkAsPipeRequest);
+
+ receiverStatusHandler =
+ new PipeReceiverStatusHandler(
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY,
+ SINK_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_KEY),
+ CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE)
+ .trim()
+ .equalsIgnoreCase("retry"),
+ parameters.getLongOrDefault(
+ Arrays.asList(
+ CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_KEY,
+ SINK_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_KEY),
+ CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE),
+ parameters.getBooleanOrDefault(
+ Arrays.asList(
+ CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_KEY,
+ SINK_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_KEY),
+ CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_DEFAULT_VALUE),
+ parameters.getLongOrDefault(
+ Arrays.asList(
+ CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_KEY,
+ SINK_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_KEY),
+ CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE),
+ parameters.getBooleanOrDefault(
+ Arrays.asList(
+ CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_KEY,
+ SINK_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_KEY),
+ CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_DEFAULT_VALUE));
+
+ shouldReceiverConvertOnTypeMismatch =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(
+ CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY,
+ SINK_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY),
+ CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_DEFAULT_VALUE);
+ LOGGER.info(
+ "IoTDBConnector {} = {}",
+ CONNECTOR_EXCEPTION_DATA_CONVERT_ON_TYPE_MISMATCH_KEY,
+ shouldReceiverConvertOnTypeMismatch);
+ }
+
+ protected LinkedHashSet<TEndPoint> parseNodeUrls(final PipeParameters parameters)
+ throws PipeParameterNotValidException {
+ final LinkedHashSet<TEndPoint> givenNodeUrls = new LinkedHashSet<>(nodeUrls);
+
+ try {
+ if (parameters.hasAttribute(CONNECTOR_IOTDB_IP_KEY)
+ && parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY)) {
+ givenNodeUrls.add(
+ new TEndPoint(
+ parameters.getStringByKeys(CONNECTOR_IOTDB_IP_KEY),
+ parameters.getIntByKeys(CONNECTOR_IOTDB_PORT_KEY)));
+ }
+
+ if (parameters.hasAttribute(SINK_IOTDB_IP_KEY)
+ && parameters.hasAttribute(SINK_IOTDB_PORT_KEY)) {
+ givenNodeUrls.add(
+ new TEndPoint(
+ parameters.getStringByKeys(SINK_IOTDB_IP_KEY),
+ parameters.getIntByKeys(SINK_IOTDB_PORT_KEY)));
+ }
+
+ if (parameters.hasAttribute(CONNECTOR_IOTDB_HOST_KEY)
+ && parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY)) {
+ givenNodeUrls.add(
+ new TEndPoint(
+ parameters.getStringByKeys(CONNECTOR_IOTDB_HOST_KEY),
+ parameters.getIntByKeys(CONNECTOR_IOTDB_PORT_KEY)));
+ }
+
+ if (parameters.hasAttribute(SINK_IOTDB_HOST_KEY)
+ && parameters.hasAttribute(SINK_IOTDB_PORT_KEY)) {
+ givenNodeUrls.add(
+ new TEndPoint(
+ parameters.getStringByKeys(SINK_IOTDB_HOST_KEY),
+ parameters.getIntByKeys(SINK_IOTDB_PORT_KEY)));
+ }
+
+ if (parameters.hasAttribute(CONNECTOR_IOTDB_NODE_URLS_KEY)) {
+ givenNodeUrls.addAll(
+ parseTEndPointUrls(
+ Arrays.asList(
+ parameters
+ .getStringByKeys(CONNECTOR_IOTDB_NODE_URLS_KEY)
+ .replace(" ", "")
+ .split(","))));
+ }
+
+ if (parameters.hasAttribute(SINK_IOTDB_NODE_URLS_KEY)) {
+ givenNodeUrls.addAll(
+ parseTEndPointUrls(
+ Arrays.asList(
+ parameters
+ .getStringByKeys(SINK_IOTDB_NODE_URLS_KEY)
+ .replace(" ", "")
+ .split(","))));
+ }
+ } catch (final Exception e) {
+ LOGGER.warn(PARSE_URL_ERROR_FORMATTER, e.toString());
+ throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE);
+ }
+
+ checkNodeUrls(givenNodeUrls);
+
+ return givenNodeUrls;
+ }
+
+ private static List<TEndPoint> parseTEndPointUrls(List<String> endPointUrls) {
+ if (endPointUrls == null) {
+ throw new NumberFormatException("endPointUrls is null");
+ }
+ List<TEndPoint> result = new ArrayList<>();
+ for (String url : endPointUrls) {
+ result.add(UrlUtils.parseTEndPointIpv4AndIpv6Url(url));
+ }
+ return result;
+ }
+
+ private static void checkNodeUrls(final Set<TEndPoint> nodeUrls)
+ throws PipeParameterNotValidException {
+ for (final TEndPoint nodeUrl : nodeUrls) {
+ if (Objects.isNull(nodeUrl.ip) || nodeUrl.ip.isEmpty()) {
+ LOGGER.warn(PARSE_URL_ERROR_FORMATTER, "host cannot be empty");
+ throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE);
+ }
+ if (nodeUrl.port == 0) {
+ LOGGER.warn(PARSE_URL_ERROR_FORMATTER, "port cannot be empty");
+ throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE);
+ }
+ }
+ }
+
+ protected TPipeTransferReq compressIfNeeded(final TPipeTransferReq req) throws IOException {
+ return isRpcCompressionEnabled
+ ? PipeTransferCompressedReq.toTPipeTransferReq(req, compressors)
+ : req;
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBDataRegionSyncConnector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBDataRegionSyncConnector.java
new file mode 100644
index 0000000..cfd7457
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBDataRegionSyncConnector.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.protocol;
+
+import org.apache.iotdb.collector.plugin.builtin.sink.client.IoTDBDataNodeSyncClientManager;
+import org.apache.iotdb.collector.plugin.builtin.sink.client.IoTDBSyncClient;
+import org.apache.iotdb.collector.plugin.builtin.sink.client.IoTDBSyncClientManager;
+import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.collector.plugin.builtin.sink.event.PipeTsFileInsertionEvent;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch.PipeTabletEventBatch;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch.PipeTabletEventPlainBatch;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch.PipeTabletEventTsFileBatch;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.evolvable.batch.PipeTransferBatchReqBuilder;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFilePieceReq;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferTabletRawReqV2;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferTsFilePieceReq;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferTsFilePieceWithModReq;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferTsFileSealWithModReq;
+import org.apache.iotdb.collector.utils.cacher.LeaderCacheUtils;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeSinkRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
+import org.apache.iotdb.pipe.api.exception.PipeConnectionException;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.NoSuchFileException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class IoTDBDataRegionSyncConnector extends IoTDBSslSyncConnector {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDataRegionSyncConnector.class);
+
+ private PipeTransferBatchReqBuilder tabletBatchBuilder;
+
+ protected IoTDBDataNodeSyncClientManager clientManager;
+
+ @Override
+ protected IoTDBSyncClientManager constructClient(
+ final List<TEndPoint> nodeUrls,
+ final boolean useSSL,
+ final String trustStorePath,
+ final String trustStorePwd,
+ /* The following parameters are used locally. */
+ final boolean useLeaderCache,
+ final String loadBalanceStrategy,
+ /* The following parameters are used to handshake with the receiver. */
+ final String username,
+ final String password,
+ final boolean shouldReceiverConvertOnTypeMismatch,
+ final String loadTsFileStrategy,
+ final boolean validateTsFile,
+ final boolean shouldMarkAsPipeRequest) {
+ clientManager =
+ new IoTDBDataNodeSyncClientManager(
+ nodeUrls,
+ useSSL,
+ Objects.nonNull(trustStorePath)
+ ? /*IoTDBConfig.addDataHomeDir(trustStorePath)*/ ""
+ : null,
+ trustStorePwd,
+ useLeaderCache,
+ loadBalanceStrategy,
+ username,
+ password,
+ shouldReceiverConvertOnTypeMismatch,
+ loadTsFileStrategy,
+ validateTsFile,
+ shouldMarkAsPipeRequest);
+ return clientManager;
+ }
+
+ @Override
+ public void customize(
+ final PipeParameters parameters, final PipeSinkRuntimeConfiguration configuration)
+ throws Exception {
+ super.customize(parameters, configuration);
+
+ // tablet batch mode configuration
+ if (isTabletBatchModeEnabled) {
+ tabletBatchBuilder = new PipeTransferBatchReqBuilder(parameters);
+ }
+ }
+
+ @Override
+ protected PipeTransferFilePieceReq getTransferSingleFilePieceReq(
+ final String fileName, final long position, final byte[] payLoad) throws IOException {
+ return PipeTransferTsFilePieceReq.toTPipeTransferReq(fileName, position, payLoad);
+ }
+
+ @Override
+ protected PipeTransferFilePieceReq getTransferMultiFilePieceReq(
+ final String fileName, final long position, final byte[] payLoad) throws IOException {
+ return PipeTransferTsFilePieceWithModReq.toTPipeTransferReq(fileName, position, payLoad);
+ }
+
+ @Override
+ public void customize(
+ PipeParameters pipeParameters, PipeConnectorRuntimeConfiguration pipeSinkRuntimeConfiguration)
+ throws Exception {}
+
+ @Override
+ public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exception {
+ if (isTabletBatchModeEnabled) {
+ final Pair<TEndPoint, PipeTabletEventBatch> endPointAndBatch =
+ tabletBatchBuilder.onEvent(tabletInsertionEvent);
+ if (Objects.nonNull(endPointAndBatch)) {
+ doTransferWrapper(endPointAndBatch);
+ }
+ } else {
+ doTransferWrapper((PipeRawTabletInsertionEvent) tabletInsertionEvent);
+ }
+ }
+
+ @Override
+ public void transfer(final TsFileInsertionEvent tsFileInsertionEvent) throws Exception {
+ // In order to commit in order
+ if (isTabletBatchModeEnabled && !tabletBatchBuilder.isEmpty()) {
+ doTransferWrapper();
+ }
+
+ doTransferWrapper((PipeTsFileInsertionEvent) tsFileInsertionEvent);
+ }
+
+ @Override
+ public void transfer(final Event event) throws Exception {
+ if (isTabletBatchModeEnabled && !tabletBatchBuilder.isEmpty()) {
+ doTransferWrapper();
+ }
+ }
+
+ private void doTransferWrapper() throws IOException, WriteProcessException {
+ for (final Pair<TEndPoint, PipeTabletEventBatch> nonEmptyBatch :
+ tabletBatchBuilder.getAllNonEmptyBatches()) {
+ doTransferWrapper(nonEmptyBatch);
+ }
+ }
+
+ private void doTransferWrapper(final Pair<TEndPoint, PipeTabletEventBatch> endPointAndBatch)
+ throws IOException, WriteProcessException {
+ final PipeTabletEventBatch batch = endPointAndBatch.getRight();
+ if (batch instanceof PipeTabletEventPlainBatch) {
+ doTransfer(endPointAndBatch.getLeft(), (PipeTabletEventPlainBatch) batch);
+ } else if (batch instanceof PipeTabletEventTsFileBatch) {
+ doTransfer((PipeTabletEventTsFileBatch) batch);
+ } else {
+ LOGGER.warn("Unsupported batch type {}.", batch.getClass());
+ }
+ batch.onSuccess();
+ }
+
+ private void doTransfer(
+ final TEndPoint endPoint, final PipeTabletEventPlainBatch batchToTransfer) {
+ final Pair<IoTDBSyncClient, Boolean> clientAndStatus = clientManager.getClient(endPoint);
+
+ final TPipeTransferResp resp;
+ try {
+ final TPipeTransferReq uncompressedReq = batchToTransfer.toTPipeTransferReq();
+ final TPipeTransferReq req = compressIfNeeded(uncompressedReq);
+
+ resp = clientAndStatus.getLeft().pipeTransfer(req);
+ } catch (final Exception e) {
+ clientAndStatus.setRight(false);
+ throw new PipeConnectionException(
+ String.format("Network error when transfer tablet batch, because %s.", e.getMessage()),
+ e);
+ }
+
+ final TSStatus status = resp.getStatus();
+ // Only handle the failed statuses to avoid string format performance overhead
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) {
+ receiverStatusHandler.handle(
+ resp.getStatus(),
+ String.format("Transfer PipeTransferTabletBatchReq error, result status %s", resp.status),
+ batchToTransfer.deepCopyEvents().toString());
+ }
+
+ for (final Pair<String, TEndPoint> redirectPair :
+ LeaderCacheUtils.parseRecommendedRedirections(status)) {
+ clientManager.updateLeaderCache(redirectPair.getLeft(), redirectPair.getRight());
+ }
+ }
+
+ private void doTransfer(final PipeTabletEventTsFileBatch batchToTransfer)
+ throws IOException, WriteProcessException {
+ final List<Pair<String, File>> dbTsFilePairs = batchToTransfer.sealTsFiles();
+ final Map<Pair<String, Long>, Double> pipe2WeightMap = batchToTransfer.deepCopyPipe2WeightMap();
+
+ for (final Pair<String, File> dbTsFile : dbTsFilePairs) {
+ doTransfer(dbTsFile.right, null, dbTsFile.left);
+ try {
+ FileUtils.delete(dbTsFile.right);
+ } catch (final NoSuchFileException e) {
+ LOGGER.info("The file {} is not found, may already be deleted.", dbTsFile);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to delete batch file {}, this file should be deleted manually later", dbTsFile);
+ }
+ }
+ }
+
+ private void doTransferWrapper(final PipeRawTabletInsertionEvent pipeRawTabletInsertionEvent)
+ throws PipeException {
+ doTransfer(pipeRawTabletInsertionEvent);
+ }
+
+ private void doTransfer(final PipeRawTabletInsertionEvent pipeRawTabletInsertionEvent)
+ throws PipeException {
+ final Pair<IoTDBSyncClient, Boolean> clientAndStatus =
+ clientManager.getClient(pipeRawTabletInsertionEvent.getDeviceId());
+ final TPipeTransferResp resp;
+
+ try {
+ final TPipeTransferReq req =
+ compressIfNeeded(
+ PipeTransferTabletRawReqV2.toTPipeTransferReq(
+ pipeRawTabletInsertionEvent.getTablet(),
+ pipeRawTabletInsertionEvent.isAligned(),
+ pipeRawTabletInsertionEvent.isTableModelEvent()
+ ? pipeRawTabletInsertionEvent.getTableModelDatabaseName()
+ : null));
+ resp = clientAndStatus.getLeft().pipeTransfer(req);
+ } catch (final Exception e) {
+ clientAndStatus.setRight(false);
+ throw new PipeConnectionException(
+ String.format(
+ "Network error when transfer raw tablet insertion event, because %s.",
+ e.getMessage()),
+ e);
+ }
+
+ final TSStatus status = resp.getStatus();
+ // Only handle the failed statuses to avoid string format performance overhead
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) {
+ receiverStatusHandler.handle(
+ status,
+ String.format(
+ "Transfer PipeRawTabletInsertionEvent %s error, result status %s",
+ pipeRawTabletInsertionEvent, status),
+ pipeRawTabletInsertionEvent.toString());
+ }
+ if (status.isSetRedirectNode()) {
+ clientManager.updateLeaderCache(
+ pipeRawTabletInsertionEvent.getDeviceId(), status.getRedirectNode());
+ }
+ }
+
+ private void doTransferWrapper(final PipeTsFileInsertionEvent pipeTsFileInsertionEvent)
+ throws PipeException, IOException {
+ doTransfer(
+ pipeTsFileInsertionEvent.getTsFile(),
+ null,
+ pipeTsFileInsertionEvent.isTableModelEvent()
+ ? pipeTsFileInsertionEvent.getTableModelDatabaseName()
+ : null);
+ }
+
+ private void doTransfer(final File tsFile, final File modFile, final String dataBaseName)
+ throws PipeException, IOException {
+
+ final Pair<IoTDBSyncClient, Boolean> clientAndStatus = clientManager.getClient();
+ final TPipeTransferResp resp;
+
+ // 1. Transfer tsFile, and mod file if exists and receiver's version >= 2
+ if (Objects.nonNull(modFile) && clientManager.supportModsIfIsDataNodeReceiver()) {
+ transferFilePieces(modFile, clientAndStatus, true);
+ transferFilePieces(tsFile, clientAndStatus, true);
+
+ // 2. Transfer file seal signal with mod, which means the file is transferred completely
+ try {
+ final TPipeTransferReq req =
+ compressIfNeeded(
+ PipeTransferTsFileSealWithModReq.toTPipeTransferReq(
+ modFile.getName(),
+ modFile.length(),
+ tsFile.getName(),
+ tsFile.length(),
+ dataBaseName));
+
+ resp = clientAndStatus.getLeft().pipeTransfer(req);
+ } catch (final Exception e) {
+ clientAndStatus.setRight(false);
+ clientManager.adjustTimeoutIfNecessary(e);
+ throw new PipeConnectionException(
+ String.format("Network error when seal file %s, because %s.", tsFile, e.getMessage()),
+ e);
+ }
+ } else {
+ transferFilePieces(tsFile, clientAndStatus, false);
+
+ // 2. Transfer file seal signal without mod, which means the file is transferred completely
+ try {
+ final TPipeTransferReq req =
+ compressIfNeeded(
+ PipeTransferTsFileSealWithModReq.toTPipeTransferReq(
+ tsFile.getName(), tsFile.length(), dataBaseName));
+
+ resp = clientAndStatus.getLeft().pipeTransfer(req);
+ } catch (final Exception e) {
+ clientAndStatus.setRight(false);
+ clientManager.adjustTimeoutIfNecessary(e);
+ throw new PipeConnectionException(
+ String.format("Network error when seal file %s, because %s.", tsFile, e.getMessage()),
+ e);
+ }
+ }
+
+ final TSStatus status = resp.getStatus();
+ // Only handle the failed statuses to avoid string format performance overhead
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) {
+ receiverStatusHandler.handle(
+ resp.getStatus(),
+ String.format("Seal file %s error, result status %s.", tsFile, resp.getStatus()),
+ tsFile.getName());
+ }
+
+ LOGGER.info("Successfully transferred file {}.", tsFile);
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (tabletBatchBuilder != null) {
+ tabletBatchBuilder.close();
+ }
+
+ super.close();
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBSslSyncConnector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBSslSyncConnector.java
new file mode 100644
index 0000000..6c616e2
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/IoTDBSslSyncConnector.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.protocol;
+
+import org.apache.iotdb.collector.config.PipeRuntimeOptions;
+import org.apache.iotdb.collector.plugin.builtin.sink.client.IoTDBSyncClient;
+import org.apache.iotdb.collector.plugin.builtin.sink.client.IoTDBSyncClientManager;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.request.PipeTransferFilePieceReq;
+import org.apache.iotdb.collector.plugin.builtin.sink.payload.thrift.response.PipeTransferFilePieceResp;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeSinkRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.exception.PipeConnectionException;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_SSL_ENABLE_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_KEY;
+import static org.apache.iotdb.collector.plugin.builtin.sink.constant.PipeConnectorConstant.SINK_LEADER_CACHE_ENABLE_KEY;
+
+public abstract class IoTDBSslSyncConnector extends IoTDBConnector {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSslSyncConnector.class);
+
+ protected IoTDBSyncClientManager clientManager;
+
+ @Override
+ public void validate(final PipeParameterValidator validator) throws Exception {
+ super.validate(validator);
+
+ final PipeParameters parameters = validator.getParameters();
+
+ final String userSpecifiedConnectorName =
+ parameters
+ .getStringOrDefault(
+ ImmutableList.of(CONNECTOR_KEY, SINK_KEY), ""
+ /*IOTDB_THRIFT_CONNECTOR.getPipePluginName()*/ )
+ .toLowerCase();
+
+ validator.validate(
+ args -> !((boolean) args[0]) || ((boolean) args[1] && (boolean) args[2]),
+ String.format(
+ "When ssl transport is enabled, %s and %s must be specified",
+ SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY, SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY),
+ parameters.getBooleanOrDefault(SINK_IOTDB_SSL_ENABLE_KEY, false),
+ parameters.hasAttribute(SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY),
+ parameters.hasAttribute(SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY));
+ }
+
+ @Override
+ public void customize(
+ final PipeParameters parameters, final PipeSinkRuntimeConfiguration configuration)
+ throws Exception {
+ super.customize(parameters, configuration);
+
+ final String trustStorePath = parameters.getString(SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY);
+ final String trustStorePwd = parameters.getString(SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY);
+
+ // leader cache configuration
+ final boolean useLeaderCache =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(SINK_LEADER_CACHE_ENABLE_KEY, CONNECTOR_LEADER_CACHE_ENABLE_KEY),
+ CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE);
+
+ clientManager =
+ constructClient(
+ nodeUrls,
+ false,
+ trustStorePath,
+ trustStorePwd,
+ useLeaderCache,
+ loadBalanceStrategy,
+ username,
+ password,
+ shouldReceiverConvertOnTypeMismatch,
+ loadTsFileStrategy,
+ loadTsFileValidation,
+ shouldMarkAsPipeRequest);
+ }
+
+ protected abstract IoTDBSyncClientManager constructClient(
+ final List<TEndPoint> nodeUrls,
+ final boolean useSSL,
+ final String trustStorePath,
+ final String trustStorePwd,
+ /* The following parameters are used locally. */
+ final boolean useLeaderCache,
+ final String loadBalanceStrategy,
+ /* The following parameters are used to handshake with the receiver. */
+ final String username,
+ final String password,
+ final boolean shouldReceiverConvertOnTypeMismatch,
+ final String loadTsFileStrategy,
+ final boolean validateTsFile,
+ final boolean shouldMarkAsPipeRequest);
+
+ @Override
+ public void handshake() throws Exception {
+ clientManager.checkClientStatusAndTryReconstructIfNecessary();
+ }
+
+ @Override
+ public void heartbeat() {
+ try {
+ handshake();
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to reconnect to target server, because: {}. Try to reconnect later.",
+ e.getMessage(),
+ e);
+ }
+ }
+
+ protected void transferFilePieces(
+ final File file,
+ final Pair<IoTDBSyncClient, Boolean> clientAndStatus,
+ final boolean isMultiFile)
+ throws PipeException, IOException {
+ final int readFileBufferSize = PipeRuntimeOptions.PIPE_CONNECTOR_READ_FILE_BUFFER_SIZE.value();
+ final byte[] readBuffer = new byte[readFileBufferSize];
+ long position = 0;
+ try (final RandomAccessFile reader = new RandomAccessFile(file, "r")) {
+ while (true) {
+ final int readLength = reader.read(readBuffer);
+ if (readLength == -1) {
+ break;
+ }
+
+ final byte[] payLoad =
+ readLength == readFileBufferSize
+ ? readBuffer
+ : Arrays.copyOfRange(readBuffer, 0, readLength);
+ final PipeTransferFilePieceResp resp;
+ try {
+ final TPipeTransferReq req =
+ compressIfNeeded(
+ isMultiFile
+ ? getTransferMultiFilePieceReq(file.getName(), position, payLoad)
+ : getTransferSingleFilePieceReq(file.getName(), position, payLoad));
+
+ resp =
+ PipeTransferFilePieceResp.fromTPipeTransferResp(
+ clientAndStatus.getLeft().pipeTransfer(req));
+ } catch (final Exception e) {
+ clientAndStatus.setRight(false);
+ throw new PipeConnectionException(
+ String.format(
+ "Network error when transfer file %s, because %s.", file, e.getMessage()),
+ e);
+ }
+
+ position += readLength;
+
+ final TSStatus status = resp.getStatus();
+ // This case only happens when the connection is broken, and the connector is reconnected
+ // to the receiver, then the receiver will redirect the file position to the last position
+ if (status.getCode() == TSStatusCode.PIPE_TRANSFER_FILE_OFFSET_RESET.getStatusCode()) {
+ position = resp.getEndWritingOffset();
+ reader.seek(position);
+ LOGGER.info("Redirect file position to {}.", position);
+ continue;
+ }
+
+ // Send handshake req and then re-transfer the event
+ if (status.getCode()
+ == TSStatusCode.PIPE_CONFIG_RECEIVER_HANDSHAKE_NEEDED.getStatusCode()) {
+ clientManager.sendHandshakeReq(clientAndStatus);
+ }
+ // Only handle the failed statuses to avoid string format performance overhead
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) {
+ receiverStatusHandler.handle(
+ resp.getStatus(),
+ String.format("Transfer file %s error, result status %s.", file, resp.getStatus()),
+ file.getName());
+ }
+ }
+ }
+ }
+
+ protected abstract PipeTransferFilePieceReq getTransferSingleFilePieceReq(
+ final String fileName, final long position, final byte[] payLoad) throws IOException;
+
+ protected abstract PipeTransferFilePieceReq getTransferMultiFilePieceReq(
+ final String fileName, final long position, final byte[] payLoad) throws IOException;
+
+ @Override
+ public void close() throws Exception {
+ if (clientManager != null) {
+ clientManager.close();
+ }
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/PipeReceiverStatusHandler.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/PipeReceiverStatusHandler.java
new file mode 100644
index 0000000..013d864
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/PipeReceiverStatusHandler.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.protocol;
+
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class PipeReceiverStatusHandler {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeReceiverStatusHandler.class);
+
+ private static final int CONFLICT_RETRY_MAX_TIMES = 100;
+
+ private final boolean isRetryAllowedWhenConflictOccurs;
+ private final long retryMaxMillisWhenConflictOccurs;
+ private final boolean shouldRecordIgnoredDataWhenConflictOccurs;
+
+ private final long retryMaxMillisWhenOtherExceptionsOccur;
+ private final boolean shouldRecordIgnoredDataWhenOtherExceptionsOccur;
+
+ private final AtomicLong exceptionFirstEncounteredTime = new AtomicLong(0);
+ private final AtomicBoolean exceptionEventHasBeenRetried = new AtomicBoolean(false);
+ private final AtomicReference<String> exceptionRecordedMessage = new AtomicReference<>("");
+
+ public PipeReceiverStatusHandler(
+ final boolean isRetryAllowedWhenConflictOccurs,
+ final long retryMaxSecondsWhenConflictOccurs,
+ final boolean shouldRecordIgnoredDataWhenConflictOccurs,
+ final long retryMaxSecondsWhenOtherExceptionsOccur,
+ final boolean shouldRecordIgnoredDataWhenOtherExceptionsOccur) {
+ this.isRetryAllowedWhenConflictOccurs = isRetryAllowedWhenConflictOccurs;
+ this.retryMaxMillisWhenConflictOccurs =
+ retryMaxSecondsWhenConflictOccurs < 0
+ ? Long.MAX_VALUE
+ : retryMaxSecondsWhenConflictOccurs * 1000;
+ this.shouldRecordIgnoredDataWhenConflictOccurs = shouldRecordIgnoredDataWhenConflictOccurs;
+
+ this.retryMaxMillisWhenOtherExceptionsOccur =
+ retryMaxSecondsWhenOtherExceptionsOccur < 0
+ ? Long.MAX_VALUE
+ : retryMaxSecondsWhenOtherExceptionsOccur * 1000;
+ this.shouldRecordIgnoredDataWhenOtherExceptionsOccur =
+ shouldRecordIgnoredDataWhenOtherExceptionsOccur;
+ }
+
+ /**
+ * Handle {@link TSStatus} returned by receiver. Do nothing if ignore the {@link Event}, and throw
+ * exception if retry the {@link Event}. Upper class must ensure that the method is invoked only
+ * by a single thread.
+ *
+ * @throws PipeException to retry the current {@link Event}
+ * @param status the {@link TSStatus} to judge
+ * @param exceptionMessage The exception message to throw
+ * @param recordMessage The message to record an ignored {@link Event}, the caller should assure
+ * that the same {@link Event} generates always the same record message, for instance, do not
+ * put any time-related info here
+ */
+ public void handle(
+ final TSStatus status, final String exceptionMessage, final String recordMessage) {
+ switch (status.getCode()) {
+ case 200: // SUCCESS_STATUS
+ case 400: // REDIRECTION_RECOMMEND
+ {
+ return;
+ }
+
+ case 1809: // PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION
+ {
+ LOGGER.info("Idempotent conflict exception: will be ignored. status: {}", status);
+ return;
+ }
+
+ case 1808: // PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION
+ {
+ LOGGER.info("Temporary unavailable exception: will retry forever. status: {}", status);
+ throw new PipeException(exceptionMessage);
+ }
+
+ case 1810: // PIPE_RECEIVER_USER_CONFLICT_EXCEPTION
+ if (!isRetryAllowedWhenConflictOccurs) {
+ LOGGER.warn(
+ "User conflict exception: will be ignored because retry is not allowed. event: {}. status: {}",
+ shouldRecordIgnoredDataWhenConflictOccurs ? recordMessage : "not recorded",
+ status);
+ return;
+ }
+
+ synchronized (this) {
+ recordExceptionStatusIfNecessary(recordMessage);
+
+ if (exceptionEventHasBeenRetried.get()
+ && System.currentTimeMillis() - exceptionFirstEncounteredTime.get()
+ > retryMaxMillisWhenConflictOccurs) {
+ LOGGER.warn(
+ "User conflict exception: retry timeout. will be ignored. event: {}. status: {}",
+ shouldRecordIgnoredDataWhenConflictOccurs ? recordMessage : "not recorded",
+ status);
+ resetExceptionStatus();
+ return;
+ }
+
+ LOGGER.warn(
+ "User conflict exception: will retry {}. status: {}",
+ retryMaxMillisWhenConflictOccurs == Long.MAX_VALUE
+ ? "forever"
+ : "for at least "
+ + (retryMaxMillisWhenConflictOccurs
+ + exceptionFirstEncounteredTime.get()
+ - System.currentTimeMillis())
+ / 1000.0
+ + " seconds",
+ status);
+ exceptionEventHasBeenRetried.set(true);
+ throw new PipeException(
+ exceptionMessage,
+ (int)
+ Math.max(
+ 5,
+ Math.min(CONFLICT_RETRY_MAX_TIMES, retryMaxMillisWhenConflictOccurs * 1.1)));
+ }
+
+ default: // Other exceptions
+ synchronized (this) {
+ recordExceptionStatusIfNecessary(recordMessage);
+
+ if (exceptionEventHasBeenRetried.get()
+ && System.currentTimeMillis() - exceptionFirstEncounteredTime.get()
+ > retryMaxMillisWhenOtherExceptionsOccur) {
+ LOGGER.warn(
+ "Unclassified exception: retry timeout. will be ignored. event: {}. status: {}",
+ shouldRecordIgnoredDataWhenOtherExceptionsOccur ? recordMessage : "not recorded",
+ status);
+ resetExceptionStatus();
+ return;
+ }
+
+ LOGGER.warn(
+ "Unclassified exception: will retry {}. status: {}",
+ retryMaxMillisWhenOtherExceptionsOccur == Long.MAX_VALUE
+ ? "forever"
+ : "for at least "
+ + (retryMaxMillisWhenOtherExceptionsOccur
+ + exceptionFirstEncounteredTime.get()
+ - System.currentTimeMillis())
+ / 1000.0
+ + " seconds",
+ status);
+ exceptionEventHasBeenRetried.set(true);
+ throw new PipeException(
+ exceptionMessage,
+ (int)
+ Math.max(
+ 5,
+ Math.min(
+ CONFLICT_RETRY_MAX_TIMES, retryMaxMillisWhenOtherExceptionsOccur * 1.1)));
+ }
+ }
+ }
+
+ private void recordExceptionStatusIfNecessary(final String message) {
+ if (!Objects.equals(exceptionRecordedMessage.get(), message)) {
+ exceptionFirstEncounteredTime.set(System.currentTimeMillis());
+ exceptionEventHasBeenRetried.set(false);
+ exceptionRecordedMessage.set(message);
+ }
+ }
+
+ private void resetExceptionStatus() {
+ exceptionFirstEncounteredTime.set(0);
+ exceptionEventHasBeenRetried.set(false);
+ exceptionRecordedMessage.set("");
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/session/IClientSession.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/session/IClientSession.java
new file mode 100644
index 0000000..0a1b414
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/protocol/session/IClientSession.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.sink.protocol.session;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public abstract class IClientSession {
+
+ private long id;
+
+ private String username;
+
+ /** ip:port for thrift-based service and client id for mqtt-based service. */
+ abstract String getConnectionId();
+
+ public String getUsername() {
+ return this.username;
+ }
+
+ public long getId() {
+ return id;
+ }
+
+ public void setId(long id) {
+ this.id = id;
+ }
+
+ public String toString() {
+ return String.format("%d-%s:%s", getId(), getUsername(), getConnectionId());
+ }
+
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ public enum SqlDialect {
+ TREE((byte) 0),
+ TABLE((byte) 1);
+
+ private final byte dialect;
+
+ SqlDialect(byte dialect) {
+ this.dialect = dialect;
+ }
+
+ public void serialize(final DataOutputStream stream) throws IOException {
+ ReadWriteIOUtils.write(dialect, stream);
+ }
+
+ public void serialize(final ByteBuffer buffer) {
+ ReadWriteIOUtils.write(dialect, buffer);
+ }
+ }
+}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpPullSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpPullSource.java
index afb8d15..ff4cffe 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpPullSource.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpPullSource.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.collector.plugin.builtin.source;
import org.apache.iotdb.collector.plugin.api.PullSource;
import org.apache.iotdb.collector.plugin.api.event.DemoEvent;
-import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration;
import org.apache.iotdb.pipe.api.customizer.configuration.PipeSourceRuntimeConfiguration;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
@@ -41,11 +40,6 @@ public class HttpPullSource extends PullSource {
@Override
public void validate(PipeParameterValidator pipeParameterValidator) {}
- @Override
- public void customize(
- PipeParameters pipeParameters,
- PipeExtractorRuntimeConfiguration pipeExtractorRuntimeConfiguration) {}
-
@Override
public void customize(
PipeParameters pipeParameters,
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpPushSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpPushSource.java
index fa81fc2..2225829 100644
--- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpPushSource.java
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpPushSource.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.collector.plugin.builtin.source;
import org.apache.iotdb.collector.plugin.api.PushSource;
import org.apache.iotdb.collector.plugin.api.event.DemoEvent;
-import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration;
import org.apache.iotdb.pipe.api.customizer.configuration.PipeSourceRuntimeConfiguration;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
@@ -43,10 +42,6 @@ public class HttpPushSource extends PushSource {
@Override
public void validate(final PipeParameterValidator validator) {}
- @Override
- public void customize(
- final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration) {}
-
@Override
public void customize(
final PipeParameters parameters, final PipeSourceRuntimeConfiguration configuration) {}
diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/IoTDBPushSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/IoTDBPushSource.java
new file mode 100644
index 0000000..6631569
--- /dev/null
+++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/IoTDBPushSource.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.collector.plugin.builtin.source;
+
+import org.apache.iotdb.collector.plugin.api.PushSource;
+import org.apache.iotdb.collector.plugin.builtin.source.constant.IoTDBPushSourceConstant;
+import org.apache.iotdb.collector.plugin.builtin.source.event.SubDemoEvent;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeSourceRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.rpc.subscription.config.ConsumerConstant;
+import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType;
+import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Properties;
+
+public class IoTDBPushSource extends PushSource {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBPushSource.class);
+
+ private String host;
+ private Integer port;
+ private String topic;
+ private Long timeout;
+ private String deviceId;
+
+ private volatile boolean isStarted = true;
+ private Thread workerThread;
+
+ @Override
+ public void validate(PipeParameterValidator validator) throws Exception {}
+
+ @Override
+ public void customize(
+ PipeParameters pipeParameters, PipeSourceRuntimeConfiguration pipeSourceRuntimeConfiguration)
+ throws Exception {
+ host =
+ pipeParameters.getStringOrDefault(
+ IoTDBPushSourceConstant.HOST_KEY, IoTDBPushSourceConstant.HOST_VALUE);
+ port =
+ pipeParameters.getIntOrDefault(
+ IoTDBPushSourceConstant.PORT_KEY, IoTDBPushSourceConstant.PORT_VALUE);
+ topic =
+ pipeParameters.getStringOrDefault(
+ IoTDBPushSourceConstant.TOPIC_KEY, IoTDBPushSourceConstant.TOPIC_VALUE);
+ timeout =
+ pipeParameters.getLongOrDefault(
+ IoTDBPushSourceConstant.TIMEOUT_KEY, IoTDBPushSourceConstant.TIMEOUT_VALUE);
+ deviceId =
+ pipeParameters.getStringOrDefault(
+ IoTDBPushSourceConstant.DEVICE_ID_KEY, IoTDBPushSourceConstant.DEVICE_ID_VALUE);
+ }
+
+ @Override
+ public void start() throws Exception {
+ if (workerThread == null || !workerThread.isAlive()) {
+ isStarted = true;
+ workerThread = new Thread(this::doWork);
+ workerThread.start();
+ }
+ }
+
+ private void doWork() {
+ final Properties pullProperties = new Properties();
+ pullProperties.put(IoTDBPushSourceConstant.HOST_KEY, host);
+ pullProperties.put(IoTDBPushSourceConstant.PORT_KEY, port);
+ pullProperties.put(ConsumerConstant.CONSUMER_ID_KEY, "r1");
+ pullProperties.put(ConsumerConstant.CONSUMER_GROUP_ID_KEY, "rg1");
+
+ try (final SubscriptionTreePullConsumer consumer =
+ new SubscriptionTreePullConsumer(pullProperties)) {
+ consumer.open();
+ consumer.subscribe(topic);
+
+ while (isStarted && !Thread.currentThread().isInterrupted()) {
+ final List<SubscriptionMessage> messages = consumer.poll(timeout);
+ for (final SubscriptionMessage message : messages) {
+ final short messageType = message.getMessageType();
+ if (SubscriptionMessageType.isValidatedMessageType(messageType)) {
... 3377 lines suppressed ...