You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by co...@apache.org on 2025/08/14 11:22:14 UTC
(seatunnel) branch dev updated: [Feature][connector-v2] Add Sensorsdata Connector Support #9323 (#9432)
This is an automated email from the ASF dual-hosted git repository.
corgy pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new bb53f77264 [Feature][connector-v2] Add Sensorsdata Connector Support #9323 (#9432)
bb53f77264 is described below
commit bb53f7726494f18bbf4fde17b1522ce96b6fb7c7
Author: zhangqingsong <zh...@126.com>
AuthorDate: Thu Aug 14 19:22:09 2025 +0800
[Feature][connector-v2] Add Sensorsdata Connector Support #9323 (#9432)
---
.github/workflows/backend.yml | 25 ++
.github/workflows/labeler/label-scope-conf.yml | 9 +-
config/plugin_config | 1 +
.../changelog/connector-sensorsdata.md | 7 +
docs/en/connector-v2/sink/SensorsData.md | 291 +++++++++++++++++
plugin-mapping.properties | 1 +
.../connector-sensorsdata/pom.xml | 56 ++++
.../sensorsdata/format/SensorsDataTypes.java | 99 ++++++
.../format/config/SensorsDataBaseOptionRules.java | 52 +++
.../format/config/SensorsDataConfigBase.java | 74 +++++
.../format/config/SensorsDataOptions.java | 131 ++++++++
.../format/config/TargetColumnConfig.java | 57 ++++
.../format/exception/SensorsDataErrorCode.java | 49 +++
.../format/exception/SensorsDataException.java | 36 ++
.../sensorsdata/format/record/RowAccessor.java | 363 +++++++++++++++++++++
.../format/record/SensorsDataJsonKeys.java | 56 ++++
.../format/record/SensorsDataLibInfo.java | 41 +++
.../format/record/SensorsDataRecord.java | 22 ++
.../format/record/SensorsDataRecordBuilder.java | 131 ++++++++
.../format/record/SensorsDataRecordType.java | 38 +++
.../format/record/SpecialItemRecord.java | 121 +++++++
.../format/record/UserDetailRecord.java | 57 ++++
.../sensorsdata/format/record/UserEventRecord.java | 55 ++++
.../sensorsdata/format/record/UserRecord.java | 53 +++
.../sensorsdata/format/record/UserRecordBase.java | 217 ++++++++++++
.../sensorsdata/format/utils/TypeUtil.java | 297 +++++++++++++++++
.../sensorsdata/format/utils/UserSchemaUtil.java | 80 +++++
.../sdk/config/SensorsDataSDKSinkConfig.java | 51 +++
.../sdk/config/SensorsDataSDKSinkOptions.java | 64 ++++
.../exception/SensorsDataConnectorErrorCode.java | 46 +++
.../exception/SensorsDataConnectorException.java | 37 +++
.../sensorsdata/sdk/sink/SensorsDataSDKSink.java | 70 ++++
.../sdk/sink/SensorsDataSDKSinkFactory.java | 59 ++++
.../sensorsdata/sdk/sink/SensorsDataSDKWriter.java | 200 ++++++++++++
.../sdk/state/SensorsDataAggregatedCommitInfo.java | 25 ++
.../sdk/state/SensorsDataCommitInfo.java | 24 ++
.../sdk/state/SensorsDataSinkState.java | 24 ++
.../sensorsdata/format/SensorsDataTypesTest.java | 31 ++
.../record/SensorsDataSpecialItemRecordTest.java | 163 +++++++++
.../format/record/SensorsDataUserRecordTest.java | 274 ++++++++++++++++
.../sensorsdata/format/utils/TypeUtilTest.java | 103 ++++++
.../sensorsdata/sdk/SensorsDataSDKFactoryTest.java | 31 ++
seatunnel-connectors-v2/pom.xml | 1 +
seatunnel-dist/pom.xml | 6 +
.../connector-sensorsdata-e2e/pom.xml | 60 ++++
.../connector/sensorsdata/sdk/SensorsDataIT.java | 80 +++++
.../resources/fake_to_sensorsdata_details.conf | 66 ++++
.../test/resources/fake_to_sensorsdata_events.conf | 78 +++++
.../test/resources/fake_to_sensorsdata_users.conf | 72 ++++
seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml | 1 +
tools/update_modules_check/update_modules_check.py | 1 +
51 files changed, 3985 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index bb06df2ee7..a528fb83ef 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -1435,3 +1435,28 @@ jobs:
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-redis-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
+
+ connector-sensorsdata-it:
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true' || contains(needs.changes.outputs.it-modules, 'connector-sensorsdata-e2e')
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ java: [ '8', '11' ]
+ os: [ 'ubuntu-latest' ]
+ timeout-minutes: 120
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
+ with:
+ java-version: ${{ matrix.java }}
+ distribution: 'temurin'
+ cache: 'maven'
+ - name: free disk space
+ run: tools/github/free_disk_space.sh
+ - name: run sensorsdata connector integration test
+ run: |
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-sensorsdata-e2e -am -Pci
+ env:
+ MAVEN_OPTS: -Xmx4096m
diff --git a/.github/workflows/labeler/label-scope-conf.yml b/.github/workflows/labeler/label-scope-conf.yml
index bd316f8f12..66e2f63e50 100644
--- a/.github/workflows/labeler/label-scope-conf.yml
+++ b/.github/workflows/labeler/label-scope-conf.yml
@@ -320,4 +320,11 @@ aerospike:
- all:
- changed-files:
- any-glob-to-any-file: seatunnel-connectors-v2/connector-aerospike/**
- - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(aerospike)/**'
\ No newline at end of file
+ - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(aerospike)/**'
+
+sensorsdata:
+ - all:
+ - changed-files:
+ - any-glob-to-any-file: seatunnel-connectors-v2/connector-sensorsdata/**
+ - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(sensorsdata)/**'
+
diff --git a/config/plugin_config b/config/plugin_config
index af39d13da7..8d3dd7dbe2 100644
--- a/config/plugin_config
+++ b/config/plugin_config
@@ -95,3 +95,4 @@ connector-sls
connector-qdrant
connector-typesense
connector-cdc-opengauss
+connector-sensorsdata
diff --git a/docs/en/connector-v2/changelog/connector-sensorsdata.md b/docs/en/connector-v2/changelog/connector-sensorsdata.md
new file mode 100644
index 0000000000..cf638b49c5
--- /dev/null
+++ b/docs/en/connector-v2/changelog/connector-sensorsdata.md
@@ -0,0 +1,7 @@
+<details><summary> Change Log </summary>
+
+| Change | Commit | Version |
+| --- | --- | --- |
+
+
+</details>
\ No newline at end of file
diff --git a/docs/en/connector-v2/sink/SensorsData.md b/docs/en/connector-v2/sink/SensorsData.md
new file mode 100644
index 0000000000..699a9610c1
--- /dev/null
+++ b/docs/en/connector-v2/sink/SensorsData.md
@@ -0,0 +1,291 @@
+import ChangeLog from '../changelog/connector-sensorsdata.md';
+
+# SensorsData
+
+> SensorsData sink connector
+
+## Support Those Engines
+
+> Spark<br/>
+> Flink<br/>
+> SeaTunnel Zeta<br/>
+
+## Key features
+
+- [ ] [exactly-once](../../concept/connector-v2-features.md)
+- [ ] [cdc](../../concept/connector-v2-features.md)
+
+## Description
+
+A sink plugin which use SensorsData SDK send data records.
+
+## Sink Options
+
+| name | type | required | default value |
+|---------------------------|---------|----------|---------------|
+| server_url | string | yes | - |
+| bulk_size | int | no | 50 |
+| max_cache_row_size | int | no | 0 |
+| consumer | string | no | batch |
+| entity_name | string | yes | users |
+| record_type | string | yes | users |
+| schema | string | yes | users |
+| distinct_id_column | string | yes | - |
+| identity_fields | array | yes | - |
+| property_fields | array | yes | - |
+| event_name | string | yes | - |
+| time_column | string | yes | - |
+| time_free | boolean | no | false |
+| detail_id_column | string | no | - |
+| item_id_column | string | no | - |
+| item_type_column | string | no | - |
+| skip_error_record | boolean | no | false |
+| instant_events | array | no | - |
+| distinct_id_by_identities | boolean | no | false |
+| null_as_profile_unset | boolean | no | false |
+| common-options | | no | - |
+
+
+## Parameter Interpretation
+### server_url [string]
+
+SensorsData data sink address, the format is `https://${host}:8106/sa?project=${project}`
+
+### bulk_size [int]
+
+Threshold for the triggering flush operation in SensorsData SDK. When the memory cache queue reaches this value, the data in the cache will be sent. The default value is 50.
+
+### max_cache_row_size[int]
+
+Maximum cache refresh size for SensorsData SDK. If it exceeds this value, the flush operation will be triggered immediately. The default value is 0, which depends on bulkSize.
+
+### consumer[string]
+
+When consumer is set to "console", the data will be output to console instead of send to the server.
+
+### entity_name[string]
+
+The entity name of the SensorsData entity data model to receive the data records.
+
+### record_type[string]
+
+The record type of the SensorsData entity data model.
+
+### schema[string]
+
+The schema name of the SensorsData entity data model.
+
+### distinct_id_column[string]
+
+The distinct id column of the user entity.
+
+### identity_fields[array]
+
+The identity fields of the user entity.
+
+### property_fields[array]
+
+The property fields of the data record. Dupported types:
+- BOOLEAN
+- DECIMAL
+- INT
+- BIGINT
+- FLOAT
+- DOUBLE
+- NUMBER
+- STRING
+- DATE
+- TIMESTAMP
+- LIST
+- LIST_COMMA
+- LIST_SEMICOLON
+
+### event_name[string]
+
+Currently, two formats are supported:
+
+1. Fill in the name of the event record.
+2. Use value of a field from upstream data as the event name, the format is `${your field name}`, where event name is the value of the columns of the upstream data.
+
+For example, Upstream data is the following:
+
+| name | prop1 | prop2 |
+|----------|-------|---------------|
+| Purchase | 16 | data-example1 |
+| Order | 23 | data-example2 |
+
+If `${name}` is set as the event name, the event name of the first row is "Purchase", and the event name of the second row is "Order".
+
+### time_column[string]
+
+The time column of the event record.
+
+### time_free[boolean]
+
+Enable historical data mode.
+
+### detail_id_column[string]
+
+The detail id column of the user entity.
+
+### item_id_column[string]
+
+The item id column of the item entity.
+
+### item_type_column[string]
+
+The item type column of the item entity.
+
+### skip_error_record[boolean]
+
+Whether ignore the error in translating the data record.
+
+### instant_events[array]
+
+Given a list of event names, mark the event as an instant event.
+
+### distinct_id_by_identities[boolean]
+
+When enabled, this option automatically fills the distinct_id using the values from identity_fields columns when the distinct_id_column value is null. This ensures that SensorsData receives a non-null distinct_id value as required.
+
+### null_as_profile_unset[boolean]
+
+When enabled, null values in profile properties will be converted to profile unset operations, effectively removing the existing value from the profile.
+
+### common options
+
+Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details
+
+## Examples
+
+### Basic Event Tracking
+
+```hocon
+sink {
+ SensorsData {
+ server_url = "http://10.1.136.63:8106/sa?project=default"
+ time_free = true
+
+ record_type = events
+ schema = events
+ event_name = "$AppStart"
+ time_column = col_date
+ distinct_id_column = col_id
+ identity_fields = [
+ { source = col_id, target = "$identity_login_id" }
+ { source = col_id, target = "$identity_distinct_id" }
+ ]
+ property_fields = [
+ { target = prop1, source = col1, type = INT }
+ { target = prop2, source = col2, type = BIGINT }
+ { target = prop3, source = col3, type = STRING }
+ { target = prop4, source = col4, type = BOOLEAN }
+ ]
+ skip_error_record = true
+ }
+}
+```
+
+### Dynamic Event Names
+
+```hocon
+sink {
+ SensorsData {
+ server_url = "http://10.1.136.63:8106/sa?project=default"
+ time_free = true
+
+ record_type = events
+ schema = events
+ event_name = "${event_type}" # Use dynamic event name from data
+ time_column = event_timestamp
+ distinct_id_column = user_id
+ identity_fields = [
+ { source = user_id, target = "$identity_login_id" }
+ { source = user_id, target = "$identity_distinct_id" }
+ ]
+ property_fields = [
+ { target = "price", source = amount, type = DECIMAL }
+ { target = "category", source = product_category, type = STRING }
+ { target = "device", source = device_type, type = STRING }
+ ]
+ instant_events = ["$AppStart", "$AppEnd"] # Mark specific events as instant
+ }
+}
+```
+
+### Profile Property Updates
+
+```hocon
+sink {
+ SensorsData {
+ server_url = "http://10.1.136.63:8106/sa?project=default"
+ time_free = true
+
+ entity_name = users
+ record_type = profile
+ schema = users
+ distinct_id_column = user_id
+ identity_fields = [
+ { source = email, target = "$identity_email" }
+ { source = phone, target = "$identity_phone" }
+ ]
+ property_fields = [
+ { target = "name", source = full_name, type = STRING }
+ { target = "age", source = user_age, type = INT }
+ { target = "gender", source = user_gender, type = STRING }
+ { target = "location", source = user_location, type = STRING }
+ ]
+ null_as_profile_unset = true # Remove properties when null
+ }
+}
+```
+
+### Item Tracking
+
+```hocon
+sink {
+ SensorsData {
+ server_url = "http://10.1.136.63:8106/sa?project=default"
+ time_free = true
+
+ record_type = items
+ schema = items
+ event_name = "$ItemViewed"
+ time_column = view_time
+ distinct_id_column = user_id
+ identity_fields = [
+ { source = user_id, target = "$identity_login_id" }
+ ]
+ property_fields = [
+ { target = "view_duration", source = duration, type = INT }
+ { target = "referrer", source = referrer_url, type = STRING }
+ ]
+ item_id_column = product_id
+ item_type_column = product_type
+ }
+}
+```
+
+### Console Output (for Testing)
+
+```hocon
+sink {
+ SensorsData {
+ server_url = "http://10.1.136.63:8106/sa?project=default"
+ consumer = "console" # Output to console instead of sending to server
+ record_type = events
+ schema = events
+ event_name = "$TestEvent"
+ time_column = timestamp
+ distinct_id_column = test_id
+ property_fields = [
+ { target = "test", source = test_field, type = STRING }
+ ]
+ }
+}
+```
+
+## Changelog
+
+<ChangeLog />
+
diff --git a/plugin-mapping.properties b/plugin-mapping.properties
index 38d74d31f3..10dfd69256 100644
--- a/plugin-mapping.properties
+++ b/plugin-mapping.properties
@@ -146,6 +146,7 @@ seatunnel.source.Opengauss-CDC = connector-cdc-opengauss
seatunnel.source.GraphQL = connector-graphql
seatunnel.sink.GraphQL = connector-graphql
seatunnel.sink.Aerospike = connector-aerospike
+seatunnel.sink.SensorsData = connector-sensorsdata
# For custom transforms, make sure to use the seatunnel.transform.[PluginIdentifier]=[JarPerfix] naming convention. For example:
# seatunnel.transform.Sql = seatunnel-transforms-v2
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/pom.xml b/seatunnel-connectors-v2/connector-sensorsdata/pom.xml
new file mode 100644
index 0000000000..e4bd490fbf
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-connectors-v2</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>connector-sensorsdata</artifactId>
+ <name>SeaTunnel : Connectors V2 : SensorsData</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+
+ <!-- SeaTunnel Libs -->
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sensorsdata.analytics.javasdk</groupId>
+ <artifactId>SensorsAnalyticsSDK</artifactId>
+ <version>3.6.9</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/SensorsDataTypes.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/SensorsDataTypes.java
new file mode 100644
index 0000000000..420d8dcbd8
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/SensorsDataTypes.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format;
+
+import org.apache.commons.lang3.StringUtils;
+
+import lombok.Getter;
+
+public class SensorsDataTypes {
+ public enum DataTypes {
+ UNKNOWN,
+ BOOLEAN,
+ DECIMAL,
+ INT,
+ BIGINT,
+ FLOAT,
+ DOUBLE,
+ NUMBER,
+ STRING,
+ DATE,
+ TIMESTAMP,
+ LIST,
+ LIST_COMMA,
+ LIST_SEMICOLON;
+
+ public static DataTypes of(String s) {
+ String str = StringUtils.upperCase(StringUtils.trim(s));
+ if (StringUtils.isBlank(str)) {
+ return DataTypes.UNKNOWN;
+ }
+ if (StringUtils.startsWith(str, "TIMESTAMP")) {
+ // TIMESTAMP include timezone, see
+ // org.apache.seatunnel.format.sensorsdata.utils.TypeUtilTest
+ return DataTypes.TIMESTAMP;
+ }
+ switch (str) {
+ case "BOOLEAN":
+ return DataTypes.BOOLEAN;
+ case "DECIMAL":
+ return DataTypes.DECIMAL;
+ case "INT":
+ return DataTypes.INT;
+ case "BIGINT":
+ case "LONG":
+ return DataTypes.BIGINT;
+ case "FLOAT":
+ return DataTypes.FLOAT;
+ case "DOUBLE":
+ return DataTypes.DOUBLE;
+ case "NUMBER":
+ return DataTypes.NUMBER;
+ case "LIST":
+ return DataTypes.LIST;
+ case "LIST_COMMA":
+ return DataTypes.LIST_COMMA;
+ case "LIST_SEMICOLON":
+ return DataTypes.LIST_SEMICOLON;
+ case "DATE":
+ return DataTypes.DATE;
+ case "STRING":
+ return DataTypes.STRING;
+ default:
+ return DataTypes.UNKNOWN;
+ }
+ }
+ }
+
+ @Getter private final DataTypes type;
+ @Getter private final String extra;
+
+ SensorsDataTypes(DataTypes type, String extra) {
+ this.type = type;
+ this.extra = extra;
+ }
+
+ public static SensorsDataTypes of(String str) {
+ DataTypes type = DataTypes.of(str);
+ String suffix =
+ StringUtils.length(str) > type.name().length()
+ ? StringUtils.trim(StringUtils.substring(str, type.name().length()))
+ : null;
+ return new SensorsDataTypes(type, suffix);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/SensorsDataBaseOptionRules.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/SensorsDataBaseOptionRules.java
new file mode 100644
index 0000000000..f5929c7b74
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/SensorsDataBaseOptionRules.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.config;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class SensorsDataBaseOptionRules {
+ public static OptionRule.Builder getBaseOptionRuleBuilder() {
+ return OptionRule.builder()
+ .required(SensorsDataOptions.ENTITY_NAME, SensorsDataOptions.RECORD_TYPE)
+ .conditional(
+ SensorsDataOptions.ENTITY_NAME,
+ "users",
+ SensorsDataOptions.SCHEMA,
+ SensorsDataOptions.DISTINCT_ID_COLUMN,
+ SensorsDataOptions.IDENTITY_FIELDS,
+ SensorsDataOptions.PROPERTY_FIELDS)
+ .conditional(
+ SensorsDataOptions.RECORD_TYPE,
+ "events",
+ SensorsDataOptions.TIME_COLUMN,
+ SensorsDataOptions.EVENT_NAME)
+ .conditional(
+ SensorsDataOptions.RECORD_TYPE,
+ "details",
+ SensorsDataOptions.DETAIL_ID_COLUMN)
+ .conditional(
+ SensorsDataOptions.RECORD_TYPE,
+ "items",
+ SensorsDataOptions.ITEM_ID_COLUMN,
+ SensorsDataOptions.ITEM_TYPE_COLUMN)
+ .optional(SensorsDataOptions.TIME_FREE);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/SensorsDataConfigBase.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/SensorsDataConfigBase.java
new file mode 100644
index 0000000000..361f659fd2
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/SensorsDataConfigBase.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.config;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+
+import lombok.Getter;
+import lombok.ToString;
+
+import java.io.Serializable;
+import java.util.List;
+
+@Getter
+@ToString
+public class SensorsDataConfigBase implements Serializable {
+ protected final String entityName;
+ protected final String recordType;
+ protected final String schema;
+ protected final String distinctIdColumn;
+ protected final List<TargetColumnConfig> identityFields;
+ protected final List<TargetColumnConfig> propertyFields;
+ protected final String eventName;
+ protected final String timeColumn;
+ protected final String detailIdColumn;
+ protected final boolean timeFree;
+ protected final boolean skipErrorRecord;
+ protected final String itemIdColumn;
+ protected final String itemTypeColumn;
+
+ /**
+ * As a supplement to the detailIdColumn configuration, if the value of distinct_id obtained
+ * from the column specified by distinctIdColumn is null, then get the value from
+ * identityFields.
+ */
+ protected final boolean distinctIdByIdentities;
+ /**
+ * null user property as profile_unset (default false) If true, in seatunnel profile_set logic,
+ * add process properties which value is null, send a profile_unset
+ */
+ protected final boolean nullAsProfileUnset;
+
+ public SensorsDataConfigBase(ReadonlyConfig config) {
+ this.entityName = config.get(SensorsDataOptions.ENTITY_NAME);
+ this.recordType = config.get(SensorsDataOptions.RECORD_TYPE);
+ this.schema = config.get(SensorsDataOptions.SCHEMA);
+ this.distinctIdColumn = config.get(SensorsDataOptions.DISTINCT_ID_COLUMN);
+ this.identityFields = config.get(SensorsDataOptions.IDENTITY_FIELDS);
+ this.propertyFields = config.get(SensorsDataOptions.PROPERTY_FIELDS);
+ this.eventName = config.get(SensorsDataOptions.EVENT_NAME);
+ this.timeColumn = config.get(SensorsDataOptions.TIME_COLUMN);
+ this.detailIdColumn = config.get(SensorsDataOptions.DETAIL_ID_COLUMN);
+ this.timeFree = config.get(SensorsDataOptions.TIME_FREE);
+ this.skipErrorRecord = config.get(SensorsDataOptions.SKIP_ERROR_RECORD);
+ this.itemIdColumn = config.get(SensorsDataOptions.ITEM_ID_COLUMN);
+ this.itemTypeColumn = config.get(SensorsDataOptions.ITEM_TYPE_COLUMN);
+ this.distinctIdByIdentities = config.get(SensorsDataOptions.DISTINCT_ID_BY_IDENTITIES);
+ this.nullAsProfileUnset = config.get(SensorsDataOptions.NULL_AS_PROFILE_UNSET);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/SensorsDataOptions.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/SensorsDataOptions.java
new file mode 100644
index 0000000000..62e749249f
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/SensorsDataOptions.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.config;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+import java.util.List;
+
+@SuppressWarnings("checkstyle:MagicNumber")
+public interface SensorsDataOptions {
+ Option<String> ENTITY_NAME =
+ Options.key("entity_name")
+ .stringType()
+ .defaultValue("users")
+ .withDescription("entity name: users(default)/items");
+
+ Option<String> RECORD_TYPE =
+ Options.key("record_type")
+ .stringType()
+ .defaultValue("users")
+ .withDescription("Record type: users/events/items/details");
+
+ Option<String> SCHEMA =
+ Options.key("schema").stringType().defaultValue("users").withDescription("Schema name");
+
+ Option<String> DISTINCT_ID_COLUMN =
+ Options.key("distinct_id_column")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Specify a column as the distinct id for users");
+
+ Option<List<TargetColumnConfig>> IDENTITY_FIELDS =
+ Options.key("identity_fields")
+ .listType(TargetColumnConfig.class)
+ .noDefaultValue()
+ .withDescription(
+ "Specify the identity fields and where they come from. format: { source = ${source_field}, target = ${identity_field} }");
+
+ Option<List<TargetColumnConfig>> PROPERTY_FIELDS =
+ Options.key("property_fields")
+ .listType(TargetColumnConfig.class)
+ .noDefaultValue()
+ .withDescription(
+ "Specify the property fields and their data types. format: { source = ${source_field}, target = ${target_property_field}, type = ${data_type} }");
+
+ Option<String> EVENT_NAME =
+ Options.key("event_name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Specify the event name when record_type = \"events\".");
+
+ Option<String> TIME_COLUMN =
+ Options.key("time_column")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Specify a column as the $time property for events");
+
+ Option<String> DETAIL_ID_COLUMN =
+ Options.key("detail_id_column")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Specify a column as the detail id for detail entity when record_type = \"details\".");
+
+ Option<Boolean> TIME_FREE =
+ Options.key("time_free")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Enable time_free for events, true/false(default)");
+
+ Option<List<TargetColumnConfig>> TARGET_COLUMNS =
+ Options.key("target_columns").listType(TargetColumnConfig.class).noDefaultValue();
+
+ Option<Boolean> SKIP_ERROR_RECORD =
+ Options.key("skip_error_record")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "While encountering an error, either skip the record or terminate the process.");
+
+ Option<String> ITEM_ID_COLUMN =
+ Options.key("item_id_column")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Specify a column as the item id for items when record_type = \"items\".");
+
+ Option<String> ITEM_TYPE_COLUMN =
+ Options.key("item_type_column")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Specify a column as the item type for items when record_type = \"items\".");
+
+ Option<String> JSON_COLUMN_NAME =
+ Options.key("json_column_name")
+ .stringType()
+ .defaultValue("json_content")
+ .withDescription(
+ "Specify the target column name for the output of the SensorsDataJson Transform. ");
+
+ Option<Boolean> DISTINCT_ID_BY_IDENTITIES =
+ Options.key("distinct_id_by_identities")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "when distinct_id_column value is null, enable get distinctId value by identityFields");
+
+ Option<Boolean> NULL_AS_PROFILE_UNSET =
+ Options.key("null_as_profile_unset")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "when properties value is null, enable send profile_unset action");
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/TargetColumnConfig.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/TargetColumnConfig.java
new file mode 100644
index 0000000000..058dceed67
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/config/TargetColumnConfig.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.config;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+public class TargetColumnConfig implements Serializable {
+
+ private String source;
+
+ private String type;
+
+ private String target;
+
+ public TargetColumnConfig() {}
+
+ public TargetColumnConfig(String source, String type) {
+ this.source = source;
+ this.type = type;
+ }
+
+ public String getTarget() {
+ return target == null ? source : target;
+ }
+
+ public static List<TargetColumnConfig> of(ReadonlyConfig connectorConfig) {
+ if (connectorConfig.getOptional(SensorsDataOptions.TARGET_COLUMNS).isPresent()) {
+ return connectorConfig.get(SensorsDataOptions.TARGET_COLUMNS);
+ } else {
+ return Collections.emptyList();
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/exception/SensorsDataErrorCode.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/exception/SensorsDataErrorCode.java
new file mode 100644
index 0000000000..46ee77ecec
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/exception/SensorsDataErrorCode.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.exception;
+
+import org.apache.seatunnel.common.exception.SeaTunnelErrorCode;
+
+public enum SensorsDataErrorCode implements SeaTunnelErrorCode {
+ DATA_TYPE_CAST_FILED("SENSORS_DATA-01", "Value type does not match column type"),
+ UNSUPPORTED_RECORD_TYPE("SENSORS_DATA-02", "Unsupported record type"),
+ EVENT_NAME_NOT_SET("SENSORS_DATA-03", "Event name not set"),
+ ILLEGAL_ARGUMENT("SENSORS_DATA-04", "Illegal argument"),
+ UNKNOWN_SOURCE_FIELD("SENSORS_DATA-05", "Unknown source field"),
+ MISSING_NECESSARY_FIELD("SENSORS_DATA-06", "Missing necessary field"),
+ ;
+
+ private final String code;
+
+ private final String description;
+
+ SensorsDataErrorCode(String code, String description) {
+ this.code = code;
+ this.description = description;
+ }
+
+ @Override
+ public String getCode() {
+ return this.code;
+ }
+
+ @Override
+ public String getDescription() {
+ return this.description;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/exception/SensorsDataException.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/exception/SensorsDataException.java
new file mode 100644
index 0000000000..34969b130d
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/exception/SensorsDataException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.exception;
+
+import org.apache.seatunnel.common.exception.SeaTunnelErrorCode;
+import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException;
+
+public class SensorsDataException extends SeaTunnelRuntimeException {
+ public SensorsDataException(SeaTunnelErrorCode seaTunnelErrorCode, String errorMessage) {
+ super(seaTunnelErrorCode, errorMessage);
+ }
+
+ public SensorsDataException(
+ SeaTunnelErrorCode seaTunnelErrorCode, String errorMessage, Throwable cause) {
+ super(seaTunnelErrorCode, errorMessage, cause);
+ }
+
+ public SensorsDataException(SeaTunnelErrorCode seaTunnelErrorCode, Throwable cause) {
+ super(seaTunnelErrorCode, cause);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/RowAccessor.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/RowAccessor.java
new file mode 100644
index 0000000000..01adec5efc
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/RowAccessor.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.sensorsdata.format.SensorsDataTypes;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.SensorsDataConfigBase;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.TargetColumnConfig;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataErrorCode;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataException;
+import org.apache.seatunnel.connectors.sensorsdata.format.utils.TypeUtil;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.sensorsdata.analytics.javasdk.SensorsConst;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static java.util.stream.Collectors.toList;
+
+public class RowAccessor implements Serializable {
+ private static final Pattern EVENT_NAME_CONFIG_PATTERN =
+ Pattern.compile("\\$\\{(.*?)\\}", Pattern.DOTALL);
+
+ private final SensorsDataConfigBase config;
+
+ private final String schema;
+
+ private final Map<String, Integer> columnIndex = new HashMap<>();
+
+ private final Integer distinctIdColumnIndex;
+ private Integer timeColumnIndex;
+ private boolean eventTimeUseCurrentTime;
+
+ private String eventName;
+ private Integer eventColumnIndex;
+
+ private final Integer detailIdColumnIndex;
+
+ private final Integer itemIdColumnIndex;
+ private final Integer itemTypeColumnIndex;
+
+ private static final String CURRENT_TIME_KEY = "current_time()";
+
+ private static final String OLD_DISTINCT_ID = "distinct_id";
+ private static final String LOGIN_ID = "$identity_login_id";
+ private static final String ANONYMOUS_ID = "$identity_anonymous_id";
+ private static final String DISTINCT_ID = "$identity_distinct_id";
+
+ public RowAccessor(SensorsDataConfigBase config, SeaTunnelRowType rowType) {
+ this.config = config;
+
+ for (int i = 0; i < rowType.getTotalFields(); i++) {
+ String fieldName = rowType.getFieldName(i);
+ columnIndex.put(fieldName, i);
+ }
+
+ this.distinctIdColumnIndex = checkAndGetColumnIndex(config.getDistinctIdColumn());
+
+ if (StringUtils.isNotBlank(config.getTimeColumn())) {
+ if (config.getTimeColumn().equals(CURRENT_TIME_KEY)) {
+ this.eventTimeUseCurrentTime = true;
+ } else {
+ this.eventTimeUseCurrentTime = false;
+ this.timeColumnIndex = checkAndGetColumnIndex(config.getTimeColumn());
+ }
+ }
+
+ initEventNameConfig(config);
+
+ this.schema = config.getSchema();
+ this.detailIdColumnIndex = checkAndGetColumnIndex(config.getDetailIdColumn());
+ this.itemIdColumnIndex = checkAndGetColumnIndex(config.getItemIdColumn());
+ this.itemTypeColumnIndex = checkAndGetColumnIndex(config.getItemTypeColumn());
+
+ checkTargetColumnConfigs();
+ }
+
+ private void initEventNameConfig(SensorsDataConfigBase config) {
+ String str = config.getEventName();
+ if (StringUtils.isBlank(str)) {
+ return;
+ }
+
+ Matcher matcher = EVENT_NAME_CONFIG_PATTERN.matcher(str);
+ if (matcher.find()) {
+ eventName = null;
+ eventColumnIndex = checkAndGetColumnIndex(matcher.group(1));
+ } else {
+ eventName = str;
+ eventColumnIndex = null;
+ }
+ }
+
+ private Integer checkAndGetColumnIndex(String columnName) {
+ if (StringUtils.isBlank(columnName)) {
+ return null;
+ }
+
+ Integer index = columnIndex.get(columnName);
+ if (index == null) {
+ String message = String.format("Field [%s] not found in source column", columnName);
+ throw new SensorsDataException(SensorsDataErrorCode.UNKNOWN_SOURCE_FIELD, message);
+ }
+ return index;
+ }
+
+ private void checkTargetColumnConfigs() {
+ ArrayList<TargetColumnConfig> targetColumnConfigs =
+ new ArrayList<>(config.getPropertyFields());
+
+ if (config.getIdentityFields() != null) {
+ targetColumnConfigs.addAll(config.getIdentityFields());
+ }
+
+ List<String> unknownSourceFields =
+ targetColumnConfigs.stream()
+ .map(TargetColumnConfig::getSource)
+ .distinct()
+ .filter(source -> !columnIndex.containsKey(source))
+ .collect(toList());
+
+ if (!unknownSourceFields.isEmpty()) {
+ String message =
+ String.format(
+ "Fields [%s] not found in source column",
+ String.join(", ", unknownSourceFields));
+ throw new SensorsDataException(SensorsDataErrorCode.UNKNOWN_SOURCE_FIELD, message);
+ }
+ }
+
+ public String getEventName(SeaTunnelRow row) {
+ if (eventName != null) {
+ return eventName;
+ }
+
+ if (eventColumnIndex != null) {
+ return (String) row.getField(eventColumnIndex);
+ }
+
+ throw new SensorsDataException(
+ SensorsDataErrorCode.EVENT_NAME_NOT_SET, "Event name not set");
+ }
+
+ public String getDistinctId(SeaTunnelRow row) {
+ Object distinctValue =
+ TypeUtil.toTargetType(
+ row.getField(this.distinctIdColumnIndex),
+ SensorsDataTypes.DataTypes.STRING);
+ if ((!config.isDistinctIdByIdentities())
+ || (distinctValue != null && StringUtils.isNotBlank((String) distinctValue))) {
+ return (String) distinctValue;
+ }
+ // if the distinctId field is not obtained from the data, it needs to be supplemented with
+ // information from the identitity fields
+ return getDistinctId(getUserIdentities(row));
+ }
+
+ /**
+ * Get the first non-null field in the order of: distinct_id, $identity_login_id,
+ * $identity_anonymous_id, $identity_distinct_id, and other identity fields as the distinct_id.
+ */
+ private String getDistinctId(Map<String, Object> userIdentities) {
+ if (userIdentities.containsKey(OLD_DISTINCT_ID)) {
+ return getIdentityValue(OLD_DISTINCT_ID, userIdentities.get(OLD_DISTINCT_ID));
+ }
+
+ if (userIdentities.containsKey(LOGIN_ID)) {
+ return getIdentityValue(LOGIN_ID, userIdentities.get(LOGIN_ID));
+ }
+
+ if (userIdentities.containsKey(ANONYMOUS_ID)) {
+ return getIdentityValue(ANONYMOUS_ID, userIdentities.get(ANONYMOUS_ID));
+ }
+
+ if (userIdentities.containsKey(DISTINCT_ID)) {
+ return getIdentityValue(DISTINCT_ID, userIdentities.get(DISTINCT_ID));
+ }
+
+ return userIdentities.entrySet().stream()
+ .findFirst()
+ .map(
+ it ->
+ String.format(
+ "%s+%s",
+ it.getKey(), getIdentityValue(it.getKey(), it.getValue())))
+ .orElse(null);
+ }
+
+ private String getIdentityValue(String field, Object value) {
+ if (value instanceof List) {
+ return ((List) value).get(0).toString();
+ } else if (value instanceof String) {
+ return (String) value;
+ }
+ throw new SensorsDataException(
+ SensorsDataErrorCode.ILLEGAL_ARGUMENT,
+ String.format("Identity value must be String or List. [field=%s]", field));
+ }
+
+ public Map<String, Object> getUserIdentities(SeaTunnelRow row) {
+ Map<String, Object> identities = new HashMap<>();
+
+ for (TargetColumnConfig col : config.getIdentityFields()) {
+ String key = col.getTarget();
+ int index = columnIndex.get(col.getSource());
+
+ Object strValue =
+ TypeUtil.toTargetType(row.getField(index), SensorsDataTypes.DataTypes.STRING);
+
+ // if the value is null or blank, skip it
+ if (strValue == null || StringUtils.isBlank((String) strValue)) {
+ continue;
+ }
+
+ Object value;
+ if (isLoginId(key)) {
+ // if it is $identity_login_id, convert and parse it as STRING
+ value = TypeUtil.toTargetType(strValue, SensorsDataTypes.DataTypes.STRING);
+ } else {
+ // otherwise, other identity value are converted and parsed as LIST
+ value = TypeUtil.toTargetType(strValue, SensorsDataTypes.DataTypes.LIST);
+ }
+
+ if (value != null) {
+ identities.put(key, value);
+ }
+ }
+
+ return identities;
+ }
+
+ /**
+ * Whether the identity field is $identity_login_id.
+ *
+ * @param field identity field name
+ * @return true if the field is $identity_login_id
+ */
+ private boolean isLoginId(String field) {
+ return LOGIN_ID.equals(field);
+ }
+
+ public Map<String, String> getIdentities(SeaTunnelRow row) {
+ Map<String, String> identities = new HashMap<>();
+
+ for (TargetColumnConfig col : config.getIdentityFields()) {
+ String key = col.getTarget();
+ int index = columnIndex.get(col.getSource());
+ String value =
+ (String)
+ TypeUtil.toTargetType(
+ row.getField(index), SensorsDataTypes.DataTypes.STRING);
+ if (value != null) {
+ identities.put(key, value);
+ }
+ }
+
+ return identities;
+ }
+
+ public Map<String, Object> getProperties(SeaTunnelRow row) {
+ Map<String, Object> properties = new HashMap<>();
+
+ for (TargetColumnConfig col : config.getPropertyFields()) {
+ String key = col.getTarget();
+ int index = columnIndex.get(col.getSource());
+ Object value = TypeUtil.toTargetType(row.getField(index), col.getType());
+ if (value != null) {
+ properties.put(key, value);
+ }
+ }
+
+ // Set $time
+ if (this.eventTimeUseCurrentTime) {
+ properties.put(SensorsConst.TIME_SYSTEM_ATTR, new Date());
+ } else {
+ if (this.timeColumnIndex != null) {
+ properties.put(
+ SensorsConst.TIME_SYSTEM_ATTR,
+ TypeUtil.toTargetType(
+ row.getField(this.timeColumnIndex),
+ SensorsDataTypes.DataTypes.DATE));
+ }
+ }
+ return properties;
+ }
+
+ public String getSchemaRequired() {
+ if (StringUtils.isBlank(schema)) {
+ throw new SensorsDataException(
+ SensorsDataErrorCode.MISSING_NECESSARY_FIELD, "'schema' is required.");
+ }
+
+ return schema;
+ }
+
+ public String getDetailIdRequired(SeaTunnelRow row) {
+ String detailId =
+ (String)
+ TypeUtil.toTargetType(
+ row.getField(detailIdColumnIndex),
+ SensorsDataTypes.DataTypes.STRING);
+
+ if (StringUtils.isBlank(detailId)) {
+ throw new SensorsDataException(
+ SensorsDataErrorCode.MISSING_NECESSARY_FIELD, "'detailId' is required.");
+ }
+
+ return detailId;
+ }
+
+ public String getItemIdRequired(SeaTunnelRow row) {
+ String itemId =
+ (String)
+ TypeUtil.toTargetType(
+ row.getField(itemIdColumnIndex), SensorsDataTypes.DataTypes.STRING);
+
+ if (StringUtils.isBlank(itemId)) {
+ throw new SensorsDataException(
+ SensorsDataErrorCode.MISSING_NECESSARY_FIELD, "'itemId' is required.");
+ }
+
+ return itemId;
+ }
+
+ public String getItemTypeRequired(SeaTunnelRow row) {
+ String itemType =
+ (String)
+ TypeUtil.toTargetType(
+ row.getField(itemTypeColumnIndex),
+ SensorsDataTypes.DataTypes.STRING);
+
+ if (StringUtils.isBlank(itemType)) {
+ throw new SensorsDataException(
+ SensorsDataErrorCode.MISSING_NECESSARY_FIELD, "'itemType' is required.");
+ }
+
+ return itemType;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataJsonKeys.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataJsonKeys.java
new file mode 100644
index 0000000000..fe559bb91d
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataJsonKeys.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class SensorsDataJsonKeys {
+
+ public static final String TRACK_ID = "_track_id";
+
+ public static final String VERSION = "version";
+
+ public static final String TYPE = "type";
+
+ public static final String LIB = "lib";
+
+ public static final String TIME = "time";
+
+ public static final String PROJECT = "project";
+
+ public static final String PROPERTIES = "properties";
+
+ public static final String TOKEN = "token";
+
+ public static final String SCHEMA = "schema";
+
+ public static final String EVENT = "event";
+
+ public static final String ID = "id";
+
+ public static final String TIME_FREE = "time_free";
+
+ public static final String IDENTITIES = "identities";
+
+ public static final String DISTINCT_ID = "distinct_id";
+
+ public static final String ITEM_ID = "item_id";
+
+ public static final String ITEM_TYPE = "item_type";
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataLibInfo.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataLibInfo.java
new file mode 100644
index 0000000000..cf24213f7b
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataLibInfo.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import org.apache.seatunnel.shade.com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+import static com.sensorsdata.analytics.javasdk.SensorsConst.DEFAULT_LIB_DETAIL;
+import static com.sensorsdata.analytics.javasdk.SensorsConst.LIB;
+import static com.sensorsdata.analytics.javasdk.SensorsConst.LIB_DETAIL_SYSTEM_ATTR;
+import static com.sensorsdata.analytics.javasdk.SensorsConst.LIB_METHOD_SYSTEM_ATTR;
+import static com.sensorsdata.analytics.javasdk.SensorsConst.LIB_SYSTEM_ATTR;
+import static com.sensorsdata.analytics.javasdk.SensorsConst.LIB_VERSION_SYSTEM_ATTR;
+import static com.sensorsdata.analytics.javasdk.SensorsConst.SDK_VERSION;
+
+public class SensorsDataLibInfo {
+
+ public static final Map<String, String> LIB_INFO =
+ ImmutableMap.<String, String>builder()
+ .put(LIB_SYSTEM_ATTR, LIB)
+ .put(LIB_VERSION_SYSTEM_ATTR, SDK_VERSION)
+ .put(LIB_METHOD_SYSTEM_ATTR, "code")
+ .put(LIB_DETAIL_SYSTEM_ATTR, DEFAULT_LIB_DETAIL)
+ .build();
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataRecord.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataRecord.java
new file mode 100644
index 0000000000..b8ad392dd9
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataRecord.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+public interface SensorsDataRecord {
+ String toJsonString();
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataRecordBuilder.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataRecordBuilder.java
new file mode 100644
index 0000000000..562be39d9d
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataRecordBuilder.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.SensorsDataConfigBase;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataErrorCode;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataException;
+
+import lombok.Getter;
+
+public class SensorsDataRecordBuilder {
+
+ // Entity Name
+ private static final String USER_ENTITY_NAME = "users";
+
+ private static final String SPECIAL_ITEM_ENTITY_NAME = "items";
+
+ private static final String USER_RECORD = "users";
+
+ private static final String ITEM_RECORD = "items";
+
+ private static final String EVENT_RECORD = "events";
+
+ private static final String DETAIL_RECORD = "details";
+
+ public static SensorsDataRecordBuilder.Builder newBuilder(
+ SensorsDataConfigBase config, RowAccessor rowAccessor) {
+ return new SensorsDataRecordBuilder.Builder(config, rowAccessor);
+ }
+
+ public static SensorsDataRecordBuilder.Builder newBuilder(
+ SensorsDataRecordType recordType, RowAccessor rowAccessor) {
+ return new SensorsDataRecordBuilder.Builder(recordType, rowAccessor);
+ }
+
+ public static class Builder {
+
+ private final RowAccessor rowAccessor;
+
+ @Getter private final SensorsDataRecordType recordType;
+
+ private UserRecordBase.Builder userRecordBuilder = null;
+
+ private SpecialItemRecord.Builder specialItemRecordBuilder = null;
+
+ private Builder(SensorsDataConfigBase config, RowAccessor rowAccessor) {
+ this.rowAccessor = rowAccessor;
+ switch (config.getEntityName().toLowerCase()) {
+ case USER_ENTITY_NAME:
+ switch (config.getRecordType().toLowerCase()) {
+ case USER_RECORD:
+ this.recordType = SensorsDataRecordType.USER;
+ break;
+ case EVENT_RECORD:
+ this.recordType = SensorsDataRecordType.USER_EVENT;
+ break;
+ case DETAIL_RECORD:
+ this.recordType = SensorsDataRecordType.USER_DETAIL;
+ break;
+ default:
+ throw new SensorsDataException(
+ SensorsDataErrorCode.UNSUPPORTED_RECORD_TYPE,
+ "Unsupported : " + config.getRecordType());
+ }
+ this.userRecordBuilder = UserRecordBase.newBuilder(rowAccessor);
+ break;
+ case SPECIAL_ITEM_ENTITY_NAME:
+ this.recordType = SensorsDataRecordType.SPECIAL_ITEM;
+ this.specialItemRecordBuilder = SpecialItemRecord.newBuilder(rowAccessor);
+ break;
+ default:
+ // not support item record yet.
+ throw new SensorsDataException(
+ SensorsDataErrorCode.UNSUPPORTED_RECORD_TYPE,
+ "Unsupported : " + config.getEntityName());
+ }
+ }
+
+ private Builder(SensorsDataRecordType recordType, RowAccessor rowAccessor) {
+ this.rowAccessor = rowAccessor;
+ this.recordType = recordType;
+ switch (recordType) {
+ case USER:
+ case USER_EVENT:
+ case USER_DETAIL:
+ this.userRecordBuilder = UserRecordBase.newBuilder(rowAccessor);
+ break;
+ case SPECIAL_ITEM:
+ this.specialItemRecordBuilder = SpecialItemRecord.newBuilder(rowAccessor);
+ break;
+ default:
+ throw new SensorsDataException(
+ SensorsDataErrorCode.UNSUPPORTED_RECORD_TYPE,
+ "Unsupported Record Type: " + recordType);
+ }
+ }
+
+ public SensorsDataRecord build(SeaTunnelRow row) {
+ switch (recordType) {
+ case USER:
+ return this.userRecordBuilder.buildUserRecord(row);
+ case USER_EVENT:
+ return this.userRecordBuilder.buildUserEventRecord(row);
+ case USER_DETAIL:
+ return this.userRecordBuilder.buildUserDetailRecord(row);
+ case SPECIAL_ITEM:
+ return this.specialItemRecordBuilder.build(row);
+ default:
+ throw new SensorsDataException(
+ SensorsDataErrorCode.UNSUPPORTED_RECORD_TYPE,
+ "Unsupported Record Type: " + recordType);
+ }
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataRecordType.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataRecordType.java
new file mode 100644
index 0000000000..7127e086c1
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataRecordType.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+public enum SensorsDataRecordType {
+ // sensorsdata users table
+ USER,
+ // sensorsdata user events table
+ USER_EVENT,
+ // sensorsdata details table
+ USER_DETAIL,
+ /**
+ * Item Table Compatible with Non-SDH Architecture (Only Dual Primary Key Main Table, Due to Its
+ * Special Nature, It Needs to Be a Separate Type)
+ */
+ SPECIAL_ITEM,
+ /** Not Implemented Yet */
+ ITEM,
+ /** Not Implemented Yet */
+ ITEM_EVENT,
+ /** Not Implemented Yet */
+ ITEM_DETAIL
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SpecialItemRecord.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SpecialItemRecord.java
new file mode 100644
index 0000000000..fa804fcc5e
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SpecialItemRecord.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import org.apache.seatunnel.shade.com.fasterxml.jackson.core.JsonProcessingException;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataErrorCode;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.sensorsdata.analytics.javasdk.SensorsConst;
+import com.sensorsdata.analytics.javasdk.bean.ItemRecord;
+import com.sensorsdata.analytics.javasdk.exceptions.InvalidArgumentException;
+import lombok.Getter;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import static com.sensorsdata.analytics.javasdk.SensorsConst.ITEM_SET_ACTION_TYPE;
+import static org.apache.seatunnel.connectors.sensorsdata.format.record.UserRecordBase.OBJECT_MAPPER;
+
+public class SpecialItemRecord implements SensorsDataRecord {
+
+ @Getter private ItemRecord itemRecord;
+
+ private SpecialItemRecord(ItemRecord itemRecord) {
+ this.itemRecord = itemRecord;
+ }
+
+ private Map<String, Object> toMap() {
+ Map<String, Object> data = new HashMap<>();
+ data.put(SensorsDataJsonKeys.TYPE, ITEM_SET_ACTION_TYPE);
+ data.put(SensorsDataJsonKeys.LIB, SensorsDataLibInfo.LIB_INFO);
+
+ data.put(SensorsDataJsonKeys.ITEM_ID, itemRecord.getItemId());
+ data.put(SensorsDataJsonKeys.ITEM_TYPE, itemRecord.getItemType());
+
+ Date time =
+ itemRecord.getPropertyMap().containsKey(SensorsConst.TIME_SYSTEM_ATTR)
+ ? (Date) itemRecord.getPropertyMap().remove(SensorsConst.TIME_SYSTEM_ATTR)
+ : new Date();
+ data.put(SensorsDataJsonKeys.TIME, time.getTime());
+
+ String project =
+ itemRecord.getPropertyMap().get(SensorsConst.PROJECT_SYSTEM_ATTR) == null
+ ? null
+ : String.valueOf(
+ itemRecord
+ .getPropertyMap()
+ .remove(SensorsConst.PROJECT_SYSTEM_ATTR));
+ if (StringUtils.isNotEmpty(project)) {
+ data.put(SensorsDataJsonKeys.PROJECT, project);
+ }
+
+ String token =
+ itemRecord.getPropertyMap().get(SensorsConst.TOKEN_SYSTEM_ATTR) == null
+ ? null
+ : String.valueOf(
+ itemRecord.getPropertyMap().remove(SensorsConst.TOKEN_SYSTEM_ATTR));
+ if (StringUtils.isNotEmpty(token)) {
+ data.put(SensorsDataJsonKeys.TOKEN, token);
+ }
+
+ data.put(SensorsDataJsonKeys.PROPERTIES, itemRecord.getPropertyMap());
+ return data;
+ }
+
+ @Override
+ public String toJsonString() {
+ try {
+ return OBJECT_MAPPER.writeValueAsString(this.toMap());
+ } catch (JsonProcessingException e) {
+ return null;
+ }
+ }
+
+ public static Builder newBuilder(RowAccessor rowAccessor) {
+ return new Builder(rowAccessor);
+ }
+
+ public static class Builder {
+
+ private final RowAccessor rowAccessor;
+
+ private Builder(RowAccessor rowAccessor) {
+ this.rowAccessor = rowAccessor;
+ }
+
+ public SpecialItemRecord build(SeaTunnelRow row) {
+ try {
+ return new SpecialItemRecord(
+ ItemRecord.builder()
+ .setItemId(rowAccessor.getItemIdRequired(row))
+ .setItemType(rowAccessor.getItemTypeRequired(row))
+ .addProperties(rowAccessor.getProperties(row))
+ .build());
+ } catch (InvalidArgumentException e) {
+ throw new SensorsDataException(
+ SensorsDataErrorCode.ILLEGAL_ARGUMENT, e.getMessage());
+ }
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserDetailRecord.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserDetailRecord.java
new file mode 100644
index 0000000000..2fb2e67262
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserDetailRecord.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import com.sensorsdata.analytics.javasdk.bean.schema.DetailSchema;
+import lombok.Getter;
+
+import java.util.Map;
+
+import static com.sensorsdata.analytics.javasdk.SensorsConst.DETAIL_SET_ACTION_TYPE;
+
+public class UserDetailRecord extends UserRecordBase {
+
+ @Getter private final DetailSchema userDetailSchema;
+
+ public UserDetailRecord(DetailSchema userDetailSchema) {
+ super(
+ userDetailSchema.getTrackId(),
+ userDetailSchema.getDistinctId(),
+ userDetailSchema.getIdentities(),
+ userDetailSchema.getProperties(),
+ DETAIL_SET_ACTION_TYPE,
+ userDetailSchema.getSchema());
+ this.userDetailSchema = userDetailSchema;
+ }
+
+ protected Map<String, Object> toMap() {
+ Map<String, Object> data = super.toMapWithOutProperties();
+ data.put(SensorsDataJsonKeys.ID, userDetailSchema.getDetailId());
+ Map<String, Object> properties = this.userDetailSchema.getProperties();
+ if (userDetailSchema.getItemPair() != null) {
+ properties.put(
+ userDetailSchema.getItemPair().getKey(),
+ userDetailSchema.getItemPair().getValue());
+ }
+ if (!userDetailSchema.getIdentities().isEmpty()) {
+ checkAndSetIdentity(properties);
+ }
+ data.put(SensorsDataJsonKeys.PROPERTIES, properties);
+ return data;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserEventRecord.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserEventRecord.java
new file mode 100644
index 0000000000..afd1576c89
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserEventRecord.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import com.sensorsdata.analytics.javasdk.SensorsConst;
+import com.sensorsdata.analytics.javasdk.bean.schema.UserEventSchema;
+import lombok.Getter;
+
+import java.util.Map;
+
+import static com.sensorsdata.analytics.javasdk.SensorsConst.TRACK_ACTION_TYPE;
+
+public class UserEventRecord extends UserRecordBase {
+
+ private String eventName;
+
+ @Getter private UserEventSchema userEventSchema;
+
+ public UserEventRecord(UserEventSchema userEventSchema) {
+ super(
+ userEventSchema.getTrackId(),
+ userEventSchema.getDistinctId(),
+ userEventSchema.getIdentityMap(),
+ userEventSchema.getPropertyMap(),
+ TRACK_ACTION_TYPE,
+ SensorsConst.USER_EVENT_SCHEMA);
+ this.userEventSchema = userEventSchema;
+ this.eventName = userEventSchema.getEventName();
+ }
+
+ protected Map<String, Object> toMap() {
+ Map<String, Object> data = super.toMapWithOutProperties();
+ addTimeFree(data);
+ data.put(SensorsDataJsonKeys.EVENT, eventName);
+ Map<String, Object> properties = this.userEventSchema.getPropertyMap();
+ checkAndSetIdentity(properties);
+ data.put(SensorsDataJsonKeys.PROPERTIES, properties);
+ return data;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserRecord.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserRecord.java
new file mode 100644
index 0000000000..449a1339f8
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserRecord.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import com.sensorsdata.analytics.javasdk.SensorsConst;
+import com.sensorsdata.analytics.javasdk.bean.schema.UserSchema;
+import lombok.Getter;
+
+import java.util.Map;
+
+import static com.sensorsdata.analytics.javasdk.SensorsConst.PROFILE_SET_ACTION_TYPE;
+
+public class UserRecord extends UserRecordBase {
+
+ @Getter private final UserSchema userSchema;
+
+ public UserRecord(UserSchema userSchema) {
+ this(userSchema, PROFILE_SET_ACTION_TYPE);
+ }
+
+ public UserRecord(UserSchema userSchema, String actionType) {
+ super(
+ userSchema.getTrackId(),
+ userSchema.getDistinctId(),
+ userSchema.getIdentityMap(),
+ userSchema.getPropertyMap(),
+ actionType,
+ SensorsConst.USER_SCHEMA);
+ this.userSchema = userSchema;
+ }
+
+ protected Map<String, Object> toMap() {
+ Map<String, Object> data = super.toMapWithOutProperties();
+ checkAndSetIdentity(data);
+ data.put(SensorsDataJsonKeys.PROPERTIES, userSchema.getPropertyMap());
+ return data;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserRecordBase.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserRecordBase.java
new file mode 100644
index 0000000000..610193b13b
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/record/UserRecordBase.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import org.apache.seatunnel.shade.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataErrorCode;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.sensorsdata.analytics.javasdk.SensorsConst;
+import com.sensorsdata.analytics.javasdk.bean.schema.DetailSchema;
+import com.sensorsdata.analytics.javasdk.bean.schema.UserEventSchema;
+import com.sensorsdata.analytics.javasdk.bean.schema.UserSchema;
+import com.sensorsdata.analytics.javasdk.exceptions.InvalidArgumentException;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import static com.sensorsdata.analytics.javasdk.SensorsConst.TRACK_ACTION_TYPE;
+
+public abstract class UserRecordBase implements SensorsDataRecord {
+
+ protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private Integer trackId;
+
+ private String distinctId;
+
+ private Map<String, Object> identities;
+
+ private String type;
+
+ private Date time;
+
+ /** property name and value */
+ private Map<String, Object> properties;
+
+ /** project name of sensorsdata */
+ private String project;
+
+ private String token;
+
+ private boolean timeFree = false;
+
+ private String schema;
+
+ public String getType() {
+ return type;
+ }
+
+ protected UserRecordBase(
+ Integer trackId,
+ String distinctId,
+ Map<String, Object> identities,
+ Map<String, Object> properties,
+ String type,
+ String schema) {
+ initBasicFields(distinctId, identities, type, properties, trackId, schema);
+ }
+
+ protected void initBasicFields(
+ String distinctId,
+ Map<String, Object> identities,
+ String type,
+ Map<String, Object> properties,
+ Integer trackId,
+ String schema) {
+ this.trackId = trackId;
+ this.distinctId = distinctId;
+ this.identities = identities;
+ this.type = type;
+ this.schema = schema;
+ this.time =
+ properties.containsKey(SensorsConst.TIME_SYSTEM_ATTR)
+ ? (Date) properties.get(SensorsConst.TIME_SYSTEM_ATTR)
+ : new Date();
+ this.properties = properties;
+ this.project =
+ properties.get(SensorsConst.PROJECT_SYSTEM_ATTR) == null
+ ? null
+ : String.valueOf(properties.get(SensorsConst.PROJECT_SYSTEM_ATTR));
+ this.token =
+ properties.get(SensorsConst.TOKEN_SYSTEM_ATTR) == null
+ ? null
+ : String.valueOf(properties.get(SensorsConst.TOKEN_SYSTEM_ATTR));
+ this.timeFree =
+ properties.containsKey(SensorsConst.TIME_FREE_ATTR)
+ && Boolean.parseBoolean(
+ properties.get(SensorsConst.TIME_FREE_ATTR).toString());
+ }
+
+ protected Map<String, Object> toMapWithOutProperties() {
+ Map<String, Object> data = new HashMap<>();
+ data.put(SensorsDataJsonKeys.TRACK_ID, trackId);
+ data.put(SensorsDataJsonKeys.VERSION, SensorsConst.PROTOCOL_VERSION);
+ data.put(SensorsDataJsonKeys.TYPE, type);
+ data.put(SensorsDataJsonKeys.SCHEMA, schema);
+ data.put(SensorsDataJsonKeys.LIB, SensorsDataLibInfo.LIB_INFO);
+ data.put(SensorsDataJsonKeys.TIME, time.getTime());
+ if (StringUtils.isNotEmpty(project)) {
+ data.put(SensorsDataJsonKeys.PROJECT, project);
+ }
+ if (StringUtils.isNotEmpty(token)) {
+ data.put(SensorsDataJsonKeys.TOKEN, token);
+ }
+
+ return data;
+ }
+
+ protected abstract Map<String, Object> toMap();
+
+ @Override
+ public String toJsonString() {
+ try {
+ return OBJECT_MAPPER.writeValueAsString(this.toMap());
+ } catch (JsonProcessingException e) {
+ return null;
+ }
+ }
+
+ protected void addTimeFree(Map<String, Object> data) {
+ if (!timeFree) {
+ return;
+ }
+ if (StringUtils.equals(type, TRACK_ACTION_TYPE)) {
+ data.put(SensorsDataJsonKeys.TIME_FREE, true);
+ }
+ }
+
+ protected void checkAndSetIdentity(Map<String, Object> data) {
+ if (null != identities && !identities.isEmpty()) {
+ data.put(SensorsDataJsonKeys.IDENTITIES, identities);
+ }
+ if (StringUtils.isNotEmpty(distinctId)) {
+ data.put(SensorsDataJsonKeys.DISTINCT_ID, distinctId);
+ }
+ }
+
+ public static Builder newBuilder(RowAccessor rowAccessor) {
+ return new Builder(rowAccessor);
+ }
+
+ public static class Builder {
+
+ private final RowAccessor rowAccessor;
+
+ private Builder(RowAccessor rowAccessor) {
+ this.rowAccessor = rowAccessor;
+ }
+
+ public UserEventRecord buildUserEventRecord(SeaTunnelRow row) {
+ try {
+ return new UserEventRecord(
+ UserEventSchema.init()
+ .setEventName(rowAccessor.getEventName(row))
+ .setDistinctId(rowAccessor.getDistinctId(row))
+ .identityMap(rowAccessor.getIdentities(row))
+ .addProperties(rowAccessor.getProperties(row))
+ .start());
+ } catch (InvalidArgumentException e) {
+ throw new SensorsDataException(
+ SensorsDataErrorCode.ILLEGAL_ARGUMENT, e.getMessage());
+ }
+ }
+
+ public UserDetailRecord buildUserDetailRecord(SeaTunnelRow row) {
+ try {
+ return new UserDetailRecord(
+ DetailSchema.init()
+ .setSchema(rowAccessor.getSchemaRequired())
+ .setDetailId(rowAccessor.getDetailIdRequired(row))
+ .setDistinctId(rowAccessor.getDistinctId(row))
+ .identityMap(rowAccessor.getIdentities(row))
+ .addProperties(rowAccessor.getProperties(row))
+ .start());
+ } catch (InvalidArgumentException e) {
+ throw new SensorsDataException(
+ SensorsDataErrorCode.ILLEGAL_ARGUMENT, e.getMessage());
+ }
+ }
+
+ public UserRecord buildUserRecord(SeaTunnelRow row) {
+ try {
+
+ return new UserRecord(
+ UserSchema.init()
+ .setDistinctId(rowAccessor.getDistinctId(row))
+ .identityMap(rowAccessor.getUserIdentities(row))
+ .addProperties(rowAccessor.getProperties(row))
+ .start());
+ } catch (InvalidArgumentException e) {
+ throw new SensorsDataException(
+ SensorsDataErrorCode.ILLEGAL_ARGUMENT, e.getMessage());
+ }
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/utils/TypeUtil.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/utils/TypeUtil.java
new file mode 100644
index 0000000000..09753c19c2
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/utils/TypeUtil.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.utils;
+
+import org.apache.seatunnel.shade.com.google.common.base.Objects;
+
+import org.apache.seatunnel.connectors.sensorsdata.format.SensorsDataTypes;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataErrorCode;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataException;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Slf4j
+@UtilityClass
+public class TypeUtil {
+
+ public static final DateTimeFormatter FULL_DATETIME_FORMATTER =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withZone(ZoneId.systemDefault());
+
+ public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd";
+ public static final DateTimeFormatter DEFAULT_DATE_FORMATTER =
+ DateTimeFormatter.ofPattern(DEFAULT_DATE_FORMAT).withZone(ZoneId.systemDefault());
+ public static final String DEFAULT_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
+ public static final DateTimeFormatter DEFAULT_DATETIME_FORMATTER =
+ DateTimeFormatter.ofPattern(DEFAULT_DATETIME_FORMAT).withZone(ZoneId.systemDefault());
+
+ public static final DateTimeFormatter SHORT_DATETIME_FORMATTER =
+ DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss").withZone(ZoneId.systemDefault());
+
+ public static final DateTimeFormatter SHORT_DAY_FORMATTER =
+ DateTimeFormatter.ofPattern("yyyyMMdd").withZone(ZoneId.systemDefault());
+ public static final DateTimeFormatter SHORT_DAY_HOUR_FORMATTER =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").withZone(ZoneId.systemDefault());
+
+ /**
+ * ORDER IS IMPORTANT Must strictly control the order of this array, otherwise it may cause
+ * DateUtil/DateTimeUtil's tryParse function to fail
+ */
+ public static final DateTimeFormatter[] INTERNAL_DATETIME_FORMATS =
+ new DateTimeFormatter[] {
+ FULL_DATETIME_FORMATTER,
+ DEFAULT_DATETIME_FORMATTER,
+ SHORT_DAY_HOUR_FORMATTER,
+ DEFAULT_DATE_FORMATTER,
+ SHORT_DATETIME_FORMATTER,
+ SHORT_DAY_FORMATTER
+ };
+
+ private static final String TRANSFORM_WARN_INFO =
+ "convert target data type error. source:{}, targetType:{}";
+
+ private static final Map<String, DateTimeFormatter> DATE_TIME_FORMATTER_MAP = new HashMap<>();
+
+ /**
+ * Write sensorsdata-inf-sdk data type logic
+ *
+ * <p>Since inf-sdk writing unsupported data types will cause an error, it is necessary to
+ * validate the data type at the beginning, and then insert it; so here supports the following
+ * data type conversion: bool: support boolean/number type/boolean string data/timestamp:
+ * support date/timestamp/ yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "yyyyMMdd", "yyyyMMdd HHmmss four
+ * date strings BigInt: support int/long/able to convert to long type string DECIMAL: support
+ * number/able to convert to decimal type string int: support int/able to convert to int type
+ * string number: support number/able to convert to number type string string: no additional
+ * processing list: not additional processing
+ */
+ public static Object toTargetType(Object source, String targetType) {
+ if (null == source || StringUtils.isBlank(targetType)) {
+ return source;
+ }
+ SensorsDataTypes type = SensorsDataTypes.of(targetType);
+ return toTargetType(source, type.getType(), type.getExtra());
+ }
+
+ public static Object toTargetType(Object source, SensorsDataTypes.DataTypes targetType) {
+ return toTargetType(source, targetType, null);
+ }
+
+ public static Object toTargetType(
+ Object source, SensorsDataTypes.DataTypes targetType, String extra) {
+ if (source == null) {
+ return null;
+ }
+ switch (targetType) {
+ case BOOLEAN:
+ return toBoolean(source, targetType);
+ case DECIMAL:
+ return toBigDecimal(source, targetType);
+ case INT:
+ case BIGINT:
+ case FLOAT:
+ case DOUBLE:
+ case NUMBER:
+ return toNumber(source, targetType);
+ case LIST:
+ return toList(source, '\n');
+ case LIST_COMMA:
+ return toList(source, ',');
+ case LIST_SEMICOLON:
+ return toList(source, ';');
+ case TIMESTAMP:
+ return toTimestamp(source, targetType, extra);
+ case DATE:
+ return toDate(source, targetType);
+ case STRING:
+ default:
+ return toString(source);
+ }
+ }
+
+ private static List<String> toList(Object str, char sep) {
+ if (str instanceof String) {
+ return Arrays.asList(StringUtils.split((String) str, sep));
+ } else {
+ throw new SensorsDataException(
+ SensorsDataErrorCode.DATA_TYPE_CAST_FILED,
+ "Value type must be STRING when target column type is LIST.");
+ }
+ }
+
+ private static Object toTimestamp(
+ Object source, SensorsDataTypes.DataTypes targetType, String format) {
+ if (source instanceof Date) {
+ return ((Date) source).getTime();
+ }
+ if (source instanceof Number) {
+ return source;
+ }
+ if (source instanceof LocalDate) {
+ return ((LocalDate) source)
+ .atStartOfDay(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli();
+ }
+ if (source instanceof LocalDateTime) {
+ return ((LocalDateTime) source)
+ .atZone(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli();
+ }
+ if (source instanceof String) {
+ Long timestamp;
+ if (format == null) {
+ timestamp = tryParse((String) source);
+ } else {
+ DateTimeFormatter formatter = parseDateTimeFormatter(format);
+ timestamp = tryParse((String) source, formatter);
+ }
+ if (timestamp != null) {
+ return timestamp;
+ }
+ }
+ log.warn(TRANSFORM_WARN_INFO, source, targetType);
+ return source;
+ }
+
+ private static Object toBoolean(Object source, SensorsDataTypes.DataTypes targetType) {
+ if (source instanceof Boolean) {
+ return source;
+ }
+ if (source instanceof Number) {
+ return !Objects.equal(0, source)
+ && !Objects.equal(0F, source)
+ && !Objects.equal(0D, source)
+ && !Objects.equal(0L, source);
+ }
+ if (source instanceof String) {
+ return StringUtils.equalsIgnoreCase("true", source.toString());
+ }
+ log.warn(TRANSFORM_WARN_INFO, source, targetType);
+ return source;
+ }
+
+ private static Object toBigDecimal(Object source, SensorsDataTypes.DataTypes targetType) {
+ if (source instanceof String) {
+ try {
+ return NumberUtils.createBigDecimal(source.toString());
+ } catch (Exception e) {
+ log.warn(TRANSFORM_WARN_INFO, source, targetType);
+ }
+ } else if (source instanceof Boolean) {
+ return BigDecimal.valueOf(Boolean.TRUE.equals(source) ? 1 : 0);
+ }
+ return source;
+ }
+
+ private static Object toNumber(Object source, SensorsDataTypes.DataTypes targetType) {
+ if (source instanceof Number) {
+ return source;
+ }
+ if (source instanceof String) {
+ try {
+ return NumberUtils.createNumber(source.toString());
+ } catch (Exception e) {
+ log.warn(TRANSFORM_WARN_INFO, source, targetType);
+ }
+ }
+ if (source instanceof Boolean) {
+ return Boolean.TRUE.equals(source) ? 1 : 0;
+ }
+ return source;
+ }
+
+ private static String toString(Object source) {
+ if (source instanceof byte[]) {
+ return new String((byte[]) source);
+ }
+ return source.toString();
+ }
+
+ private static Long tryParse(String str) {
+ for (DateTimeFormatter formatter : INTERNAL_DATETIME_FORMATS) {
+ Long timestamp = tryParse(str, formatter);
+ if (timestamp != null) {
+ return timestamp;
+ }
+ }
+ return null;
+ }
+
+ private static Long tryParse(String str, DateTimeFormatter formatter) {
+ // Since parse fails, it will return null, and the outside world should have some
+ // expectations for this method to return null
+ // But in the process of loop parsing, only ParseException is processed, so the null value
+ // passed in is separated for processing to prevent NPE
+ if (StringUtils.isBlank(str)) {
+ return null;
+ }
+ ZonedDateTime time;
+ try {
+ time = ZonedDateTime.from(formatter.parse(str));
+ return time.toInstant().toEpochMilli();
+ } catch (DateTimeParseException e) {
+ // This error should be ignored
+ log.debug("Failed to parse date time. [str='{}', formatter='{}']", str, formatter, e);
+ return null;
+ }
+ }
+
+ private static DateTimeFormatter parseDateTimeFormatter(String str) {
+ return DATE_TIME_FORMATTER_MAP.computeIfAbsent(str, k -> DateTimeFormatter.ofPattern(str));
+ }
+
+ private static Object toDate(Object source, SensorsDataTypes.DataTypes targetType) {
+ if (source instanceof Date) {
+ return source;
+ }
+ if (source instanceof Number) {
+ return new Date((long) source);
+ }
+ if (source instanceof LocalDate) {
+ return Date.from(((LocalDate) source).atStartOfDay(ZoneId.systemDefault()).toInstant());
+ }
+ if (source instanceof LocalDateTime) {
+ return Date.from(((LocalDateTime) source).atZone(ZoneId.systemDefault()).toInstant());
+ }
+ if (source instanceof String) {
+ Long timestamp = tryParse((String) source);
+ if (timestamp != null) {
+ return new Date(timestamp);
+ }
+ }
+ log.warn(TRANSFORM_WARN_INFO, source, targetType);
+ return source;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/utils/UserSchemaUtil.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/utils/UserSchemaUtil.java
new file mode 100644
index 0000000000..1c75429f45
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/format/utils/UserSchemaUtil.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.utils;
+
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataErrorCode;
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataException;
+
+import org.apache.commons.collections4.MapUtils;
+
+import com.sensorsdata.analytics.javasdk.bean.schema.UserSchema;
+import com.sensorsdata.analytics.javasdk.exceptions.InvalidArgumentException;
+import lombok.experimental.UtilityClass;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static com.sensorsdata.analytics.javasdk.SensorsConst.PROJECT_SYSTEM_ATTR;
+
+@UtilityClass
+public class UserSchemaUtil {
+
+ public UserSchema buildUnsetUserSchema(UserSchema userSchema, Set<String> allProperties) {
+ try {
+ Map<String, Object> propertyMap =
+ buildUnsetPropertyMap(userSchema.getPropertyMap(), allProperties);
+ // If the propertyMap is empty, no need to build userSchema
+ // Because if there are no properties in userSchema, it means that there is no need to
+ // perform unset operation
+ if (MapUtils.isEmpty(propertyMap)) {
+ return null;
+ }
+ return UserSchema.init()
+ .setDistinctId(userSchema.getDistinctId())
+ .identityMap(userSchema.getIdentityMap())
+ .addProperties(propertyMap)
+ .start();
+ } catch (InvalidArgumentException e) {
+ throw new SensorsDataException(SensorsDataErrorCode.ILLEGAL_ARGUMENT, e.getMessage());
+ }
+ }
+
+ public Map<String, Object> buildUnsetPropertyMap(
+ Map<String, Object> propertyMap, Set<String> allProperties) {
+ Map<String, Object> unsetMap = new HashMap<>();
+ if (MapUtils.isNotEmpty(propertyMap)) {
+ for (Map.Entry<String, Object> entry : propertyMap.entrySet()) {
+ if (entry.getValue() == null && !PROJECT_SYSTEM_ATTR.equals(entry.getKey())) {
+ unsetMap.put(entry.getKey(), Boolean.TRUE);
+ }
+ }
+ } else {
+ propertyMap = new HashMap<>();
+ }
+ // If the corresponding property is not read, complete the unset list
+ Set<String> dataProperties = propertyMap.keySet();
+ allProperties.forEach(
+ name -> {
+ if (!dataProperties.contains(name) && !PROJECT_SYSTEM_ATTR.equals(name)) {
+ unsetMap.put(name, Boolean.TRUE);
+ }
+ });
+ return unsetMap;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/config/SensorsDataSDKSinkConfig.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/config/SensorsDataSDKSinkConfig.java
new file mode 100644
index 0000000000..0cbb769d5d
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/config/SensorsDataSDKSinkConfig.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk.config;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.SensorsDataConfigBase;
+
+import lombok.Getter;
+import lombok.ToString;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+@Getter
+@ToString
+public class SensorsDataSDKSinkConfig extends SensorsDataConfigBase {
+
+ private final String serverUrl;
+ private final int bulkSize;
+ private final int maxCacheRowSize;
+ private final String consumer;
+ private final List<String> instantEvents;
+
+ public SensorsDataSDKSinkConfig(ReadonlyConfig config) {
+ super(config);
+ // sensorsdata server
+ this.serverUrl = config.get(SensorsDataSDKSinkOptions.SERVER_URL);
+ this.bulkSize = config.get(SensorsDataSDKSinkOptions.BULK_SIZE);
+ this.maxCacheRowSize = config.get(SensorsDataSDKSinkOptions.MAX_CACHE_ROW_SIZE);
+ this.consumer = config.get(SensorsDataSDKSinkOptions.CONSUMER);
+ this.instantEvents =
+ Optional.ofNullable(config.get(SensorsDataSDKSinkOptions.INSTANT_EVENT_LIST))
+ .orElse(new ArrayList<>());
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/config/SensorsDataSDKSinkOptions.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/config/SensorsDataSDKSinkOptions.java
new file mode 100644
index 0000000000..406ad026c3
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/config/SensorsDataSDKSinkOptions.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk.config;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.SensorsDataOptions;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@SuppressWarnings("checkstyle:MagicNumber")
+public interface SensorsDataSDKSinkOptions extends SensorsDataOptions {
+
+ Option<String> SERVER_URL =
+ Options.key("server_url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Format:https://{ip}:8106/sa?project={project}");
+
+ Option<Integer> BULK_SIZE =
+ Options.key("bulk_size")
+ .intType()
+ .defaultValue(50)
+ .withDescription(
+ "Threshold for triggering flush operation. When the memory cache queue reaches this value, "
+ + "the data in the cache will be batch uploaded.");
+
+ Option<Integer> MAX_CACHE_ROW_SIZE =
+ Options.key("max_cache_row_size")
+ .intType()
+ .defaultValue(0)
+ .withDescription(
+ "Maximum cache refresh size. If it exceeds this value, the flush operation will "
+ + "be triggered immediately. The default value is 0, which depends on bulkSize.");
+
+ Option<String> CONSUMER =
+ Options.key("consumer")
+ .stringType()
+ .defaultValue("batch")
+ .withDescription("batch/console");
+
+ Option<List<String>> INSTANT_EVENT_LIST =
+ Options.key("instant_events")
+ .listType()
+ .defaultValue(new ArrayList<>())
+ .withDescription(
+ "Given a list of event names, mark the event as an instant event.");
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/exception/SensorsDataConnectorErrorCode.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/exception/SensorsDataConnectorErrorCode.java
new file mode 100644
index 0000000000..8dbe328cc3
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/exception/SensorsDataConnectorErrorCode.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk.exception;
+
+import org.apache.seatunnel.common.exception.SeaTunnelErrorCode;
+
+public enum SensorsDataConnectorErrorCode implements SeaTunnelErrorCode {
+ SEND_RECORD_FAILED("SENSORS_DATA-01", "Send record failed"),
+ UNKNOWN_RECORD_TYPE("SENSORS_DATA-02", "Unknown record type"),
+ UNSUPPORTED_RECORD_TYPE("SENSORS_DATA-03", "Unsupported record type"),
+ ;
+
+ private final String code;
+
+ private final String description;
+
+ SensorsDataConnectorErrorCode(String code, String description) {
+ this.code = code;
+ this.description = description;
+ }
+
+ @Override
+ public String getCode() {
+ return this.code;
+ }
+
+ @Override
+ public String getDescription() {
+ return this.description;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/exception/SensorsDataConnectorException.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/exception/SensorsDataConnectorException.java
new file mode 100644
index 0000000000..7649e83ce8
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/exception/SensorsDataConnectorException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk.exception;
+
+import org.apache.seatunnel.common.exception.SeaTunnelErrorCode;
+import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException;
+
+public class SensorsDataConnectorException extends SeaTunnelRuntimeException {
+ public SensorsDataConnectorException(
+ SeaTunnelErrorCode seaTunnelErrorCode, String errorMessage) {
+ super(seaTunnelErrorCode, errorMessage);
+ }
+
+ public SensorsDataConnectorException(
+ SeaTunnelErrorCode seaTunnelErrorCode, String errorMessage, Throwable cause) {
+ super(seaTunnelErrorCode, errorMessage, cause);
+ }
+
+ public SensorsDataConnectorException(SeaTunnelErrorCode seaTunnelErrorCode, Throwable cause) {
+ super(seaTunnelErrorCode, cause);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/sink/SensorsDataSDKSink.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/sink/SensorsDataSDKSink.java
new file mode 100644
index 0000000000..5ba3e9584f
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/sink/SensorsDataSDKSink.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk.sink;
+
+import org.apache.seatunnel.api.sink.SeaTunnelSink;
+import org.apache.seatunnel.api.sink.SinkWriter;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSimpleSink;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.config.SensorsDataSDKSinkConfig;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.state.SensorsDataAggregatedCommitInfo;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.state.SensorsDataCommitInfo;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.state.SensorsDataSinkState;
+
+import java.util.Optional;
+
+/**
+ * Sensors Data Sink implementation by using SeaTunnel sink API. This class contains the method to
+ * create {@link AbstractSimpleSink}.
+ */
+public class SensorsDataSDKSink
+ implements SeaTunnelSink<
+ SeaTunnelRow,
+ SensorsDataSinkState,
+ SensorsDataCommitInfo,
+ SensorsDataAggregatedCommitInfo> {
+
+ private final SensorsDataSDKSinkConfig sinkConfig;
+ private final SeaTunnelRowType seaTunnelRowType;
+
+ private final CatalogTable catalogTable;
+
+ public SensorsDataSDKSink(SensorsDataSDKSinkConfig sinkConfig, CatalogTable catalogTable) {
+ this.catalogTable = catalogTable;
+ this.sinkConfig = sinkConfig;
+ this.seaTunnelRowType = catalogTable.getTableSchema().toPhysicalRowDataType();
+ }
+
+ @Override
+ public String getPluginName() {
+ return "SensorsData";
+ }
+
+ @Override
+ public SinkWriter<SeaTunnelRow, SensorsDataCommitInfo, SensorsDataSinkState> createWriter(
+ SinkWriter.Context context) {
+ return new SensorsDataSDKWriter(seaTunnelRowType, sinkConfig);
+ }
+
+ @Override
+ public Optional<CatalogTable> getWriteCatalogTable() {
+ return Optional.ofNullable(catalogTable);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/sink/SensorsDataSDKSinkFactory.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/sink/SensorsDataSDKSinkFactory.java
new file mode 100644
index 0000000000..d965a4e400
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/sink/SensorsDataSDKSinkFactory.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk.sink;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.connector.TableSink;
+import org.apache.seatunnel.api.table.factory.Factory;
+import org.apache.seatunnel.api.table.factory.TableSinkFactory;
+import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.SensorsDataBaseOptionRules;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.SensorsDataOptions;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.config.SensorsDataSDKSinkConfig;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.config.SensorsDataSDKSinkOptions;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(Factory.class)
+public class SensorsDataSDKSinkFactory implements TableSinkFactory {
+ @Override
+ public String factoryIdentifier() {
+ return "SensorsData";
+ }
+
+ @Override
+ public OptionRule optionRule() {
+ return SensorsDataBaseOptionRules.getBaseOptionRuleBuilder()
+ .optional(
+ SensorsDataSDKSinkOptions.BULK_SIZE,
+ SensorsDataSDKSinkOptions.MAX_CACHE_ROW_SIZE,
+ SensorsDataOptions.SKIP_ERROR_RECORD,
+ SensorsDataSDKSinkOptions.INSTANT_EVENT_LIST)
+ .build();
+ }
+
+ @Override
+ public TableSink createSink(TableSinkFactoryContext context) {
+ ReadonlyConfig config = context.getOptions();
+ CatalogTable catalogTable = context.getCatalogTable();
+ SensorsDataSDKSinkConfig sinkConfig = new SensorsDataSDKSinkConfig(config);
+ return () -> new SensorsDataSDKSink(sinkConfig, catalogTable);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/sink/SensorsDataSDKWriter.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/sink/SensorsDataSDKWriter.java
new file mode 100644
index 0000000000..3ddfbf0a80
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/sink/SensorsDataSDKWriter.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk.sink;
+
+import org.apache.seatunnel.api.sink.SinkWriter;
+import org.apache.seatunnel.api.sink.SupportMultiTableSinkWriter;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.common.utils.JsonUtils;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.TargetColumnConfig;
+import org.apache.seatunnel.connectors.sensorsdata.format.record.RowAccessor;
+import org.apache.seatunnel.connectors.sensorsdata.format.record.SensorsDataRecordBuilder;
+import org.apache.seatunnel.connectors.sensorsdata.format.record.SpecialItemRecord;
+import org.apache.seatunnel.connectors.sensorsdata.format.record.UserDetailRecord;
+import org.apache.seatunnel.connectors.sensorsdata.format.record.UserEventRecord;
+import org.apache.seatunnel.connectors.sensorsdata.format.record.UserRecord;
+import org.apache.seatunnel.connectors.sensorsdata.format.utils.UserSchemaUtil;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.config.SensorsDataSDKSinkConfig;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.exception.SensorsDataConnectorErrorCode;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.exception.SensorsDataConnectorException;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.state.SensorsDataCommitInfo;
+import org.apache.seatunnel.connectors.sensorsdata.sdk.state.SensorsDataSinkState;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.sensorsdata.analytics.javasdk.SensorsAnalytics;
+import com.sensorsdata.analytics.javasdk.bean.schema.UserSchema;
+import com.sensorsdata.analytics.javasdk.consumer.BatchConsumer;
+import com.sensorsdata.analytics.javasdk.consumer.ConsoleConsumer;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class SensorsDataSDKWriter
+ implements SinkWriter<SeaTunnelRow, SensorsDataCommitInfo, SensorsDataSinkState>,
+ SupportMultiTableSinkWriter<Void> {
+
+ private final SensorsAnalytics sa;
+ private final RowAccessor rowAccessor;
+ private final SeaTunnelRowType seaTunnelRowType;
+ private final boolean isSkipErrorRecord;
+ private final boolean nullAsProfileUnset;
+ private final Set<String> allProperties;
+
+ /** for convenient testing */
+ private static final String CONSUMER_TYPE_CONSOLE = "console";
+
+ private final SensorsDataRecordBuilder.Builder recordBuilder;
+
+ public SensorsDataSDKWriter(
+ @NonNull SeaTunnelRowType seaTunnelRowType,
+ @NonNull SensorsDataSDKSinkConfig sinkConfig) {
+ if (CONSUMER_TYPE_CONSOLE.equalsIgnoreCase(sinkConfig.getConsumer())) {
+ sa = new SensorsAnalytics(new ConsoleConsumer(new PrintWriter(System.out)));
+ } else {
+ sa =
+ new SensorsAnalytics(
+ new BatchConsumer(
+ sinkConfig.getServerUrl(),
+ sinkConfig.getBulkSize(),
+ sinkConfig.getMaxCacheRowSize(),
+ false,
+ 3,
+ sinkConfig.getInstantEvents()));
+ }
+ sa.setEnableTimeFree(sinkConfig.isTimeFree());
+ rowAccessor = new RowAccessor(sinkConfig, seaTunnelRowType);
+ this.seaTunnelRowType = seaTunnelRowType;
+ isSkipErrorRecord = sinkConfig.isSkipErrorRecord();
+ nullAsProfileUnset = sinkConfig.isNullAsProfileUnset();
+ recordBuilder = SensorsDataRecordBuilder.newBuilder(sinkConfig, rowAccessor);
+ this.allProperties =
+ sinkConfig.getPropertyFields().stream()
+ .map(TargetColumnConfig::getTarget)
+ .collect(Collectors.toSet());
+ }
+
+ @Override
+ public void write(SeaTunnelRow row) throws IOException {
+ try {
+ switch (recordBuilder.getRecordType()) {
+ case USER:
+ UserSchema userSchema = ((UserRecord) recordBuilder.build(row)).getUserSchema();
+ sa.profileSet(userSchema);
+ if (nullAsProfileUnset) {
+ UserSchema unsetUserSchema =
+ UserSchemaUtil.buildUnsetUserSchema(userSchema, allProperties);
+ if (unsetUserSchema != null) {
+ // do not send profile_unset if all fields are not null
+ sa.profileUnset(unsetUserSchema);
+ }
+ }
+ break;
+ case USER_EVENT:
+ sa.track(((UserEventRecord) recordBuilder.build(row)).getUserEventSchema());
+ break;
+ case USER_DETAIL:
+ sa.detailSet(
+ ((UserDetailRecord) recordBuilder.build(row)).getUserDetailSchema());
+ break;
+ case SPECIAL_ITEM:
+ sa.itemSet(((SpecialItemRecord) recordBuilder.build(row)).getItemRecord());
+ break;
+ default:
+ throw new SensorsDataConnectorException(
+ SensorsDataConnectorErrorCode.UNSUPPORTED_RECORD_TYPE,
+ "Unsupported record type");
+ }
+ } catch (Exception e) {
+ log.error("Write error", e);
+ log.error(
+ "Write error, SeaTunnelRow#tableId={} SeaTunnelRow#kind={} : [{}]",
+ row.getTableId(),
+ row.getRowKind(),
+ fieldsToString(row));
+ if (!isSkipErrorRecord) {
+ throw new SensorsDataConnectorException(
+ SensorsDataConnectorErrorCode.SEND_RECORD_FAILED, e.getMessage(), e);
+ }
+ }
+ }
+
+ /** Convert the SeaTunnelRow data to a string */
+ private String fieldsToString(SeaTunnelRow row) {
+ String[] arr = new String[seaTunnelRowType.getTotalFields()];
+ SeaTunnelDataType<?>[] fieldTypes = seaTunnelRowType.getFieldTypes();
+ Object[] fields = row.getFields();
+ for (int i = 0; i < fieldTypes.length; i++) {
+ arr[i] = fieldToString(fieldTypes[i], fields[i]);
+ }
+ return StringUtils.join(arr, ", ");
+ }
+
+ /** copy from ConsoleSinkWriter */
+ private String fieldToString(SeaTunnelDataType<?> type, Object value) {
+ if (value == null) {
+ return null;
+ }
+ switch (type.getSqlType()) {
+ case ARRAY:
+ case BYTES:
+ List<String> arrayData = new ArrayList<>();
+ for (int i = 0; i < Array.getLength(value); i++) {
+ arrayData.add(String.valueOf(Array.get(value, i)));
+ }
+ return arrayData.toString();
+ case MAP:
+ return JsonUtils.toJsonString(value);
+ case ROW:
+ List<String> rowData = new ArrayList<>();
+ SeaTunnelRowType rowType = (SeaTunnelRowType) type;
+ for (int i = 0; i < rowType.getTotalFields(); i++) {
+ rowData.add(
+ fieldToString(
+ rowType.getFieldTypes()[i],
+ ((SeaTunnelRow) value).getField(i)));
+ }
+ return rowData.toString();
+ default:
+ return String.valueOf(value);
+ }
+ }
+
+ @Override
+ public Optional<SensorsDataCommitInfo> prepareCommit() throws IOException {
+ sa.flush();
+ return Optional.empty();
+ }
+
+ @Override
+ public void abortPrepare() {}
+
+ @Override
+ public void close() throws IOException {}
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/state/SensorsDataAggregatedCommitInfo.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/state/SensorsDataAggregatedCommitInfo.java
new file mode 100644
index 0000000000..2a5ce36e38
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/state/SensorsDataAggregatedCommitInfo.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk.state;
+
+import java.io.Serializable;
+
+public class SensorsDataAggregatedCommitInfo implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/state/SensorsDataCommitInfo.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/state/SensorsDataCommitInfo.java
new file mode 100644
index 0000000000..98347de798
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/state/SensorsDataCommitInfo.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk.state;
+
+import java.io.Serializable;
+
+public class SensorsDataCommitInfo implements Serializable {
+ private static final long serialVersionUID = 1L;
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/state/SensorsDataSinkState.java b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/state/SensorsDataSinkState.java
new file mode 100644
index 0000000000..9e04744849
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/main/java/org/apache/seatunnel/connectors/sensorsdata/sdk/state/SensorsDataSinkState.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk.state;
+
+import java.io.Serializable;
+
+public class SensorsDataSinkState implements Serializable {
+ private static final long serialVersionUID = 1L;
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/SensorsDataTypesTest.java b/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/SensorsDataTypesTest.java
new file mode 100644
index 0000000000..342cd6605c
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/SensorsDataTypesTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class SensorsDataTypesTest {
+
+ @Test
+ public void of() {
+ SensorsDataTypes type = SensorsDataTypes.of("TIMESTAMP yyyy-MM-dd'T'HH:mm:ssZ");
+ Assertions.assertEquals(SensorsDataTypes.DataTypes.TIMESTAMP, type.getType());
+ Assertions.assertEquals("yyyy-MM-dd'T'HH:mm:ssZ", type.getExtra());
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataSpecialItemRecordTest.java b/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataSpecialItemRecordTest.java
new file mode 100644
index 0000000000..e9bb948525
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataSpecialItemRecordTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import org.apache.seatunnel.shade.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.seatunnel.shade.com.google.common.collect.ImmutableMap;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.SensorsDataConfigBase;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.TargetColumnConfig;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Arrays;
+
+@Slf4j
+class SensorsDataSpecialItemRecordTest {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private SeaTunnelRowType rowType;
+ private SeaTunnelRow row;
+
+ @BeforeEach
+ public void setUp() {
+ rowType =
+ new SeaTunnelRowType(
+ new String[] {
+ "id",
+ "name",
+ "int_col",
+ "bigint_col",
+ "double_col",
+ "float_col",
+ "str_col",
+ "list_col1",
+ "list_col2",
+ "list_col3",
+ "time_int_col",
+ "time_str_col1",
+ "time_str_col2",
+ "bool_col",
+ "item_id_col",
+ "item_type_col",
+ "project_col",
+ "token_col",
+ },
+ new SeaTunnelDataType[] {
+ BasicType.INT_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.INT_TYPE,
+ BasicType.LONG_TYPE,
+ BasicType.DOUBLE_TYPE,
+ BasicType.FLOAT_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.LONG_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.BOOLEAN_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ });
+ Object[] values = new Object[rowType.getFieldNames().length];
+ values[0] = 123;
+ values[1] = "abc";
+ values[2] = 3;
+ values[3] = 1711423014152L;
+ values[4] = 123.12;
+ values[5] = (float) 2.2;
+ values[6] = "abc";
+ values[7] = "abc\nbcd\ncdb";
+ values[8] = "abc,bcd,cdb";
+ values[9] = "abc;bcd;cdb";
+ values[10] = 1711423014152L;
+ values[11] = "1711423014152";
+ values[12] = "2024-03-26 11:31:12";
+ values[13] = true;
+ values[14] = "123";
+ values[15] = "items";
+ values[16] = "production";
+ values[17] = "12345678";
+ row = new SeaTunnelRow(values);
+ }
+
+ @Test
+ public void testUserRecord() {
+ try {
+ ReadonlyConfig readonlyConfig =
+ ReadonlyConfig.fromMap(
+ ImmutableMap.<String, Object>builder()
+ .put("entity_name", "items")
+ .put("record_type", "items")
+ .put("item_id_column", "item_id_col")
+ .put("item_type_column", "item_type_col")
+ .put(
+ "property_fields",
+ Arrays.asList(
+ new TargetColumnConfig("name", "String"),
+ new TargetColumnConfig(
+ "str_col", "String", "str_prop"),
+ new TargetColumnConfig(
+ "list_col2", "LIST_COMMA", "list_prop"),
+ new TargetColumnConfig(
+ "double_col", "DOUBLE", "double_prop"),
+ new TargetColumnConfig(
+ "time_str_col1", "DOUBLE", "date_prop"),
+ new TargetColumnConfig(
+ "bool_col", "BOOLEAN", "bool_prop"),
+ new TargetColumnConfig(
+ "token_col", "STRING", "$token"),
+ new TargetColumnConfig(
+ "project_col", "STRING", "$project")))
+ .build());
+ SensorsDataConfigBase config = new SensorsDataConfigBase(readonlyConfig);
+ RowAccessor ra = new RowAccessor(config, rowType);
+ SensorsDataRecord record = SensorsDataRecordBuilder.newBuilder(config, ra).build(row);
+ String json = record.toJsonString();
+ log.info("ItemRecord: " + json);
+ try {
+ ObjectNode node = (ObjectNode) OBJECT_MAPPER.readTree(json);
+ node.remove("_track_id");
+ node.remove("time");
+ Assertions.assertEquals(
+ "{\"lib\":{\"$lib\":\"Java\",\"$lib_version\":\"3.6.9\",\"$lib_method\":\"code\",\"$lib_detail\":\"JavaSDK##generateLibInfo\"},\"item_id\":\"123\",\"item_type\":\"items\",\"project\":\"production\",\"type\":\"item_set\",\"properties\":{\"str_prop\":\"abc\",\"double_prop\":123.12,\"name\":\"abc\",\"bool_prop\":true,\"list_prop\":[\"abc\",\"bcd\",\"cdb\"],\"date_prop\":1711423014152},\"token\":\"12345678\"}",
+ OBJECT_MAPPER.writeValueAsString(node));
+ } catch (JsonProcessingException e) {
+ Assertions.fail(e.getMessage());
+ }
+ } catch (Exception e) {
+ log.error("fail", e);
+ Assertions.fail(e.getMessage());
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataUserRecordTest.java b/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataUserRecordTest.java
new file mode 100644
index 0000000000..2af4366f1c
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/record/SensorsDataUserRecordTest.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.record;
+
+import org.apache.seatunnel.shade.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.seatunnel.shade.com.google.common.collect.ImmutableMap;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.SensorsDataConfigBase;
+import org.apache.seatunnel.connectors.sensorsdata.format.config.TargetColumnConfig;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Arrays;
+
+@Slf4j
+class SensorsDataUserRecordTest {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private SeaTunnelRowType rowType;
+ private SeaTunnelRow row;
+
+ @BeforeEach
+ public void setUp() {
+ rowType =
+ new SeaTunnelRowType(
+ new String[] {
+ "id",
+ "name",
+ "int_col",
+ "bigint_col",
+ "double_col",
+ "float_col",
+ "str_col",
+ "list_col1",
+ "list_col2",
+ "list_col3",
+ "time_int_col",
+ "time_str_col1",
+ "time_str_col2",
+ "bool_col",
+ },
+ new SeaTunnelDataType[] {
+ BasicType.INT_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.INT_TYPE,
+ BasicType.LONG_TYPE,
+ BasicType.DOUBLE_TYPE,
+ BasicType.FLOAT_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.LONG_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.BOOLEAN_TYPE,
+ });
+ Object[] values = new Object[rowType.getFieldNames().length];
+ values[0] = 123;
+ values[1] = "abc";
+ values[2] = 3;
+ values[3] = 1711423014152L;
+ values[4] = 123.12;
+ values[5] = (float) 2.2;
+ values[6] = "abc";
+ values[7] = "abc\nbcd\ncdb";
+ values[8] = "abc,bcd,cdb";
+ values[9] = "abc;bcd;cdb";
+ values[10] = 1711423014152L;
+ values[11] = "1711423014152";
+ values[12] = "2024-03-26 11:31:12";
+ values[13] = true;
+ row = new SeaTunnelRow(values);
+ }
+
+ @Test
+ public void testUserRecord() {
+ try {
+ ReadonlyConfig readonlyConfig =
+ ReadonlyConfig.fromMap(
+ ImmutableMap.<String, Object>builder()
+ .put("record_type", "users")
+ .put("distinct_id_column", "name")
+ .put(
+ "identity_fields",
+ Arrays.asList(
+ new TargetColumnConfig(
+ "name", "String", "$identity_name"),
+ new TargetColumnConfig(
+ "list_col1",
+ "List",
+ "$identity_distinct_id")))
+ .put(
+ "property_fields",
+ Arrays.asList(
+ new TargetColumnConfig("name", "String"),
+ new TargetColumnConfig(
+ "str_col", "String", "str_prop"),
+ new TargetColumnConfig(
+ "list_col2", "LIST_COMMA", "list_prop"),
+ new TargetColumnConfig(
+ "double_col", "DOUBLE", "double_prop")))
+ .build());
+ SensorsDataConfigBase config = new SensorsDataConfigBase(readonlyConfig);
+ RowAccessor ra = new RowAccessor(config, rowType);
+ UserRecordBase record = UserRecordBase.newBuilder(ra).buildUserRecord(row);
+ String json = record.toJsonString();
+ log.info("UserRecord: " + json);
+ try {
+ ObjectNode node = (ObjectNode) OBJECT_MAPPER.readTree(json);
+ node.remove("_track_id");
+ node.remove("time");
+ Assertions.assertEquals(
+ "{\"schema\":\"users\",\"identities\":{\"$identity_distinct_id\":[\"abc\",\"bcd\",\"cdb\"],\"$identity_name\":[\"abc\"]},\"lib\":{\"$lib\":\"Java\",\"$lib_version\":\"3.6.9\",\"$lib_method\":\"code\",\"$lib_detail\":\"JavaSDK##generateLibInfo\"},\"distinct_id\":\"abc\",\"type\":\"profile_set\",\"version\":\"2.0\",\"properties\":{\"str_prop\":\"abc\",\"double_prop\":123.12,\"name\":\"abc\",\"list_prop\":[\"abc\",\"bcd\",\"cdb\"]}}",
+ OBJECT_MAPPER.writeValueAsString(node));
+ } catch (JsonProcessingException e) {
+ Assertions.fail(e.getMessage());
+ }
+ } catch (Exception e) {
+ log.error("fail", e);
+ }
+ }
+
+ @Test
+ public void testEventRecord1() {
+ ReadonlyConfig readonlyConfig =
+ ReadonlyConfig.fromMap(
+ ImmutableMap.<String, Object>builder()
+ .put("record_type", "events")
+ .put("distinct_id_column", "name")
+ .put("event_name", "${str_col}")
+ .put("time_column", "time_int_col")
+ .put(
+ "identity_fields",
+ Arrays.asList(
+ new TargetColumnConfig(
+ "str_col", "String", "$identity_login_id"),
+ new TargetColumnConfig(
+ "name", "string", "$identity_name")))
+ .put(
+ "property_fields",
+ Arrays.asList(
+ new TargetColumnConfig("name", "string"),
+ new TargetColumnConfig(
+ "str_col", "string", "str_prop"),
+ new TargetColumnConfig(
+ "list_col2", "LIST_COMMA", "list_prop")))
+ .build());
+ SensorsDataConfigBase config = new SensorsDataConfigBase(readonlyConfig);
+ RowAccessor ra = new RowAccessor(config, rowType);
+ String json = SensorsDataRecordBuilder.newBuilder(config, ra).build(row).toJsonString();
+ log.info("UserEventRecord1: " + json);
+ try {
+ ObjectNode node = (ObjectNode) OBJECT_MAPPER.readTree(json);
+ node.remove("_track_id");
+ Assertions.assertEquals(
+ "{\"schema\":\"events\",\"lib\":{\"$lib\":\"Java\",\"$lib_version\":\"3.6.9\",\"$lib_method\":\"code\",\"$lib_detail\":\"JavaSDK##generateLibInfo\"},\"time\":1711423014152,\"type\":\"track\",\"event\":\"abc\",\"version\":\"2.0\",\"properties\":{\"str_prop\":\"abc\",\"$time\":1711423014152,\"name\":\"abc\",\"identities\":{\"$identity_name\":\"abc\",\"$identity_login_id\":\"abc\"},\"list_prop\":[\"abc\",\"bcd\",\"cdb\"],\"distinct_id\":\"abc\"}}",
+ OBJECT_MAPPER.writeValueAsString(node));
+ } catch (JsonProcessingException e) {
+ Assertions.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testEventRecord2() {
+ ReadonlyConfig readonlyConfig =
+ ReadonlyConfig.fromMap(
+ ImmutableMap.<String, Object>builder()
+ .put("record_type", "events")
+ .put("distinct_id_column", "name")
+ .put("event_name", "$AppStart")
+ .put("time_column", "time_int_col")
+ .put(
+ "identity_fields",
+ Arrays.asList(
+ new TargetColumnConfig(
+ "str_col", "String", "$identity_login_id"),
+ new TargetColumnConfig(
+ "name", "string", "$identity_name")))
+ .put(
+ "property_fields",
+ Arrays.asList(
+ new TargetColumnConfig(
+ "str_col", "string", "$project"),
+ new TargetColumnConfig(
+ "str_col", "string", "$token"),
+ new TargetColumnConfig("name", "string"),
+ new TargetColumnConfig(
+ "str_col", "string", "str_prop"),
+ new TargetColumnConfig(
+ "list_col2", "LIST_COMMA", "list_prop")))
+ .build());
+ SensorsDataConfigBase config = new SensorsDataConfigBase(readonlyConfig);
+ RowAccessor ra = new RowAccessor(config, rowType);
+ String json = SensorsDataRecordBuilder.newBuilder(config, ra).build(row).toJsonString();
+ log.info("UserEventRecord2: " + json);
+ try {
+ ObjectNode node = (ObjectNode) OBJECT_MAPPER.readTree(json);
+ node.remove("_track_id");
+ Assertions.assertEquals(
+ "{\"schema\":\"events\",\"lib\":{\"$lib\":\"Java\",\"$lib_version\":\"3.6.9\",\"$lib_method\":\"code\",\"$lib_detail\":\"JavaSDK##generateLibInfo\"},\"project\":\"abc\",\"time\":1711423014152,\"type\":\"track\",\"event\":\"$AppStart\",\"version\":\"2.0\",\"properties\":{\"$token\":\"abc\",\"str_prop\":\"abc\",\"$time\":1711423014152,\"identities\":{\"$identity_name\":\"abc\",\"$identity_login_id\":\"abc\"},\"distinct_id\":\"abc\",\"name\":\"abc\",\"list_prop\":[\"abc\ [...]
+ OBJECT_MAPPER.writeValueAsString(node));
+ } catch (JsonProcessingException e) {
+ Assertions.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testDetailRecord1() {
+ ReadonlyConfig readonlyConfig =
+ ReadonlyConfig.fromMap(
+ ImmutableMap.<String, Object>builder()
+ .put("record_type", "details")
+ .put("schema", "s_order")
+ .put("distinct_id_column", "name")
+ .put("detail_id_column", "name")
+ .put(
+ "identity_fields",
+ Arrays.asList(
+ new TargetColumnConfig(
+ "str_col", "String", "$identity_login_id"),
+ new TargetColumnConfig(
+ "name", "string", "$identity_name")))
+ .put(
+ "property_fields",
+ Arrays.asList(
+ new TargetColumnConfig("name", "string"),
+ new TargetColumnConfig(
+ "str_col", "string", "str_prop"),
+ new TargetColumnConfig(
+ "list_col2", "LIST_COMMA", "list_prop")))
+ .build());
+ SensorsDataConfigBase config = new SensorsDataConfigBase(readonlyConfig);
+ RowAccessor ra = new RowAccessor(config, rowType);
+ String json = SensorsDataRecordBuilder.newBuilder(config, ra).build(row).toJsonString();
+ log.info("UserDetailRecord: " + json);
+ try {
+ ObjectNode node = (ObjectNode) OBJECT_MAPPER.readTree(json);
+ node.remove("_track_id");
+ node.remove("time");
+ Assertions.assertEquals(
+ "{\"schema\":\"s_order\",\"lib\":{\"$lib\":\"Java\",\"$lib_version\":\"3.6.9\",\"$lib_method\":\"code\",\"$lib_detail\":\"JavaSDK##generateLibInfo\"},\"id\":\"abc\",\"type\":\"detail_set\",\"version\":\"2.0\",\"properties\":{\"str_prop\":\"abc\",\"name\":\"abc\",\"identities\":{\"$identity_name\":\"abc\",\"$identity_login_id\":\"abc\"},\"list_prop\":[\"abc\",\"bcd\",\"cdb\"],\"distinct_id\":\"abc\"}}",
+ OBJECT_MAPPER.writeValueAsString(node));
+ } catch (JsonProcessingException e) {
+ Assertions.fail(e.getMessage());
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/utils/TypeUtilTest.java b/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/utils/TypeUtilTest.java
new file mode 100644
index 0000000000..b0ef3ebf8d
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/format/utils/TypeUtilTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.format.utils;
+
+import org.apache.seatunnel.connectors.sensorsdata.format.exception.SensorsDataException;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+
+class TypeUtilTest {
+
+ DateTimeFormatter formatter3 =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withZone(ZoneId.systemDefault());
+ DateTimeFormatter formatter4 =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ").withZone(ZoneId.systemDefault());
+
+ @Test
+ void testToTargetType() {
+ // 1. Number
+ Assertions.assertEquals(123, TypeUtil.toTargetType(123, "NUMBER"));
+ Assertions.assertEquals(123L, TypeUtil.toTargetType(123L, "NUMBER"));
+ Assertions.assertEquals(123.1, TypeUtil.toTargetType(123.1, "NUMBER"));
+ Assertions.assertEquals(123, TypeUtil.toTargetType("123", "NUMBER"));
+ Assertions.assertEquals(
+ ((Double) 123.1).floatValue(), TypeUtil.toTargetType("123.1", "NUMBER"));
+ // 2. Boolean
+ Assertions.assertEquals(true, TypeUtil.toTargetType(1, "BOOLEAN"));
+ Assertions.assertEquals(false, TypeUtil.toTargetType(0, "BOOLEAN"));
+ Assertions.assertEquals(false, TypeUtil.toTargetType(0.0, "BOOLEAN"));
+ Assertions.assertEquals(true, TypeUtil.toTargetType("true", "BOOLEAN"));
+ Assertions.assertEquals(false, TypeUtil.toTargetType("f", "BOOLEAN"));
+ // 3. Timestamp
+ DateTimeFormatter formatter =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault());
+ Assertions.assertEquals(
+ ZonedDateTime.from(formatter.parse("2024-03-16 19:25:07"))
+ .toInstant()
+ .toEpochMilli(),
+ TypeUtil.toTargetType("2024-03-16 19:25:07", "TIMESTAMP"));
+ Assertions.assertEquals(
+ 1710588307000L, TypeUtil.toTargetType(new Date(1710588307000L), "TIMESTAMP"));
+ Assertions.assertEquals(
+ 1710588307000.0, TypeUtil.toTargetType(1710588307000.0, "TIMESTAMP"));
+
+ formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss").withZone(ZoneId.systemDefault());
+ Assertions.assertEquals(
+ ZonedDateTime.from(formatter.parse("20240316_192507")).toInstant().toEpochMilli(),
+ TypeUtil.toTargetType("20240316_192507", "TIMESTAMP"));
+
+ formatter =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+ .withZone(ZoneId.systemDefault());
+ Assertions.assertEquals(
+ ZonedDateTime.from(formatter.parse("2024-03-16 19:25:07.123"))
+ .toInstant()
+ .toEpochMilli(),
+ TypeUtil.toTargetType("2024-03-16 19:25:07.123", "TIMESTAMP"));
+
+ formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ");
+ Assertions.assertEquals(
+ ZonedDateTime.from(formatter.parse("2024-03-16T19:25:07+0100"))
+ .toInstant()
+ .toEpochMilli(),
+ TypeUtil.toTargetType(
+ "2024-03-16T19:25:07+0100", "TIMESTAMP yyyy-MM-dd'T'HH:mm:ssZ"));
+ Assertions.assertEquals(
+ "20240316 192507", TypeUtil.toTargetType("20240316 192507", "TIMESTAMP"));
+
+ // 4. List
+ Assertions.assertEquals(
+ Arrays.asList("123", "456"), TypeUtil.toTargetType("123\n456", "LIST"));
+ Assertions.assertEquals(
+ Arrays.asList("123", "456"), TypeUtil.toTargetType("123,456", "LIST_COMMA"));
+ Assertions.assertEquals(
+ Collections.singletonList("456"), TypeUtil.toTargetType(";456", "LIST_SEMICOLON"));
+ Assertions.assertEquals(
+ Collections.singletonList("123"), TypeUtil.toTargetType("123", "LIST"));
+ Assertions.assertThrowsExactly(
+ SensorsDataException.class, () -> TypeUtil.toTargetType(123, "LIST"));
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/sdk/SensorsDataSDKFactoryTest.java b/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/sdk/SensorsDataSDKFactoryTest.java
new file mode 100644
index 0000000000..3537e61773
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-sensorsdata/src/test/java/org/apache/seatunnel/connectors/sensorsdata/sdk/SensorsDataSDKFactoryTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.sensorsdata.sdk;
+
+import org.apache.seatunnel.connectors.sensorsdata.sdk.sink.SensorsDataSDKSinkFactory;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class SensorsDataSDKFactoryTest {
+
+ @Test
+ void optionRule() {
+ Assertions.assertNotNull((new SensorsDataSDKSinkFactory()).optionRule());
+ }
+}
diff --git a/seatunnel-connectors-v2/pom.xml b/seatunnel-connectors-v2/pom.xml
index 1284914b40..ce221386af 100644
--- a/seatunnel-connectors-v2/pom.xml
+++ b/seatunnel-connectors-v2/pom.xml
@@ -86,6 +86,7 @@
<module>connector-typesense</module>
<module>connector-graphql</module>
<module>connector-aerospike</module>
+ <module>connector-sensorsdata</module>
</modules>
<dependencyManagement>
diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml
index 4940ee2786..1554a7c791 100644
--- a/seatunnel-dist/pom.xml
+++ b/seatunnel-dist/pom.xml
@@ -154,6 +154,12 @@
<scope>provided</scope>
</dependency>
<!-- connectors -->
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-sensorsdata</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>connector-fake</artifactId>
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/pom.xml
new file mode 100644
index 0000000000..efcc8f049d
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/pom.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-connector-v2-e2e</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>connector-sensorsdata-e2e</artifactId>
+ <name>SeaTunnel : E2E : Connector V2 : SensorsData</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-sensorsdata</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-jdbc</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>5.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>${testcontainer.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/java/org/apache/seatunnel/e2e/connector/sensorsdata/sdk/SensorsDataIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/java/org/apache/seatunnel/e2e/connector/sensorsdata/sdk/SensorsDataIT.java
new file mode 100644
index 0000000000..f875acbb24
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/java/org/apache/seatunnel/e2e/connector/sensorsdata/sdk/SensorsDataIT.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.e2e.connector.sensorsdata.sdk;
+
+import org.apache.seatunnel.e2e.common.TestResource;
+import org.apache.seatunnel.e2e.common.TestSuiteBase;
+import org.apache.seatunnel.e2e.common.container.EngineType;
+import org.apache.seatunnel.e2e.common.container.TestContainer;
+import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestTemplate;
+import org.testcontainers.containers.Container;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class SensorsDataIT extends TestSuiteBase implements TestResource {
+
+ @BeforeAll
+ @Override
+ public void startUp() throws Exception {}
+
+ @AfterAll
+ @Override
+ public void tearDown() throws Exception {}
+
+ @TestTemplate
+ @DisabledOnContainer(
+ value = {},
+ type = {EngineType.SPARK},
+ disabledReason =
+ "spark involves the old version of jackson(2.4.0 involved, but 2.12.x is required) will cause an serialize error.")
+ public void testEvents(TestContainer container) throws Exception {
+ String jobConfig = "/fake_to_sensorsdata_events.conf";
+ Container.ExecResult execResult = container.executeJob(jobConfig);
+ Assertions.assertEquals(0, execResult.getExitCode());
+ }
+
+ @TestTemplate
+ @DisabledOnContainer(
+ value = {},
+ type = {EngineType.SPARK},
+ disabledReason =
+ "spark involves the old version of jackson(2.4.0 involved, but 2.12.x is required) will cause an serialize error.")
+ public void testUsers(TestContainer container) throws Exception {
+ String jobConfig = "/fake_to_sensorsdata_users.conf";
+ Container.ExecResult execResult = container.executeJob(jobConfig);
+ Assertions.assertEquals(0, execResult.getExitCode());
+ }
+
+ @TestTemplate
+ @DisabledOnContainer(
+ value = {},
+ type = {EngineType.SPARK},
+ disabledReason =
+ "spark involves the old version of jackson(2.4.0 involved, but 2.12.x is required) will cause an serialize error.")
+ public void testDetails(TestContainer container) throws Exception {
+ String jobConfig = "/fake_to_sensorsdata_details.conf";
+ Container.ExecResult execResult = container.executeJob(jobConfig);
+ Assertions.assertEquals(0, execResult.getExitCode());
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/resources/fake_to_sensorsdata_details.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/resources/fake_to_sensorsdata_details.conf
new file mode 100644
index 0000000000..18b9f96395
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/resources/fake_to_sensorsdata_details.conf
@@ -0,0 +1,66 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+######
+###### This config file is a demonstration of streaming processing in seatunnel config
+######
+
+env {
+ # You can set engine configuration here
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ # This is a example source plugin **only for test and demonstrate the feature source plugin**
+ FakeSource {
+ parallelism = 1
+ row.num = 10
+ schema = {
+ fields {
+ c_id = string
+ c_boolean = boolean
+ c_int = int
+ c_bigint = bigint
+ }
+ }
+ }
+}
+
+transform {
+}
+
+sink {
+ SensorsData {
+ # send data to console, to test and check
+ consumer = console
+ server_url = "http://10.129.27.43:8106/sa?project=sditest"
+ time_free = true
+
+ record_type = details
+ schema = fund_manager
+ distinct_id_column = c_id
+ detail_id_column = c_id
+ identity_fields = [
+ { target = "$identity_distinct_id", source = c_id }
+ ]
+ property_fields = [
+ { target = c_id, source = c_id, type = STRING }
+ { target = fund_amount, source = c_int, type = INT }
+ { target = "$is_valid", source = c_boolean, type = BOOLEAN }
+ ]
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/resources/fake_to_sensorsdata_events.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/resources/fake_to_sensorsdata_events.conf
new file mode 100644
index 0000000000..bbf5648a15
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/resources/fake_to_sensorsdata_events.conf
@@ -0,0 +1,78 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+######
+###### This config file is a demonstration of streaming processing in seatunnel config
+######
+
+env {
+ # You can set engine configuration here
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ # This is a example source plugin **only for test and demonstrate the feature source plugin**
+ FakeSource {
+ parallelism = 1
+ row.num = 10
+ tinyint.min = 20
+ tinyint.max = 60
+ string.fake.mode = "template"
+ string.template = ["foo", "bar", "baz", "qux"]
+ schema = {
+ fields {
+ c_event = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_int = int
+ c_bigint = bigint
+ c_date = date
+ c_null = "null"
+ }
+ }
+ }
+}
+
+transform {
+}
+
+sink {
+ SensorsData {
+ # send data to console, to test and check
+ consumer = console
+ server_url = "http://10.1.136.63:8106/sa?project=default"
+ time_free = true
+
+ record_type = events
+ schema = events
+ event_name = "${c_event}"
+ time_column = c_date
+ distinct_id_column = c_bigint
+ identity_fields = [
+ { source = c_bigint, target = "$identity_login_id" }
+ { source = c_bigint, target = "$identity_distinct_id" }
+ ]
+ property_fields = [
+ { target = c_tinyint, source = c_tinyint, type = INT }
+ { target = c_bigint, source = c_bigint, type = BIGINT }
+ { target = c_int, source = c_int, type = INT }
+ { target = c_boolean, source = c_boolean, type = BOOLEAN }
+# { target = c_null, source = c_null, type = STRING }
+ ]
+ skip_error_record = true
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/resources/fake_to_sensorsdata_users.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/resources/fake_to_sensorsdata_users.conf
new file mode 100644
index 0000000000..36724b36ad
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-sensorsdata-e2e/src/test/resources/fake_to_sensorsdata_users.conf
@@ -0,0 +1,72 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+######
+###### This config file is a demonstration of streaming processing in seatunnel config
+######
+
+env {
+ # You can set engine configuration here
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ # This is a example source plugin **only for test and demonstrate the feature source plugin**
+ FakeSource {
+ parallelism = 1
+ row.num = 10
+ tinyint.min = 20
+ tinyint.max = 60
+ schema = {
+ fields {
+ c_id = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_int = int
+ c_bigint = bigint
+ c_null = "null"
+ }
+ }
+ }
+}
+
+transform {
+}
+
+sink {
+ SensorsData {
+ # send data to console, to test and check
+ consumer = console
+ server_url = "http://10.129.27.43:8106/sa?project=sditest"
+ time_free = true
+
+ record_type = users
+ schema = users
+ distinct_id_column = c_id
+ identity_fields = [
+ { target = "$identity_login_id", source = c_id }
+ { target = "$identity_distinct_id", source = c_id }
+ { target = "$identity_mobile", source = c_bigint }
+ ]
+ property_fields = [
+ { target = age, source = c_tinyint, type = INT }
+ { target = mobile, source = c_bigint, type = BIGINT }
+ { target = c_int, source = c_int, type = INT }
+ { target = c_null, source = c_null, type = STRING }
+ ]
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml
index daf9ac94a8..f14c628a6e 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml
@@ -86,6 +86,7 @@
<module>connector-cdc-tidb-e2e</module>
<module>connector-graphql-e2e</module>
<module>connector-aerospike-e2e</module>
+ <module>connector-sensorsdata-e2e</module>
</modules>
<dependencies>
diff --git a/tools/update_modules_check/update_modules_check.py b/tools/update_modules_check/update_modules_check.py
index 7b3fddfab1..3507cb229c 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -154,6 +154,7 @@ def get_sub_it_modules(modules, total_num, current_num):
modules_arr.remove("connector-file-local-e2e")
modules_arr.remove("connector-file-sftp-e2e")
modules_arr.remove("connector-redis-e2e")
+ modules_arr.remove("connector-sensorsdata-e2e")
if "connector-seatunnel-e2e-base" in modules_arr:
modules_arr.remove("connector-seatunnel-e2e-base")
if "connector-console-seatunnel-e2e" in modules_arr: