You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by di...@apache.org on 2023/12/15 03:49:36 UTC
(doris-flink-connector) branch master updated: [step2][Improve]Use flink’s checkstyle to format Sink Connector (#272)
This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-flink-connector.git
The following commit(s) were added to refs/heads/master by this push:
new f60bebf [step2][Improve]Use flink’s checkstyle to format Sink Connector (#272)
f60bebf is described below
commit f60bebf4c18a5dc5cca3aaa7ca0c17dc74384f64
Author: wudongliang <46...@users.noreply.github.com>
AuthorDate: Fri Dec 15 11:49:28 2023 +0800
[step2][Improve]Use flink’s checkstyle to format Sink Connector (#272)
---
flink-doris-connector/pom.xml | 75 ++---
.../apache/doris/flink/catalog/DorisCatalog.java | 105 ++++---
.../doris/flink/catalog/DorisCatalogFactory.java | 10 +-
.../doris/flink/catalog/DorisCatalogOptions.java | 6 +-
.../doris/flink/catalog/DorisTypeMapper.java | 26 +-
.../doris/flink/catalog/doris/DataModel.java | 3 +-
.../doris/flink/catalog/doris/DorisSystem.java | 100 +++----
.../doris/flink/catalog/doris/DorisType.java | 2 +-
.../doris/flink/catalog/doris/FieldSchema.java | 4 +-
.../doris/flink/catalog/doris/TableSchema.java | 1 +
.../flink/connection/JdbcConnectionProvider.java | 6 +-
.../connection/SimpleJdbcConnectionProvider.java | 9 +-
.../DorisDeserializationSchema.java | 3 +-
.../DorisJsonDebeziumDeserializationSchema.java | 138 +++++----
.../RowDataDeserializationSchema.java | 10 +-
.../SimpleListDeserializationSchema.java | 5 +-
.../converter/DorisRowConverter.java | 118 ++++----
.../doris/flink/lookup/DorisJdbcLookupReader.java | 47 +--
.../apache/doris/flink/lookup/ExecutionPool.java | 48 +--
.../apache/doris/flink/lookup/LookupSchema.java | 27 +-
.../java/org/apache/doris/flink/lookup/Record.java | 9 +-
.../org/apache/doris/flink/lookup/RecordKey.java | 1 -
.../java/org/apache/doris/flink/lookup/Worker.java | 57 ++--
.../org/apache/doris/flink/sink/BackendUtil.java | 31 +-
.../apache/doris/flink/sink/DorisCommittable.java | 25 +-
.../flink/sink/DorisCommittableSerializer.java | 9 +-
.../org/apache/doris/flink/sink/DorisSink.java | 66 +++--
.../org/apache/doris/flink/sink/EscapeHandler.java | 14 +-
.../apache/doris/flink/sink/HttpGetWithEntity.java | 4 +-
.../apache/doris/flink/sink/HttpPutBuilder.java | 10 +-
.../java/org/apache/doris/flink/sink/HttpUtil.java | 22 +-
.../org/apache/doris/flink/sink/LoadStatus.java | 5 +-
.../org/apache/doris/flink/sink/ResponseUtil.java | 9 +-
.../doris/flink/sink/batch/BatchRecordBuffer.java | 48 ++-
.../doris/flink/sink/batch/DorisBatchSink.java | 33 ++-
.../flink/sink/batch/DorisBatchStreamLoad.java | 122 +++++---
.../doris/flink/sink/batch/DorisBatchWriter.java | 53 ++--
.../doris/flink/sink/batch/RecordWithMeta.java | 4 +-
.../doris/flink/sink/committer/DorisCommitter.java | 58 ++--
.../flink/sink/schema/SchemaChangeHelper.java | 25 +-
.../flink/sink/schema/SchemaChangeManager.java | 111 ++++---
.../doris/flink/sink/writer/CacheRecordBuffer.java | 21 +-
.../doris/flink/sink/writer/DorisStreamLoad.java | 102 ++++---
.../doris/flink/sink/writer/DorisWriter.java | 185 +++++++-----
.../doris/flink/sink/writer/DorisWriterState.java | 24 +-
.../sink/writer/DorisWriterStateSerializer.java | 27 +-
.../doris/flink/sink/writer/LabelGenerator.java | 8 +-
.../doris/flink/sink/writer/LoadConstants.java | 5 +-
.../doris/flink/sink/writer/RecordBuffer.java | 18 +-
.../doris/flink/sink/writer/RecordStream.java | 10 +-
.../flink/sink/writer/serializer/DorisRecord.java | 13 +-
.../writer/serializer/DorisRecordSerializer.java | 3 +-
.../serializer/JsonDebeziumSchemaSerializer.java | 206 +++++++------
.../serializer/RecordWithMetaSerializer.java | 16 +-
.../sink/writer/serializer/RowDataSerializer.java | 27 +-
.../sink/writer/serializer/RowSerializer.java | 43 +--
.../writer/serializer/SimpleStringSerializer.java | 6 +-
.../doris/flink/table/DorisConfigOptions.java | 323 ++++++++++++---------
.../flink/table/DorisDynamicTableFactory.java | 51 ++--
.../doris/flink/table/DorisDynamicTableSink.java | 65 +++--
.../doris/flink/table/DorisDynamicTableSource.java | 92 +++---
.../table/DorisRowDataAsyncLookupFunction.java | 76 +++--
.../doris/flink/table/DorisRowDataInputFormat.java | 63 ++--
.../table/DorisRowDataJdbcLookupFunction.java | 62 ++--
.../flink/table/DorisRowDataLookupFunction.java | 48 +--
.../doris/flink/table/DorisTableInputSplit.java | 14 +-
.../org/apache/doris/flink/tools/cdc/CdcTools.java | 37 ++-
.../apache/doris/flink/tools/cdc/DatabaseSync.java | 261 +++++++++--------
.../flink/tools/cdc/ParsingProcessFunction.java | 13 +-
.../doris/flink/tools/cdc/SourceConnector.java | 1 -
.../apache/doris/flink/tools/cdc/SourceSchema.java | 26 +-
.../tools/cdc/mysql/DateToStringConverter.java | 26 +-
.../flink/tools/cdc/mysql/MysqlDatabaseSync.java | 103 +++----
.../doris/flink/tools/cdc/mysql/MysqlSchema.java | 7 +-
.../doris/flink/tools/cdc/mysql/MysqlType.java | 18 +-
.../flink/tools/cdc/oracle/OracleDatabaseSync.java | 134 +++++----
.../tools/cdc/oracle/OracleDateConverter.java | 115 ++++----
.../doris/flink/tools/cdc/oracle/OracleSchema.java | 11 +-
.../doris/flink/tools/cdc/oracle/OracleType.java | 18 +-
.../tools/cdc/postgres/PostgresDatabaseSync.java | 116 ++++----
.../tools/cdc/postgres/PostgresDateConverter.java | 19 +-
.../flink/tools/cdc/postgres/PostgresSchema.java | 11 +-
.../flink/tools/cdc/postgres/PostgresType.java | 69 +++--
.../tools/cdc/sqlserver/SqlServerDatabaseSync.java | 104 ++++---
.../cdc/sqlserver/SqlServerDateConverter.java | 22 +-
.../flink/tools/cdc/sqlserver/SqlServerSchema.java | 11 +-
.../flink/tools/cdc/sqlserver/SqlServerType.java | 19 +-
.../java/org/apache/doris/flink/DorisTestBase.java | 3 +-
.../apache/doris/flink/catalog/CatalogTest.java | 3 +-
.../doris/flink/serialization/TestRowBatch.java | 5 +-
.../flink/sink/schema/SchemaChangeHelperTest.java | 3 +-
.../writer/TestJsonDebeziumSchemaSerializer.java | 3 +-
.../table/DorisRowDataLookupFunctionTest.java | 3 +-
.../doris/flink/utils/DateToStringConverter.java | 3 +-
tools/maven/checkstyle.xml | 8 +-
95 files changed, 2350 insertions(+), 1789 deletions(-)
diff --git a/flink-doris-connector/pom.xml b/flink-doris-connector/pom.xml
index 2bb1f49..2d7c3fe 100644
--- a/flink-doris-connector/pom.xml
+++ b/flink-doris-connector/pom.xml
@@ -222,7 +222,6 @@ under the License.
<artifactId>guava</artifactId>
<version>31.1-jre</version>
</dependency>
-
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
@@ -460,38 +459,36 @@ under the License.
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
</plugin>
-
-<!-- TODO Wait until all code is formatted before introducing it.-->
-<!-- <plugin>-->
-<!-- <groupId>com.diffplug.spotless</groupId>-->
-<!-- <artifactId>spotless-maven-plugin</artifactId>-->
-<!-- <version>${spotless.version}</version>-->
-<!-- <configuration>-->
-<!-- <java>-->
-<!-- <googleJavaFormat>-->
-<!-- <version>1.7</version>-->
-<!-- <style>AOSP</style>-->
-<!-- </googleJavaFormat>-->
-
-<!-- <!– \# refers to the static imports –>-->
-<!-- <importOrder>-->
-<!-- <order>org.apache.flink,org.apache.flink.shaded,,javax,java,scala,\#-->
-<!-- </order>-->
-<!-- </importOrder>-->
-
-<!-- <removeUnusedImports/>-->
-<!-- </java>-->
-<!-- </configuration>-->
-<!-- <executions>-->
-<!-- <execution>-->
-<!-- <id>spotless-check</id>-->
-<!-- <phase>validate</phase>-->
-<!-- <goals>-->
-<!-- <goal>check</goal>-->
-<!-- </goals>-->
-<!-- </execution>-->
-<!-- </executions>-->
-<!-- </plugin>-->
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <version>${spotless.version}</version>
+ <configuration>
+ <java>
+ <googleJavaFormat>
+ <version>1.7</version>
+ <style>AOSP</style>
+ </googleJavaFormat>
+
+ <!-- \# refers to the static imports -->
+ <importOrder>
+ <order>org.apache.flink,org.apache.flink.shaded,,javax,java,scala,\#
+ </order>
+ </importOrder>
+
+ <removeUnusedImports/>
+ </java>
+ </configuration>
+ <executions>
+ <execution>
+ <id>spotless-check</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@@ -519,18 +516,6 @@ under the License.
<configLocation>../tools/maven/checkstyle.xml</configLocation>
<logViolationsToConsole>true</logViolationsToConsole>
<failOnViolation>true</failOnViolation>
- <!-- TODO It needs to be deleted after all code formatting is completed.-->
- <sourceDirectories>
- <directory>src/main/java/org/apache/doris/flink/source/</directory>
- <directory>src/main/java/org/apache/doris/flink/rest</directory>
- <directory>src/main/java/org/apache/doris/flink/serialization</directory>
- <directory>src/main/java/org/apache/doris/flink/cfg</directory>
- <directory>src/main/java/org/apache/doris/flink/datastream</directory>
- <directory>src/main/java/org/apache/doris/flink/backend</directory>
- <directory>src/main/java/org/apache/doris/flink/exception</directory>
- <directory>src/main/java/org/apache/doris/flink/rest</directory>
- <directory>src/main/java/org/apache/doris/flink/serialization</directory>
- </sourceDirectories>
</configuration>
</plugin>
<plugin>
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
index e495146..a6535cd 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
@@ -14,15 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.catalog;
-import org.apache.commons.compress.utils.Lists;
-import org.apache.doris.flink.catalog.doris.DataModel;
-import org.apache.doris.flink.catalog.doris.DorisSystem;
-import org.apache.doris.flink.catalog.doris.FieldSchema;
-import org.apache.doris.flink.catalog.doris.TableSchema;
-import org.apache.doris.flink.cfg.DorisConnectionOptions;
-import org.apache.doris.flink.table.DorisDynamicTableFactory;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.table.api.Schema;
import org.apache.flink.table.catalog.AbstractCatalog;
@@ -54,6 +48,14 @@ import org.apache.flink.table.factories.Factory;
import org.apache.flink.table.types.DataType;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.StringUtils;
+
+import org.apache.commons.compress.utils.Lists;
+import org.apache.doris.flink.catalog.doris.DataModel;
+import org.apache.doris.flink.catalog.doris.DorisSystem;
+import org.apache.doris.flink.catalog.doris.FieldSchema;
+import org.apache.doris.flink.catalog.doris.TableSchema;
+import org.apache.doris.flink.cfg.DorisConnectionOptions;
+import org.apache.doris.flink.table.DorisDynamicTableFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,9 +84,7 @@ import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
-/**
- * catalog for flink
- */
+/** catalog for flink. */
public class DorisCatalog extends AbstractCatalog {
private static final Logger LOG = LoggerFactory.getLogger(DorisCatalog.class);
@@ -147,12 +147,12 @@ public class DorisCatalog extends AbstractCatalog {
@Override
public void createDatabase(String name, CatalogDatabase database, boolean ignoreIfExists)
throws DatabaseAlreadyExistException, CatalogException {
- if(databaseExists(name)){
- if(ignoreIfExists){
+ if (databaseExists(name)) {
+ if (ignoreIfExists) {
return;
}
throw new DatabaseAlreadyExistException(getName(), name);
- }else {
+ } else {
dorisSystem.createDatabase(name);
}
}
@@ -160,7 +160,7 @@ public class DorisCatalog extends AbstractCatalog {
@Override
public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade)
throws DatabaseNotEmptyException, CatalogException, DatabaseNotExistException {
- if(!databaseExists(name)){
+ if (!databaseExists(name)) {
if (ignoreIfNotExists) {
return;
}
@@ -168,7 +168,7 @@ public class DorisCatalog extends AbstractCatalog {
}
if (!cascade && listTables(name).size() > 0) {
- throw new DatabaseNotEmptyException(getName(),name);
+ throw new DatabaseNotEmptyException(getName(), name);
}
dorisSystem.dropDatabase(name);
}
@@ -185,7 +185,8 @@ public class DorisCatalog extends AbstractCatalog {
public List<String> listTables(String databaseName)
throws DatabaseNotExistException, CatalogException {
Preconditions.checkState(
- org.apache.commons.lang3.StringUtils.isNotBlank(databaseName), "Database name must not be blank.");
+ org.apache.commons.lang3.StringUtils.isNotBlank(databaseName),
+ "Database name must not be blank.");
if (!databaseExists(databaseName)) {
throw new DatabaseNotExistException(getName(), databaseName);
}
@@ -220,19 +221,21 @@ public class DorisCatalog extends AbstractCatalog {
props.put(PASSWORD.key(), connectionOptions.getPassword());
props.put(TABLE_IDENTIFIER.key(), databaseName + "." + tableName);
- String labelPrefix = props.getOrDefault(SINK_LABEL_PREFIX.key(),"");
- props.put(SINK_LABEL_PREFIX.key(), String.join("_",labelPrefix,databaseName,tableName));
- //remove catalog option
+ String labelPrefix = props.getOrDefault(SINK_LABEL_PREFIX.key(), "");
+ props.put(SINK_LABEL_PREFIX.key(), String.join("_", labelPrefix, databaseName, tableName));
+ // remove catalog option
props.remove(DEFAULT_DATABASE.key());
- return CatalogTable.of(createTableSchema(databaseName, tableName), null, Lists.newArrayList(), props);
-
+ return CatalogTable.of(
+ createTableSchema(databaseName, tableName), null, Lists.newArrayList(), props);
}
@VisibleForTesting
protected String queryFenodes() {
- try (Connection conn = DriverManager.getConnection(connectionOptions.getJdbcUrl(),
- connectionOptions.getUsername(),
- connectionOptions.getPassword())) {
+ try (Connection conn =
+ DriverManager.getConnection(
+ connectionOptions.getJdbcUrl(),
+ connectionOptions.getUsername(),
+ connectionOptions.getPassword())) {
StringJoiner fenodes = new StringJoiner(",");
PreparedStatement ps = conn.prepareStatement("SHOW FRONTENDS");
ResultSet resultSet = ps.executeQuery();
@@ -248,12 +251,16 @@ public class DorisCatalog extends AbstractCatalog {
}
private Schema createTableSchema(String databaseName, String tableName) {
- try (Connection conn = DriverManager.getConnection(connectionOptions.getJdbcUrl(),
- connectionOptions.getUsername(),
- connectionOptions.getPassword())) {
+ try (Connection conn =
+ DriverManager.getConnection(
+ connectionOptions.getJdbcUrl(),
+ connectionOptions.getUsername(),
+ connectionOptions.getPassword())) {
PreparedStatement ps =
conn.prepareStatement(
- String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_SIZE,DECIMAL_DIGITS FROM `information_schema`.`COLUMNS` WHERE `TABLE_SCHEMA`= '%s' AND `TABLE_NAME`= '%s'", databaseName, tableName));
+ String.format(
+ "SELECT COLUMN_NAME,DATA_TYPE,COLUMN_SIZE,DECIMAL_DIGITS FROM `information_schema`.`COLUMNS` WHERE `TABLE_SCHEMA`= '%s' AND `TABLE_NAME`= '%s'",
+ databaseName, tableName));
List<String> columnNames = new ArrayList<>();
List<DataType> columnTypes = new ArrayList<>();
@@ -264,7 +271,9 @@ public class DorisCatalog extends AbstractCatalog {
long columnSize = resultSet.getLong("COLUMN_SIZE");
long columnDigit = resultSet.getLong("DECIMAL_DIGITS");
- DataType flinkType = DorisTypeMapper.toFlinkType(columnName, columnType, (int) columnSize, (int) columnDigit);
+ DataType flinkType =
+ DorisTypeMapper.toFlinkType(
+ columnName, columnType, (int) columnSize, (int) columnDigit);
columnNames.add(columnName);
columnTypes.add(flinkType);
}
@@ -273,7 +282,10 @@ public class DorisCatalog extends AbstractCatalog {
return tableSchema;
} catch (Exception e) {
throw new CatalogException(
- String.format("Failed getting catalog %s database %s table %s", getName(), databaseName, tableName), e);
+ String.format(
+ "Failed getting catalog %s database %s table %s",
+ getName(), databaseName, tableName),
+ e);
}
}
@@ -290,14 +302,15 @@ public class DorisCatalog extends AbstractCatalog {
@Override
public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists)
throws TableNotExistException, CatalogException {
- if(!tableExists(tablePath)){
- if(ignoreIfNotExists){
+ if (!tableExists(tablePath)) {
+ if (ignoreIfNotExists) {
return;
}
throw new TableNotExistException(getName(), tablePath);
}
- dorisSystem.dropTable(String.format("%s.%s", tablePath.getDatabaseName(), tablePath.getObjectName()));
+ dorisSystem.dropTable(
+ String.format("%s.%s", tablePath.getDatabaseName(), tablePath.getObjectName()));
}
@Override
@@ -312,11 +325,11 @@ public class DorisCatalog extends AbstractCatalog {
checkNotNull(tablePath, "tablePath cannot be null");
checkNotNull(table, "table cannot be null");
- if(!databaseExists(tablePath.getDatabaseName())) {
+ if (!databaseExists(tablePath.getDatabaseName())) {
throw new DatabaseNotExistException(getName(), tablePath.getDatabaseName());
}
- if(tableExists(tablePath)){
- if(ignoreIfExists){
+ if (tableExists(tablePath)) {
+ if (ignoreIfExists) {
return;
}
throw new TableAlreadyExistException(getName(), tablePath);
@@ -341,22 +354,22 @@ public class DorisCatalog extends AbstractCatalog {
dorisSystem.createTable(schema);
}
- public List<String> getCreateDorisKeys(org.apache.flink.table.api.TableSchema schema){
- Preconditions.checkState(schema.getPrimaryKey().isPresent(),"primary key cannot be null");
+ public List<String> getCreateDorisKeys(org.apache.flink.table.api.TableSchema schema) {
+ Preconditions.checkState(schema.getPrimaryKey().isPresent(), "primary key cannot be null");
return schema.getPrimaryKey().get().getColumns();
}
- public Map<String, FieldSchema> getCreateDorisColumns(org.apache.flink.table.api.TableSchema schema){
+ public Map<String, FieldSchema> getCreateDorisColumns(
+ org.apache.flink.table.api.TableSchema schema) {
String[] fieldNames = schema.getFieldNames();
DataType[] fieldTypes = schema.getFieldDataTypes();
Map<String, FieldSchema> fields = new LinkedHashMap<>();
for (int i = 0; i < fieldNames.length; i++) {
- fields.put(fieldNames[i],
+ fields.put(
+ fieldNames[i],
new FieldSchema(
- fieldNames[i],
- DorisTypeMapper.toDorisType(fieldTypes[i]),
- null));
+ fieldNames[i], DorisTypeMapper.toDorisType(fieldTypes[i]), null));
}
return fields;
}
@@ -380,7 +393,7 @@ public class DorisCatalog extends AbstractCatalog {
public List<CatalogPartitionSpec> listPartitions(
ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
throws TableNotExistException, TableNotPartitionedException,
- PartitionSpecInvalidException, CatalogException {
+ PartitionSpecInvalidException, CatalogException {
return Collections.emptyList();
}
@@ -410,8 +423,8 @@ public class DorisCatalog extends AbstractCatalog {
CatalogPartition partition,
boolean ignoreIfExists)
throws TableNotExistException, TableNotPartitionedException,
- PartitionSpecInvalidException, PartitionAlreadyExistsException,
- CatalogException {
+ PartitionSpecInvalidException, PartitionAlreadyExistsException,
+ CatalogException {
throw new UnsupportedOperationException();
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
index c00e5f1..d837395 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
@@ -14,20 +14,21 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.catalog;
-import org.apache.doris.flink.cfg.DorisConnectionOptions;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.table.catalog.Catalog;
import org.apache.flink.table.factories.CatalogFactory;
import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.doris.flink.cfg.DorisConnectionOptions;
+
import java.util.HashSet;
import java.util.Set;
import static org.apache.doris.flink.catalog.DorisCatalogOptions.DEFAULT_DATABASE;
-import static org.apache.doris.flink.table.DorisConfigOptions.JDBC_URL;
import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_BATCH_SIZE;
import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_DESERIALIZE_ARROW_ASYNC;
import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_DESERIALIZE_QUEUE_SIZE;
@@ -41,6 +42,7 @@ import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_RETR
import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_TABLET_SIZE;
import static org.apache.doris.flink.table.DorisConfigOptions.FENODES;
import static org.apache.doris.flink.table.DorisConfigOptions.IDENTIFIER;
+import static org.apache.doris.flink.table.DorisConfigOptions.JDBC_URL;
import static org.apache.doris.flink.table.DorisConfigOptions.PASSWORD;
import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_COUNT;
import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_SIZE;
@@ -56,9 +58,7 @@ import static org.apache.doris.flink.table.DorisConfigOptions.STREAM_LOAD_PROP_P
import static org.apache.doris.flink.table.DorisConfigOptions.TABLE_IDENTIFIER;
import static org.apache.doris.flink.table.DorisConfigOptions.USERNAME;
-/**
- * Factory for {@link DorisCatalog}.
- */
+/** Factory for {@link DorisCatalog}. */
public class DorisCatalogFactory implements CatalogFactory {
@Override
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java
index cc5772e..8f8d476 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.catalog;
import org.apache.flink.configuration.ConfigOption;
@@ -24,7 +25,10 @@ import java.util.HashMap;
import java.util.Map;
public class DorisCatalogOptions {
- public static final ConfigOption<String> DEFAULT_DATABASE = ConfigOptions.key(CommonCatalogOptions.DEFAULT_DATABASE_KEY).stringType().noDefaultValue();
+ public static final ConfigOption<String> DEFAULT_DATABASE =
+ ConfigOptions.key(CommonCatalogOptions.DEFAULT_DATABASE_KEY)
+ .stringType()
+ .noDefaultValue();
public static final String TABLE_PROPERTIES_PREFIX = "table.properties.";
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java
index 7da45b9..c388700 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java
@@ -14,9 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.catalog;
-import org.apache.doris.flink.catalog.doris.DorisType;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.logical.ArrayType;
@@ -38,6 +38,8 @@ import org.apache.flink.table.types.logical.VarBinaryType;
import org.apache.flink.table.types.logical.VarCharType;
import org.apache.flink.table.types.logical.utils.LogicalTypeDefaultVisitor;
+import org.apache.doris.flink.catalog.doris.DorisType;
+
import static org.apache.doris.flink.catalog.doris.DorisType.BIGINT;
import static org.apache.doris.flink.catalog.doris.DorisType.BOOLEAN;
import static org.apache.doris.flink.catalog.doris.DorisType.CHAR;
@@ -65,14 +67,16 @@ public class DorisTypeMapper {
/** Max size of varchar type of Doris. */
public static final int MAX_VARCHAR_SIZE = 65533;
- public static DataType toFlinkType(String columnName, String columnType, int precision, int scale) {
+ public static DataType toFlinkType(
+ String columnName, String columnType, int precision, int scale) {
columnType = columnType.toUpperCase();
switch (columnType) {
case BOOLEAN:
return DataTypes.BOOLEAN();
case TINYINT:
if (precision == 0) {
- //The boolean type will become tinyint when queried in information_schema, and precision=0
+ // The boolean type will become tinyint when queried in information_schema, and
+ // precision=0
return DataTypes.BOOLEAN();
} else {
return DataTypes.TINYINT();
@@ -107,11 +111,12 @@ public class DorisTypeMapper {
default:
throw new UnsupportedOperationException(
String.format(
- "Doesn't support Doris type '%s' on column '%s'", columnType, columnName));
+ "Doesn't support Doris type '%s' on column '%s'",
+ columnType, columnName));
}
}
- public static String toDorisType(DataType flinkType){
+ public static String toDorisType(DataType flinkType) {
LogicalType logicalType = flinkType.getLogicalType();
return logicalType.accept(new LogicalTypeVisitor(logicalType));
}
@@ -135,7 +140,7 @@ public class DorisTypeMapper {
@Override
public String visit(VarCharType varCharType) {
- //Flink varchar length max value is int, it may overflow after multiplying by 3
+ // Flink varchar length max value is int, it may overflow after multiplying by 3
long length = varCharType.getLength() * 3L;
return length >= MAX_VARCHAR_SIZE ? STRING : String.format("%s(%s)", VARCHAR, length);
}
@@ -151,11 +156,12 @@ public class DorisTypeMapper {
}
@Override
- public String visit(DecimalType decimalType) {
+ public String visit(DecimalType decimalType) {
int precision = decimalType.getPrecision();
int scale = decimalType.getScale();
return precision <= 38
- ? String.format("%s(%s,%s)", DorisType.DECIMAL_V3, precision, Math.max(scale, 0))
+ ? String.format(
+ "%s(%s,%s)", DorisType.DECIMAL_V3, precision, Math.max(scale, 0))
: DorisType.STRING;
}
@@ -197,7 +203,8 @@ public class DorisTypeMapper {
@Override
public String visit(TimestampType timestampType) {
int precision = timestampType.getPrecision();
- return String.format("%s(%s)", DorisType.DATETIME_V2, Math.min(Math.max(precision, 0), 6));
+ return String.format(
+ "%s(%s)", DorisType.DATETIME_V2, Math.min(Math.max(precision, 0), 6));
}
@Override
@@ -223,5 +230,4 @@ public class DorisTypeMapper {
type.toString()));
}
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java
index 03b87a0..a1085b5 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java
@@ -14,10 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.catalog.doris;
public enum DataModel {
DUPLICATE,
UNIQUE,
AGGREGATE
-}
\ No newline at end of file
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java
index 7be9a5f..11e19b9 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java
@@ -14,8 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.catalog.doris;
+import org.apache.flink.annotation.Public;
+import org.apache.flink.util.StringUtils;
import org.apache.commons.compress.utils.Lists;
import org.apache.doris.flink.cfg.DorisConnectionOptions;
@@ -24,8 +27,6 @@ import org.apache.doris.flink.connection.SimpleJdbcConnectionProvider;
import org.apache.doris.flink.exception.CreateTableException;
import org.apache.doris.flink.exception.DorisRuntimeException;
import org.apache.doris.flink.exception.DorisSystemException;
-import org.apache.flink.annotation.Public;
-import org.apache.flink.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,15 +43,14 @@ import java.util.stream.Collectors;
import static org.apache.flink.util.Preconditions.checkArgument;
-/**
- * Doris System Operate
- */
+/** Doris System Operate. */
@Public
public class DorisSystem implements Serializable {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(DorisSystem.class);
private final JdbcConnectionProvider jdbcConnectionProvider;
- private static final List<String> builtinDatabases = Collections.singletonList("information_schema");
+ private static final List<String> builtinDatabases =
+ Collections.singletonList("information_schema");
public DorisSystem(DorisConnectionOptions options) {
this.jdbcConnectionProvider = new SimpleJdbcConnectionProvider(options);
@@ -78,22 +78,22 @@ public class DorisSystem implements Serializable {
return true;
}
- public boolean tableExists(String database, String table){
- return databaseExists(database)
- && listTables(database).contains(table);
+ public boolean tableExists(String database, String table) {
+ return databaseExists(database) && listTables(database).contains(table);
}
- public boolean columnExists(String database, String table, String columnName){
- if(tableExists(database, table)){
- List<String> columns = extractColumnValuesBySQL(
- "SELECT COLUMN_NAME FROM information_schema.`COLUMNS` WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ?",
- 1,
- null,
- database,
- table,
- columnName);
- if(columns != null && !columns.isEmpty()){
- return true;
+ public boolean columnExists(String database, String table, String columnName) {
+ if (tableExists(database, table)) {
+ List<String> columns =
+ extractColumnValuesBySQL(
+ "SELECT COLUMN_NAME FROM information_schema.`COLUMNS` WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ?",
+ 1,
+ null,
+ database,
+ table,
+ columnName);
+ if (columns != null && !columns.isEmpty()) {
+ return true;
}
}
return false;
@@ -121,21 +121,21 @@ public class DorisSystem implements Serializable {
}
public void execute(String sql) {
- try (Statement statement = jdbcConnectionProvider.getOrEstablishConnection().createStatement()) {
+ try (Statement statement =
+ jdbcConnectionProvider.getOrEstablishConnection().createStatement()) {
statement.execute(sql);
- } catch (Exception e){
- throw new DorisSystemException(String.format("SQL query could not be executed: %s", sql), e);
+ } catch (Exception e) {
+ throw new DorisSystemException(
+ String.format("SQL query could not be executed: %s", sql), e);
}
}
public List<String> extractColumnValuesBySQL(
- String sql,
- int columnIndex,
- Predicate<String> filterFunc,
- Object... params) {
+ String sql, int columnIndex, Predicate<String> filterFunc, Object... params) {
List<String> columnValues = Lists.newArrayList();
- try (PreparedStatement ps = jdbcConnectionProvider.getOrEstablishConnection().prepareStatement(sql)) {
+ try (PreparedStatement ps =
+ jdbcConnectionProvider.getOrEstablishConnection().prepareStatement(sql)) {
if (Objects.nonNull(params) && params.length > 0) {
for (int i = 0; i < params.length; i++) {
ps.setObject(i + 1, params[i]);
@@ -151,9 +151,7 @@ public class DorisSystem implements Serializable {
return columnValues;
} catch (Exception e) {
throw new DorisSystemException(
- String.format(
- "The following SQL query could not be executed: %s", sql),
- e);
+ String.format("The following SQL query could not be executed: %s", sql), e);
}
}
@@ -166,47 +164,44 @@ public class DorisSystem implements Serializable {
Map<String, FieldSchema> fields = schema.getFields();
List<String> keys = schema.getKeys();
- //append keys
- for(String key : keys){
- if(!fields.containsKey(key)){
+ // append keys
+ for (String key : keys) {
+ if (!fields.containsKey(key)) {
throw new CreateTableException("key " + key + " not found in column list");
}
FieldSchema field = fields.get(key);
buildColumn(sb, field, true);
}
- //append values
+ // append values
for (Map.Entry<String, FieldSchema> entry : fields.entrySet()) {
- if(keys.contains(entry.getKey())){
+ if (keys.contains(entry.getKey())) {
continue;
}
FieldSchema field = entry.getValue();
buildColumn(sb, field, false);
-
}
- sb = sb.deleteCharAt(sb.length() -1);
+ sb = sb.deleteCharAt(sb.length() - 1);
sb.append(" ) ");
- //append uniq model
- if(DataModel.UNIQUE.equals(schema.getModel())){
+ // append uniq model
+ if (DataModel.UNIQUE.equals(schema.getModel())) {
sb.append(schema.getModel().name())
.append(" KEY(")
.append(String.join(",", identifier(schema.getKeys())))
.append(")");
}
- //append table comment
- if(!StringUtils.isNullOrWhitespaceOnly(schema.getTableComment())){
- sb.append(" COMMENT '")
- .append(quoteComment(schema.getTableComment()))
- .append("' ");
+ // append table comment
+ if (!StringUtils.isNullOrWhitespaceOnly(schema.getTableComment())) {
+ sb.append(" COMMENT '").append(quoteComment(schema.getTableComment())).append("' ");
}
- //append distribute key
+ // append distribute key
sb.append(" DISTRIBUTED BY HASH(")
.append(String.join(",", identifier(schema.getDistributeKeys())))
.append(") BUCKETS AUTO ");
- //append properties
+ // append properties
int index = 0;
for (Map.Entry<String, String> entry : schema.getProperties().entrySet()) {
if (index == 0) {
@@ -227,9 +222,9 @@ public class DorisSystem implements Serializable {
return sb.toString();
}
- private static void buildColumn(StringBuilder sql, FieldSchema field, boolean isKey){
+ private static void buildColumn(StringBuilder sql, FieldSchema field, boolean isKey) {
String fieldType = field.getTypeString();
- if(isKey && DorisType.STRING.equals(fieldType)){
+ if (isKey && DorisType.STRING.equals(fieldType)) {
fieldType = String.format("%s(%s)", DorisType.VARCHAR, 65533);
}
sql.append(identifier(field.getName()))
@@ -240,11 +235,11 @@ public class DorisSystem implements Serializable {
.append("',");
}
- private static String quoteComment(String comment){
- if(comment == null){
+ private static String quoteComment(String comment) {
+ if (comment == null) {
return "";
} else {
- return comment.replaceAll("'","\\\\'");
+ return comment.replaceAll("'", "\\\\'");
}
}
@@ -260,5 +255,4 @@ public class DorisSystem implements Serializable {
private static String quoteProperties(String name) {
return "'" + name + "'";
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java
index 1e43ac1..d242320 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.catalog.doris;
public class DorisType {
@@ -38,5 +39,4 @@ public class DorisType {
public static final String BITMAP = "BITMAP";
public static final String ARRAY = "ARRAY";
public static final String JSONB = "JSONB";
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java
index 3c28b74..3d7f276 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.catalog.doris;
public class FieldSchema {
@@ -22,8 +23,7 @@ public class FieldSchema {
private String defaultValue;
private String comment;
- public FieldSchema() {
- }
+ public FieldSchema() {}
public FieldSchema(String name, String typeString, String comment) {
this.name = name;
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java
index 8f04705..a9b2af0 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.catalog.doris;
import java.util.ArrayList;
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java
index 3e840c0..1403f55 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java
@@ -21,13 +21,9 @@ import java.sql.Connection;
public interface JdbcConnectionProvider {
- /**
- * Get existing connection or establish an new one if there is none.
- */
+ /** Get existing connection or establish an new one if there is none. */
Connection getOrEstablishConnection() throws Exception;
-
/** Close possible existing connection. */
void closeConnection();
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java
index 73141a3..68a9231 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java
@@ -36,7 +36,7 @@ public class SimpleJdbcConnectionProvider implements JdbcConnectionProvider, Ser
private transient Connection connection;
- public SimpleJdbcConnectionProvider(DorisConnectionOptions options){
+ public SimpleJdbcConnectionProvider(DorisConnectionOptions options) {
this.options = options;
}
@@ -48,11 +48,14 @@ public class SimpleJdbcConnectionProvider implements JdbcConnectionProvider, Ser
try {
Class.forName("com.mysql.cj.jdbc.Driver");
} catch (ClassNotFoundException ex) {
- LOG.warn("can not found class com.mysql.cj.jdbc.Driver, use class com.mysql.jdbc.Driver");
+ LOG.warn(
+ "can not found class com.mysql.cj.jdbc.Driver, use class com.mysql.jdbc.Driver");
Class.forName("com.mysql.jdbc.Driver");
}
if (!Objects.isNull(options.getUsername())) {
- connection = DriverManager.getConnection(options.getJdbcUrl(), options.getUsername(), options.getPassword());
+ connection =
+ DriverManager.getConnection(
+ options.getJdbcUrl(), options.getUsername(), options.getPassword());
} else {
connection = DriverManager.getConnection(options.getJdbcUrl());
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
index 608fa5c..179a284 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.deserialization;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
@@ -25,7 +26,7 @@ import java.util.List;
/**
* The deserialization schema describes how to turn the doris list record into data types
* (Java/Scala objects) that are processed by Flink.
- **/
+ */
public interface DorisDeserializationSchema<T> extends Serializable, ResultTypeQueryable<T> {
void deserialize(List<?> record, Collector<T> out) throws Exception;
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisJsonDebeziumDeserializationSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisJsonDebeziumDeserializationSchema.java
index 9c54ade..729ba7d 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisJsonDebeziumDeserializationSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisJsonDebeziumDeserializationSchema.java
@@ -17,7 +17,9 @@
package org.apache.doris.flink.deserialization;
-import org.apache.doris.flink.exception.DorisException;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.util.Collector;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -30,9 +32,7 @@ import com.ververica.cdc.connectors.shaded.org.apache.kafka.connect.data.Schema;
import com.ververica.cdc.connectors.shaded.org.apache.kafka.connect.data.Struct;
import com.ververica.cdc.connectors.shaded.org.apache.kafka.connect.source.SourceRecord;
import com.ververica.cdc.debezium.DebeziumDeserializationSchema;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.util.Collector;
+import org.apache.doris.flink.exception.DorisException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
@@ -40,12 +40,12 @@ import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Map;
-/**
- * Currently just use for synchronous mysql non-default.
- */
-public class DorisJsonDebeziumDeserializationSchema implements DebeziumDeserializationSchema<String> {
+/** Currently just use for synchronous mysql non-default. */
+public class DorisJsonDebeziumDeserializationSchema
+ implements DebeziumDeserializationSchema<String> {
- private static final JsonNodeFactory JSON_NODE_FACTORY = JsonNodeFactory.withExactBigDecimals(true);
+ private static final JsonNodeFactory JSON_NODE_FACTORY =
+ JsonNodeFactory.withExactBigDecimals(true);
private final ObjectMapper objectMapper;
public DorisJsonDebeziumDeserializationSchema() {
@@ -53,7 +53,8 @@ public class DorisJsonDebeziumDeserializationSchema implements DebeziumDeseriali
}
@Override
- public void deserialize(SourceRecord sourceRecord, Collector<String> collector) throws Exception {
+ public void deserialize(SourceRecord sourceRecord, Collector<String> collector)
+ throws Exception {
Schema schema = sourceRecord.valueSchema();
Object value = sourceRecord.value();
JsonNode jsonValue = convertToJson(schema, value);
@@ -63,8 +64,8 @@ public class DorisJsonDebeziumDeserializationSchema implements DebeziumDeseriali
private JsonNode convertToJson(Schema schema, Object value) throws DorisException {
if (value == null) {
- if (schema == null) // Any schema is valid and we don't have a default, so treat this as an optional schema
- {
+ // Any schema is valid and we don't have a default, so treat this as an optional schema
+ if (schema == null) {
return null;
}
if (schema.isOptional()) {
@@ -80,7 +81,9 @@ public class DorisJsonDebeziumDeserializationSchema implements DebeziumDeseriali
schemaType = ConnectSchema.schemaType(value.getClass());
if (schemaType == null) {
throw new DorisException(
- "Java class " + value.getClass() + " does not have corresponding schema type.");
+ "Java class "
+ + value.getClass()
+ + " does not have corresponding schema type.");
}
} else {
schemaType = schema.type();
@@ -111,65 +114,74 @@ public class DorisJsonDebeziumDeserializationSchema implements DebeziumDeseriali
} else if (value instanceof BigDecimal) {
return JSON_NODE_FACTORY.numberNode((BigDecimal) value);
} else {
- throw new DorisException("Invalid type for bytes type: " + value.getClass());
+ throw new DorisException(
+ "Invalid type for bytes type: " + value.getClass());
}
- case ARRAY: {
- Collection<?> collection = (Collection<?>) value;
- ArrayNode list = JSON_NODE_FACTORY.arrayNode();
- for (Object elem : collection) {
- Schema valueSchema = schema == null ? null : schema.valueSchema();
- JsonNode fieldValue = convertToJson(valueSchema, elem);
- list.add(fieldValue);
+ case ARRAY:
+ {
+ Collection<?> collection = (Collection<?>) value;
+ ArrayNode list = JSON_NODE_FACTORY.arrayNode();
+ for (Object elem : collection) {
+ Schema valueSchema = schema == null ? null : schema.valueSchema();
+ JsonNode fieldValue = convertToJson(valueSchema, elem);
+ list.add(fieldValue);
+ }
+ return list;
}
- return list;
- }
- case MAP: {
- Map<?, ?> map = (Map<?, ?>) value;
- // If true, using string keys and JSON object; if false, using non-string keys and Array-encoding
- boolean objectMode;
- if (schema == null) {
- objectMode = true;
- for (Map.Entry<?, ?> entry : map.entrySet()) {
- if (!(entry.getKey() instanceof String)) {
- objectMode = false;
- break;
+ case MAP:
+ {
+ Map<?, ?> map = (Map<?, ?>) value;
+ // If true, using string keys and JSON object; if false, using non-string
+ // keys and Array-encoding
+ boolean objectMode;
+ if (schema == null) {
+ objectMode = true;
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ if (!(entry.getKey() instanceof String)) {
+ objectMode = false;
+ break;
+ }
}
+ } else {
+ objectMode = schema.keySchema().type() == Schema.Type.STRING;
}
- } else {
- objectMode = schema.keySchema().type() == Schema.Type.STRING;
- }
- ObjectNode obj = null;
- ArrayNode list = null;
- if (objectMode) {
- obj = JSON_NODE_FACTORY.objectNode();
- } else {
- list = JSON_NODE_FACTORY.arrayNode();
- }
- for (Map.Entry<?, ?> entry : map.entrySet()) {
- Schema keySchema = schema == null ? null : schema.keySchema();
- Schema valueSchema = schema == null ? null : schema.valueSchema();
- JsonNode mapKey = convertToJson(keySchema, entry.getKey());
- JsonNode mapValue = convertToJson(valueSchema, entry.getValue());
-
+ ObjectNode obj = null;
+ ArrayNode list = null;
if (objectMode) {
- obj.set(mapKey.asText(), mapValue);
+ obj = JSON_NODE_FACTORY.objectNode();
} else {
- list.add(JSON_NODE_FACTORY.arrayNode().add(mapKey).add(mapValue));
+ list = JSON_NODE_FACTORY.arrayNode();
}
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ Schema keySchema = schema == null ? null : schema.keySchema();
+ Schema valueSchema = schema == null ? null : schema.valueSchema();
+ JsonNode mapKey = convertToJson(keySchema, entry.getKey());
+ JsonNode mapValue = convertToJson(valueSchema, entry.getValue());
+
+ if (objectMode) {
+ obj.set(mapKey.asText(), mapValue);
+ } else {
+ list.add(JSON_NODE_FACTORY.arrayNode().add(mapKey).add(mapValue));
+ }
+ }
+ return objectMode ? obj : list;
}
- return objectMode ? obj : list;
- }
- case STRUCT: {
- Struct struct = (Struct) value;
- if (!struct.schema().equals(schema)) {
- throw new DorisException("Mismatching schema.");
- }
- ObjectNode obj = JSON_NODE_FACTORY.objectNode();
- for (Field field : schema.fields()) {
- obj.set(field.name(), convertToJson(field.schema(), struct.getWithoutDefault(field.name())));
+ case STRUCT:
+ {
+ Struct struct = (Struct) value;
+ if (!struct.schema().equals(schema)) {
+ throw new DorisException("Mismatching schema.");
+ }
+ ObjectNode obj = JSON_NODE_FACTORY.objectNode();
+ for (Field field : schema.fields()) {
+ obj.set(
+ field.name(),
+ convertToJson(
+ field.schema(),
+ struct.getWithoutDefault(field.name())));
+ }
+ return obj;
}
- return obj;
- }
}
throw new DorisException("Couldn't convert " + value + " to JSON.");
} catch (ClassCastException e) {
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java
index 3342b66..4a23ba5 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java
@@ -14,20 +14,22 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.deserialization;
-import org.apache.doris.flink.deserialization.converter.DorisRowConverter;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.logical.RowType;
import org.apache.flink.util.Collector;
-import java.util.List;
+import org.apache.doris.flink.deserialization.converter.DorisRowConverter;
+
+import java.util.List;
/**
- * A simple implementation of {@link DorisDeserializationSchema} which converts the received
- * list record into {@link GenericRowData}.
+ * A simple implementation of {@link DorisDeserializationSchema} which converts the received list
+ * record into {@link GenericRowData}.
*/
public class RowDataDeserializationSchema implements DorisDeserializationSchema<RowData> {
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
index 43e68ef..78d12de 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
@@ -14,8 +14,8 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.deserialization;
+package org.apache.doris.flink.deserialization;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -27,8 +27,7 @@ public class SimpleListDeserializationSchema implements DorisDeserializationSche
@Override
public TypeInformation<List<?>> getProducedType() {
- return TypeInformation.of(new TypeHint<List<?>>() {
- });
+ return TypeInformation.of(new TypeHint<List<?>>() {});
}
@Override
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java
index c334d3b..b2a49d1 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java
@@ -14,11 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.deserialization.converter;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.doris.flink.serialization.RowBatch;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
import org.apache.flink.table.data.GenericArrayData;
@@ -42,6 +40,10 @@ import org.apache.flink.table.types.logical.TimestampType;
import org.apache.flink.table.types.logical.ZonedTimestampType;
import org.apache.flink.util.Preconditions;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.doris.flink.serialization.RowBatch;
+
import java.io.Serializable;
import java.math.BigDecimal;
import java.sql.Date;
@@ -62,7 +64,7 @@ public class DorisRowConverter implements Serializable {
private SerializationConverter[] serializationConverters;
private static final ObjectMapper objectMapper = new ObjectMapper();
- public DorisRowConverter(){}
+ public DorisRowConverter() {}
public DorisRowConverter(RowType rowType) {
checkNotNull(rowType);
@@ -96,20 +98,21 @@ public class DorisRowConverter implements Serializable {
}
/**
- * Convert data retrieved from {@link RowBatch} to {@link RowData}.
+ * Convert data retrieved from {@link RowBatch} to {@link RowData}.
*
* @param record from rowBatch
*/
public GenericRowData convertInternal(List record) {
GenericRowData rowData = new GenericRowData(deserializationConverters.length);
- for (int i = 0; i < deserializationConverters.length ; i++) {
+ for (int i = 0; i < deserializationConverters.length; i++) {
rowData.setField(i, deserializationConverters[i].deserialize(record.get(i)));
}
return rowData;
}
/**
- * Convert data from {@link RowData} to {@link RowBatch}
+ * Convert data from {@link RowData} to {@link RowBatch}.
+ *
* @param rowData record from flink rowdata
* @param index the field index
* @return java type value.
@@ -118,10 +121,8 @@ public class DorisRowConverter implements Serializable {
return serializationConverters[index].serialize(index, rowData);
}
-
/**
- * Create a nullable runtime {@link DeserializationConverter} from given {@link
- * LogicalType}.
+ * Create a nullable runtime {@link DeserializationConverter} from given {@link LogicalType}.
*/
public DeserializationConverter createNullableInternalConverter(LogicalType type) {
return wrapIntoNullableInternalConverter(createInternalConverter(type));
@@ -142,7 +143,8 @@ public class DorisRowConverter implements Serializable {
return wrapIntoNullableExternalConverter(createExternalConverter(type));
}
- public static SerializationConverter wrapIntoNullableExternalConverter(SerializationConverter serializationConverter) {
+ public static SerializationConverter wrapIntoNullableExternalConverter(
+ SerializationConverter serializationConverter) {
return (index, val) -> {
if (val == null || val.isNullAt(index)) {
return null;
@@ -156,16 +158,14 @@ public class DorisRowConverter implements Serializable {
@FunctionalInterface
public interface DeserializationConverter extends Serializable {
/**
- * Convert a doris field object of {@link RowBatch } to the data structure object.
+ * Convert a doris field object of {@link RowBatch } to the data structure object.
*
* @param field
*/
Object deserialize(Object field);
}
- /**
- * Runtime converter to convert {@link RowData} type object to doris field.
- */
+ /** Runtime converter to convert {@link RowData} type object to doris field. */
@FunctionalInterface
public interface SerializationConverter extends Serializable {
Object serialize(int index, RowData field);
@@ -196,19 +196,23 @@ public class DorisRowConverter implements Serializable {
if (val instanceof LocalDateTime) {
return TimestampData.fromLocalDateTime((LocalDateTime) val);
} else {
- throw new UnsupportedOperationException("timestamp type must be java.time.LocalDateTime, the actual type is: " + val.getClass().getName());
+ throw new UnsupportedOperationException(
+ "timestamp type must be java.time.LocalDateTime, the actual type is: "
+ + val.getClass().getName());
}
};
case DATE:
return val -> {
if (val instanceof LocalDate) {
- //doris source
+ // doris source
return (int) ((LocalDate) val).toEpochDay();
} else if (val instanceof Date) {
- //doris lookup
+ // doris lookup
return (int) ((Date) val).toLocalDate().toEpochDay();
} else {
- throw new UnsupportedOperationException("timestamp type must be java.time.LocalDate, the actual type is: " + val.getClass());
+ throw new UnsupportedOperationException(
+ "timestamp type must be java.time.LocalDate, the actual type is: "
+ + val.getClass());
}
};
case CHAR:
@@ -245,7 +249,8 @@ public class DorisRowConverter implements Serializable {
case DECIMAL:
final int decimalPrecision = ((DecimalType) type).getPrecision();
final int decimalScale = ((DecimalType) type).getScale();
- return (index, val) -> val.getDecimal(index, decimalPrecision, decimalScale).toBigDecimal();
+ return (index, val) ->
+ val.getDecimal(index, decimalPrecision, decimalScale).toBigDecimal();
case TINYINT:
return (index, val) -> val.getByte(index);
case SMALLINT:
@@ -300,12 +305,14 @@ public class DorisRowConverter implements Serializable {
return arrayData;
}
- private MapData convertMapData(Map<Object, Object> map, LogicalType type){
+ private MapData convertMapData(Map<Object, Object> map, LogicalType type) {
MapType mapType = (MapType) type;
- DeserializationConverter keyConverter = createNullableInternalConverter(mapType.getKeyType());
- DeserializationConverter valueConverter = createNullableInternalConverter(mapType.getValueType());
+ DeserializationConverter keyConverter =
+ createNullableInternalConverter(mapType.getKeyType());
+ DeserializationConverter valueConverter =
+ createNullableInternalConverter(mapType.getValueType());
Map<Object, Object> result = new HashMap<>();
- for(Map.Entry<Object, Object> entry : map.entrySet()){
+ for (Map.Entry<Object, Object> entry : map.entrySet()) {
Object key = keyConverter.deserialize(entry.getKey());
Object value = valueConverter.deserialize(entry.getValue());
result.put(key, value);
@@ -318,8 +325,9 @@ public class DorisRowConverter implements Serializable {
RowType rowType = (RowType) type;
GenericRowData rowData = new GenericRowData(row.size());
int index = 0;
- for(Map.Entry<String, ?> entry : row.entrySet()){
- DeserializationConverter converter = createNullableInternalConverter(rowType.getTypeAt(index));
+ for (Map.Entry<String, ?> entry : row.entrySet()) {
+ DeserializationConverter converter =
+ createNullableInternalConverter(rowType.getTypeAt(index));
Object value = converter.deserialize(entry.getValue());
rowData.setField(index, value);
index++;
@@ -327,18 +335,23 @@ public class DorisRowConverter implements Serializable {
return rowData;
}
- private static List<Object> convertArrayData(ArrayData array, LogicalType type){
- if(array instanceof GenericArrayData){
- return Arrays.asList(((GenericArrayData)array).toObjectArray());
+ private static List<Object> convertArrayData(ArrayData array, LogicalType type) {
+ if (array instanceof GenericArrayData) {
+ return Arrays.asList(((GenericArrayData) array).toObjectArray());
}
- if(array instanceof BinaryArrayData){
- LogicalType elementType = ((ArrayType)type).getElementType();
- List<Object> values = Arrays.asList(((BinaryArrayData) array).toObjectArray(elementType));
- if(LogicalTypeRoot.DATE.equals(elementType.getTypeRoot())) {
- return values.stream().map(date -> Date.valueOf(LocalDate.ofEpochDay((Integer)date))).collect(Collectors.toList());
+ if (array instanceof BinaryArrayData) {
+ LogicalType elementType = ((ArrayType) type).getElementType();
+ List<Object> values =
+ Arrays.asList(((BinaryArrayData) array).toObjectArray(elementType));
+ if (LogicalTypeRoot.DATE.equals(elementType.getTypeRoot())) {
+ return values.stream()
+ .map(date -> Date.valueOf(LocalDate.ofEpochDay((Integer) date)))
+ .collect(Collectors.toList());
}
if (LogicalTypeRoot.ARRAY.equals(elementType.getTypeRoot())) {
- return values.stream().map(arr -> convertArrayData((ArrayData)arr, elementType)).collect(Collectors.toList());
+ return values.stream()
+ .map(arr -> convertArrayData((ArrayData) arr, elementType))
+ .collect(Collectors.toList());
}
return values;
}
@@ -348,27 +361,30 @@ public class DorisRowConverter implements Serializable {
private static Object convertMapData(MapData map, LogicalType type) {
Map<Object, Object> result = new HashMap<>();
if (map instanceof GenericMapData) {
- GenericMapData gMap = (GenericMapData)map;
- for (Object key : ((GenericArrayData)gMap.keyArray()).toObjectArray()) {
+ GenericMapData gMap = (GenericMapData) map;
+ for (Object key : ((GenericArrayData) gMap.keyArray()).toObjectArray()) {
result.put(key, gMap.get(key));
}
return result;
}
if (map instanceof BinaryMapData) {
- BinaryMapData bMap = (BinaryMapData)map;
- LogicalType valueType = ((MapType)type).getValueType();
+ BinaryMapData bMap = (BinaryMapData) map;
+ LogicalType valueType = ((MapType) type).getValueType();
Map<?, ?> javaMap = bMap.toJavaMap(((MapType) type).getKeyType(), valueType);
- for (Map.Entry<?,?> entry : javaMap.entrySet()) {
+ for (Map.Entry<?, ?> entry : javaMap.entrySet()) {
String key = entry.getKey().toString();
if (LogicalTypeRoot.MAP.equals(valueType.getTypeRoot())) {
- result.put(key, convertMapData((MapData)entry.getValue(), valueType));
- }else if (LogicalTypeRoot.DATE.equals(valueType.getTypeRoot())) {
- result.put(key, Date.valueOf(LocalDate.ofEpochDay((Integer)entry.getValue())).toString());
- }else if (LogicalTypeRoot.ARRAY.equals(valueType.getTypeRoot())) {
- result.put(key, convertArrayData((ArrayData)entry.getValue(), valueType));
- }else if(entry.getValue() instanceof TimestampData){
- result.put(key, ((TimestampData)entry.getValue()).toTimestamp().toString());
- }else{
+ result.put(key, convertMapData((MapData) entry.getValue(), valueType));
+ } else if (LogicalTypeRoot.DATE.equals(valueType.getTypeRoot())) {
+ result.put(
+ key,
+ Date.valueOf(LocalDate.ofEpochDay((Integer) entry.getValue()))
+ .toString());
+ } else if (LogicalTypeRoot.ARRAY.equals(valueType.getTypeRoot())) {
+ result.put(key, convertArrayData((ArrayData) entry.getValue(), valueType));
+ } else if (entry.getValue() instanceof TimestampData) {
+ result.put(key, ((TimestampData) entry.getValue()).toTimestamp().toString());
+ } else {
result.put(key, entry.getValue().toString());
}
}
@@ -378,12 +394,12 @@ public class DorisRowConverter implements Serializable {
}
private static Object convertRowData(RowData val, int index, LogicalType type) {
- RowType rowType = (RowType)type;
+ RowType rowType = (RowType) type;
Map<String, Object> value = new HashMap<>();
RowData row = val.getRow(index, rowType.getFieldCount());
List<RowType.RowField> fields = rowType.getFields();
- for(int i = 0;i< fields.size(); i++){
+ for (int i = 0; i < fields.size(); i++) {
RowType.RowField rowField = fields.get(i);
SerializationConverter converter = createNullableExternalConverter(rowField.getType());
Object valTmp = converter.serialize(i, row);
@@ -392,7 +408,7 @@ public class DorisRowConverter implements Serializable {
return value;
}
- private static String writeValueAsString(Object object){
+ private static String writeValueAsString(Object object) {
try {
return objectMapper.writeValueAsString(object);
} catch (JsonProcessingException e) {
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
index d935a8d..958aa9e 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
@@ -17,12 +17,13 @@
package org.apache.doris.flink.lookup;
-import org.apache.doris.flink.cfg.DorisLookupOptions;
-import org.apache.doris.flink.cfg.DorisOptions;
-import org.apache.doris.flink.deserialization.converter.DorisRowConverter;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.DataType;
+
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.deserialization.converter.DorisRowConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,7 +46,8 @@ public class DorisJdbcLookupReader extends DorisLookupReader {
private LookupSchema schema;
- public DorisJdbcLookupReader(DorisOptions options, DorisLookupOptions lookupOptions, LookupSchema lookupSchema) {
+ public DorisJdbcLookupReader(
+ DorisOptions options, DorisLookupOptions lookupOptions, LookupSchema lookupSchema) {
this.converter = new DorisRowConverter(lookupSchema.getFieldTypes());
this.pool = new ExecutionPool(options, lookupOptions);
this.schema = lookupSchema;
@@ -68,24 +70,25 @@ public class DorisJdbcLookupReader extends DorisLookupReader {
Record record = convertRecord(recordIn);
try {
pool.get(new Get(record))
- .handleAsync((resultRow, throwable) -> {
- try {
- if (throwable != null) {
- result.completeExceptionally(throwable);
- } else {
- if (resultRow == null) {
- result.complete(new ArrayList<>());
- } else {
- //convert Record to RowData
- List<RowData> rowDatas = convertRowDataList(resultRow);
- result.complete(rowDatas);
+ .handleAsync(
+ (resultRow, throwable) -> {
+ try {
+ if (throwable != null) {
+ result.completeExceptionally(throwable);
+ } else {
+ if (resultRow == null) {
+ result.complete(new ArrayList<>());
+ } else {
+ // convert Record to RowData
+ List<RowData> rowDatas = convertRowDataList(resultRow);
+ result.complete(rowDatas);
+ }
+ }
+ } catch (Throwable e) {
+ result.completeExceptionally(e);
}
- }
- } catch (Throwable e) {
- result.completeExceptionally(e);
- }
- return null;
- });
+ return null;
+ });
} catch (Exception e) {
result.completeExceptionally(e);
}
@@ -132,7 +135,7 @@ public class DorisJdbcLookupReader extends DorisLookupReader {
@Override
public void close() throws IOException {
- if(this.pool != null){
+ if (this.pool != null) {
this.pool.close();
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java
index 94cb35c..94d28b5 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java
@@ -44,8 +44,8 @@ public class ExecutionPool implements Closeable {
private static final Logger LOG = LoggerFactory.getLogger(ExecutionPool.class);
private ActionWatcher readActionWatcher;
final ArrayBlockingQueue<Get> queue;
- private AtomicBoolean started; //determine whether the executionPool is running
- private AtomicBoolean workerStated; //determine whether the worker is running
+ private AtomicBoolean started; // determine whether the executionPool is running
+ private AtomicBoolean workerStated; // determine whether the worker is running
ExecutorService actionWatcherExecutorService;
ExecutorService workerExecutorService;
ThreadFactory workerThreadFactory;
@@ -74,8 +74,24 @@ public class ExecutionPool implements Closeable {
private void start() {
if (started.compareAndSet(false, true)) {
workerStated.set(true);
- workerExecutorService = new ThreadPoolExecutor(workers.length, workers.length, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1), workerThreadFactory, new ThreadPoolExecutor.AbortPolicy());
- actionWatcherExecutorService = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1), actionWatcherThreadFactory, new ThreadPoolExecutor.AbortPolicy());
+ workerExecutorService =
+ new ThreadPoolExecutor(
+ workers.length,
+ workers.length,
+ 0L,
+ TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<>(1),
+ workerThreadFactory,
+ new ThreadPoolExecutor.AbortPolicy());
+ actionWatcherExecutorService =
+ new ThreadPoolExecutor(
+ 1,
+ 1,
+ 0L,
+ TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<>(1),
+ actionWatcherThreadFactory,
+ new ThreadPoolExecutor.AbortPolicy());
for (int i = 0; i < workers.length; ++i) {
workerExecutorService.execute(workers[i]);
}
@@ -114,7 +130,7 @@ public class ExecutionPool implements Closeable {
}
public boolean submit(GetAction action) {
- //if has semaphore, try to obtain the semaphore, otherwise return submit failure
+ // if has semaphore, try to obtain the semaphore, otherwise return submit failure
if (semaphore != null) {
try {
boolean acquire = semaphore.tryAcquire(2000L, TimeUnit.MILLISECONDS);
@@ -127,24 +143,22 @@ public class ExecutionPool implements Closeable {
action.setSemaphore(semaphore);
}
- //try to submit to worker
+ // try to submit to worker
for (int i = 0; i < workers.length; ++i) {
Worker worker = workers[i];
if (worker.offer(action)) {
return true;
}
}
- //If submit fails, it will be released, and if successful, the worker will be responsible for the release
+ // If submit fails, it will be released, and if successful, the worker will be responsible
+ // for the release
if (semaphore != null) {
semaphore.release();
}
return false;
}
- /**
- * monitor the query action
- * and submit it to the worker as soon as the data arrives
- */
+ /** monitor the query action and submit it to the worker as soon as the data arrives. */
class ActionWatcher implements Runnable {
private int batchSize;
@@ -166,13 +180,15 @@ public class ExecutionPool implements Closeable {
LOG.debug("fetch {} records from queue", recordList.size());
Map<String, List<Get>> getsByTable = new HashMap<>();
for (Get get : recordList) {
- List<Get> list = getsByTable.computeIfAbsent(get.getRecord().getTableIdentifier(), (s) -> new ArrayList<>());
+ List<Get> list =
+ getsByTable.computeIfAbsent(
+ get.getRecord().getTableIdentifier(),
+ (s) -> new ArrayList<>());
list.add(get);
}
for (Map.Entry<String, List<Get>> entry : getsByTable.entrySet()) {
GetAction getAction = new GetAction(entry.getValue());
- while (!submit(getAction)) {
- }
+ while (!submit(getAction)) {}
}
}
} catch (InterruptedException e) {
@@ -190,9 +206,7 @@ public class ExecutionPool implements Closeable {
@Override
public String toString() {
- return "ActionWatcher{" +
- "batchSize=" + batchSize +
- '}';
+ return "ActionWatcher{" + "batchSize=" + batchSize + '}';
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java
index 698427f..a62c0fd 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java
@@ -30,7 +30,12 @@ public class LookupSchema implements Serializable {
private DataType[] fieldTypes;
private int[] keyIndex;
- public LookupSchema(String tableIdentifier, String[] selectFields, DataType[] fieldTypes, String[] conditionFields, int[] keyIndex) {
+ public LookupSchema(
+ String tableIdentifier,
+ String[] selectFields,
+ DataType[] fieldTypes,
+ String[] conditionFields,
+ int[] keyIndex) {
this.tableIdentifier = tableIdentifier;
this.selectFields = selectFields;
this.fieldTypes = fieldTypes;
@@ -80,12 +85,18 @@ public class LookupSchema implements Serializable {
@Override
public String toString() {
- return "LookupSchema{" +
- "tableIdentifier='" + tableIdentifier + '\'' +
- ", selectFields=" + Arrays.toString(selectFields) +
- ", conditionFields=" + Arrays.toString(conditionFields) +
- ", fieldTypes=" + Arrays.toString(fieldTypes) +
- ", keyIndex=" + Arrays.toString(keyIndex) +
- '}';
+ return "LookupSchema{"
+ + "tableIdentifier='"
+ + tableIdentifier
+ + '\''
+ + ", selectFields="
+ + Arrays.toString(selectFields)
+ + ", conditionFields="
+ + Arrays.toString(conditionFields)
+ + ", fieldTypes="
+ + Arrays.toString(fieldTypes)
+ + ", keyIndex="
+ + Arrays.toString(keyIndex)
+ + '}';
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java
index 172d892..1704e95 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java
@@ -22,9 +22,7 @@ import org.apache.flink.table.types.DataType;
import java.io.Serializable;
import java.util.Arrays;
-/**
- * record
- */
+/** record. */
public class Record implements Serializable {
LookupSchema schema;
@@ -73,9 +71,6 @@ public class Record implements Serializable {
@Override
public String toString() {
- return "Record{" +
- "schema=" + schema +
- ", values=" + Arrays.toString(values) +
- '}';
+ return "Record{" + "schema=" + schema + ", values=" + Arrays.toString(values) + '}';
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java
index 805a03f..1f1ab6d 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java
@@ -118,7 +118,6 @@ public class RecordKey {
return hash;
}
-
public static int hashCode(Object obj) {
if (obj == null) {
return 0;
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java
index f4f92a9..e4e08f7 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java
@@ -17,12 +17,13 @@
package org.apache.doris.flink.lookup;
+import org.apache.flink.annotation.VisibleForTesting;
+
import org.apache.doris.flink.cfg.DorisLookupOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.connection.JdbcConnectionProvider;
import org.apache.doris.flink.connection.SimpleJdbcConnectionProvider;
import org.apache.doris.flink.exception.DorisRuntimeException;
-import org.apache.flink.annotation.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,7 +51,11 @@ public class Worker implements Runnable {
private final int maxRetryTimes;
private AtomicReference<Throwable> exception = new AtomicReference<>(null);
- public Worker(AtomicBoolean started, DorisOptions options, DorisLookupOptions lookupOptions, int index) {
+ public Worker(
+ AtomicBoolean started,
+ DorisOptions options,
+ DorisLookupOptions lookupOptions,
+ int index) {
this.started = started;
this.name = "Worker-" + index;
this.jdbcConnectionProvider = new SimpleJdbcConnectionProvider(options);
@@ -71,9 +76,9 @@ public class Worker implements Runnable {
try {
GetAction action = queue.poll(2000L, TimeUnit.MILLISECONDS);
if (action != null) {
- try{
+ try {
handle(action);
- }finally {
+ } finally {
if (action.getSemaphore() != null) {
action.getSemaphore().release();
}
@@ -96,7 +101,10 @@ public class Worker implements Runnable {
LookupSchema schema = action.getGetList().get(0).getRecord().getSchema();
List<Get> recordList = action.getGetList();
List<Get> deduplicateList = deduplicateRecords(recordList);
- LOG.debug("record size {}, after deduplicate size {}", recordList.size(), deduplicateList.size());
+ LOG.debug(
+ "record size {}, after deduplicate size {}",
+ recordList.size(),
+ deduplicateList.size());
StringBuilder sb = new StringBuilder();
boolean first;
for (int i = 0; i < deduplicateList.size(); i++) {
@@ -118,7 +126,8 @@ public class Worker implements Runnable {
String sql = sb.toString();
try {
- Map<RecordKey, List<Record>> resultRecordMap = executeQuery(sql, deduplicateList, schema);
+ Map<RecordKey, List<Record>> resultRecordMap =
+ executeQuery(sql, deduplicateList, schema);
for (Get get : recordList) {
Record record = get.getRecord();
if (get.getFuture() != null) {
@@ -137,20 +146,27 @@ public class Worker implements Runnable {
}
/**
- * Sometimes, there will be duplicate key filtering conditions in a batch of data,
- * which can be removed in advance to reduce query pressure.
- * */
+ * Sometimes, there will be duplicate key filtering conditions in a batch of data, which can be
+ * removed in advance to reduce query pressure.
+ */
@VisibleForTesting
public static List<Get> deduplicateRecords(List<Get> recordList) {
- if(recordList == null || recordList.size() <= 1){
+ if (recordList == null || recordList.size() <= 1) {
return recordList;
}
- Set<Get> recordSet = new TreeSet<>((r1, r2) -> Arrays.equals(r1.getRecord().getValues(), r2.getRecord().getValues()) ? 0 : -1);
+ Set<Get> recordSet =
+ new TreeSet<>(
+ (r1, r2) ->
+ Arrays.equals(
+ r1.getRecord().getValues(),
+ r2.getRecord().getValues())
+ ? 0
+ : -1);
recordSet.addAll(recordList);
return new ArrayList<>(recordSet);
}
- private void appendSelect(StringBuilder sb, LookupSchema schema){
+ private void appendSelect(StringBuilder sb, LookupSchema schema) {
String[] selectFields = schema.getSelectFields();
sb.append("select ");
for (int i = 0; i < selectFields.length; i++) {
@@ -163,11 +179,10 @@ public class Worker implements Runnable {
sb.append(" from ").append(schema.getTableIdentifier());
}
- private Map<RecordKey, List<Record>> executeQuery(String sql,
- List<Get> recordList,
- LookupSchema schema) {
+ private Map<RecordKey, List<Record>> executeQuery(
+ String sql, List<Get> recordList, LookupSchema schema) {
Map<RecordKey, List<Record>> resultRecordMap = new HashMap<>();
- //retry strategy
+ // retry strategy
for (int retry = 0; retry <= maxRetryTimes; retry++) {
resultRecordMap = new HashMap<>();
try {
@@ -188,12 +203,18 @@ public class Worker implements Runnable {
for (int index = 0; index < schema.getFieldTypes().length; index++) {
record.setObject(index, rs.getObject(index + 1));
}
- List<Record> records = resultRecordMap.computeIfAbsent(new RecordKey(record), m -> new ArrayList<>());
+ List<Record> records =
+ resultRecordMap.computeIfAbsent(
+ new RecordKey(record), m -> new ArrayList<>());
records.add(record);
}
}
}
- LOG.debug("query cost {}ms, batch {} records, sql is {}", System.currentTimeMillis()-start, recordList.size(), sql);
+ LOG.debug(
+ "query cost {}ms, batch {} records, sql is {}",
+ System.currentTimeMillis() - start,
+ recordList.size(),
+ sql);
return resultRecordMap;
} catch (Exception e) {
LOG.error(String.format("query doris error, retry times = %d", retry), e);
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java
index 954bdd0..f909bb6 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java
@@ -24,7 +24,6 @@ import org.apache.doris.flink.exception.DorisRuntimeException;
import org.apache.doris.flink.rest.RestService;
import org.apache.doris.flink.rest.models.BackendV2;
import org.apache.doris.flink.rest.models.BackendV2.BackendRowV2;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,22 +51,24 @@ public class BackendUtil {
private List<BackendV2.BackendRowV2> initBackends(String beNodes) {
List<BackendV2.BackendRowV2> backends = new ArrayList<>();
List<String> nodes = Arrays.asList(beNodes.split(","));
- nodes.forEach(node -> {
- if (tryHttpConnection(node)) {
- node = node.trim();
- String[] ipAndPort = node.split(":");
- BackendRowV2 backendRowV2 = new BackendRowV2();
- backendRowV2.setIp(ipAndPort[0]);
- backendRowV2.setHttpPort(Integer.parseInt(ipAndPort[1]));
- backendRowV2.setAlive(true);
- backends.add(backendRowV2);
- }
- });
+ nodes.forEach(
+ node -> {
+ if (tryHttpConnection(node)) {
+ node = node.trim();
+ String[] ipAndPort = node.split(":");
+ BackendRowV2 backendRowV2 = new BackendRowV2();
+ backendRowV2.setIp(ipAndPort[0]);
+ backendRowV2.setHttpPort(Integer.parseInt(ipAndPort[1]));
+ backendRowV2.setAlive(true);
+ backends.add(backendRowV2);
+ }
+ });
return backends;
}
- public static BackendUtil getInstance(DorisOptions dorisOptions, DorisReadOptions readOptions, Logger logger){
- if(StringUtils.isNotEmpty(dorisOptions.getBenodes())){
+ public static BackendUtil getInstance(
+ DorisOptions dorisOptions, DorisReadOptions readOptions, Logger logger) {
+ if (StringUtils.isNotEmpty(dorisOptions.getBenodes())) {
return new BackendUtil(dorisOptions.getBenodes());
} else {
return new BackendUtil(RestService.getBackendsV2(dorisOptions, readOptions, logger));
@@ -90,7 +91,7 @@ public class BackendUtil {
try {
backend = "http://" + backend;
URL url = new URL(backend);
- HttpURLConnection co = (HttpURLConnection) url.openConnection();
+ HttpURLConnection co = (HttpURLConnection) url.openConnection();
co.setConnectTimeout(60000);
co.connect();
co.disconnect();
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java
index 70ee154..ab6ee1f 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java
@@ -19,9 +19,7 @@ package org.apache.doris.flink.sink;
import java.util.Objects;
-/**
- * DorisCommittable hold the info for Committer to commit.
- */
+/** DorisCommittable hold the info for Committer to commit. */
public class DorisCommittable {
private final String hostPort;
private final String db;
@@ -54,9 +52,9 @@ public class DorisCommittable {
return false;
}
DorisCommittable that = (DorisCommittable) o;
- return txnID == that.txnID &&
- Objects.equals(hostPort, that.hostPort) &&
- Objects.equals(db, that.db);
+ return txnID == that.txnID
+ && Objects.equals(hostPort, that.hostPort)
+ && Objects.equals(db, that.db);
}
@Override
@@ -66,10 +64,15 @@ public class DorisCommittable {
@Override
public String toString() {
- return "DorisCommittable{" +
- "hostPort='" + hostPort + '\'' +
- ", db='" + db + '\'' +
- ", txnID=" + txnID +
- '}';
+ return "DorisCommittable{"
+ + "hostPort='"
+ + hostPort
+ + '\''
+ + ", db='"
+ + db
+ + '\''
+ + ", txnID="
+ + txnID
+ + '}';
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java
index 7f7fbad..6dcf41a 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.sink;
import org.apache.flink.core.io.SimpleVersionedSerializer;
@@ -24,9 +25,7 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-/**
- * define how to serialize DorisCommittable.
- */
+/** define how to serialize DorisCommittable. */
class DorisCommittableSerializer implements SimpleVersionedSerializer<DorisCommittable> {
@Override
@@ -37,7 +36,7 @@ class DorisCommittableSerializer implements SimpleVersionedSerializer<DorisCommi
@Override
public byte[] serialize(DorisCommittable dorisCommittable) throws IOException {
try (final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- final DataOutputStream out = new DataOutputStream(baos)) {
+ final DataOutputStream out = new DataOutputStream(baos)) {
out.writeUTF(dorisCommittable.getHostPort());
out.writeUTF(dorisCommittable.getDb());
out.writeLong(dorisCommittable.getTxnID());
@@ -50,7 +49,7 @@ class DorisCommittableSerializer implements SimpleVersionedSerializer<DorisCommi
@Override
public DorisCommittable deserialize(int version, byte[] serialized) throws IOException {
try (final ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
- final DataInputStream in = new DataInputStream(bais)) {
+ final DataInputStream in = new DataInputStream(bais)) {
final String hostPort = in.readUTF();
final String db = in.readUTF();
final long txnId = in.readLong();
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java
index ec2b736..3da50b2 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java
@@ -17,20 +17,21 @@
package org.apache.doris.flink.sink;
+import org.apache.flink.api.connector.sink2.Committer;
+import org.apache.flink.api.connector.sink2.StatefulSink;
+import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.util.Preconditions;
+
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.doris.flink.rest.RestService;
import org.apache.doris.flink.sink.committer.DorisCommitter;
-import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
import org.apache.doris.flink.sink.writer.DorisWriter;
import org.apache.doris.flink.sink.writer.DorisWriterState;
import org.apache.doris.flink.sink.writer.DorisWriterStateSerializer;
-import org.apache.flink.api.connector.sink2.Committer;
-import org.apache.flink.api.connector.sink2.StatefulSink;
-import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
-import org.apache.flink.core.io.SimpleVersionedSerializer;
-import org.apache.flink.util.Preconditions;
+import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,13 +40,13 @@ import java.util.Collection;
import java.util.Collections;
/**
- * Load data into Doris based on 2PC.
- * see {@link DorisWriter} and {@link DorisCommitter}.
+ * Load data into Doris based on 2PC. see {@link DorisWriter} and {@link DorisCommitter}.
+ *
* @param <IN> type of record.
*/
public class DorisSink<IN>
implements StatefulSink<IN, DorisWriterState>,
- TwoPhaseCommittingSink<IN, DorisCommittable>{
+ TwoPhaseCommittingSink<IN, DorisCommittable> {
private static final Logger LOG = LoggerFactory.getLogger(DorisSink.class);
private final DorisOptions dorisOptions;
@@ -53,10 +54,11 @@ public class DorisSink<IN>
private final DorisExecutionOptions dorisExecutionOptions;
private final DorisRecordSerializer<IN> serializer;
- public DorisSink(DorisOptions dorisOptions,
- DorisReadOptions dorisReadOptions,
- DorisExecutionOptions dorisExecutionOptions,
- DorisRecordSerializer<IN> serializer) {
+ public DorisSink(
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions dorisExecutionOptions,
+ DorisRecordSerializer<IN> serializer) {
this.dorisOptions = dorisOptions;
this.dorisReadOptions = dorisReadOptions;
this.dorisExecutionOptions = dorisExecutionOptions;
@@ -64,31 +66,46 @@ public class DorisSink<IN>
checkKeyType();
}
- /**
- * The uniq model has 2pc close by default unless 2pc is forced open
- */
+ /** The uniq model has 2pc close by default unless 2pc is forced open. */
private void checkKeyType() {
if (dorisExecutionOptions.enabled2PC()
&& !dorisExecutionOptions.force2PC()
- && RestService.isUniqueKeyType(dorisOptions, dorisReadOptions, LOG)){
+ && RestService.isUniqueKeyType(dorisOptions, dorisReadOptions, LOG)) {
dorisExecutionOptions.setEnable2PC(false);
}
}
@Override
public DorisWriter<IN> createWriter(InitContext initContext) throws IOException {
- DorisWriter<IN> dorisWriter = new DorisWriter<>(initContext, Collections.emptyList(), serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions);
+ DorisWriter<IN> dorisWriter =
+ new DorisWriter<>(
+ initContext,
+ Collections.emptyList(),
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ dorisExecutionOptions);
return dorisWriter;
}
@Override
public Committer<DorisCommittable> createCommitter() throws IOException {
- return new DorisCommitter(dorisOptions, dorisReadOptions, dorisExecutionOptions.getMaxRetries());
+ return new DorisCommitter(
+ dorisOptions, dorisReadOptions, dorisExecutionOptions.getMaxRetries());
}
@Override
- public DorisWriter<IN> restoreWriter(InitContext initContext, Collection<DorisWriterState> recoveredState) throws IOException {
- DorisWriter<IN> dorisWriter = new DorisWriter<>(initContext, recoveredState, serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions);
+ public DorisWriter<IN> restoreWriter(
+ InitContext initContext, Collection<DorisWriterState> recoveredState)
+ throws IOException {
+ DorisWriter<IN> dorisWriter =
+ new DorisWriter<>(
+ initContext,
+ recoveredState,
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ dorisExecutionOptions);
return dorisWriter;
}
@@ -102,13 +119,13 @@ public class DorisSink<IN>
return new DorisCommittableSerializer();
}
-
public static <IN> Builder<IN> builder() {
return new Builder<>();
}
/**
* build for DorisSink.
+ *
* @param <IN> record type.
*/
public static class Builder<IN> {
@@ -141,10 +158,11 @@ public class DorisSink<IN>
Preconditions.checkNotNull(dorisOptions);
Preconditions.checkNotNull(dorisExecutionOptions);
Preconditions.checkNotNull(serializer);
- if(dorisReadOptions == null) {
+ if (dorisReadOptions == null) {
dorisReadOptions = DorisReadOptions.builder().build();
}
- return new DorisSink<>(dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer);
+ return new DorisSink<>(
+ dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer);
}
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java
index f83cc79..4f4a616 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java
@@ -26,9 +26,7 @@ import static org.apache.doris.flink.sink.writer.LoadConstants.FIELD_DELIMITER_K
import static org.apache.doris.flink.sink.writer.LoadConstants.LINE_DELIMITER_DEFAULT;
import static org.apache.doris.flink.sink.writer.LoadConstants.LINE_DELIMITER_KEY;
-/**
- * Handler for escape in properties.
- */
+/** Handler for escape in properties. */
public class EscapeHandler {
public static final String ESCAPE_DELIMITERS_FLAGS = "\\x";
public static final Pattern ESCAPE_PATTERN = Pattern.compile("\\\\x([0-9|a-f|A-F]{2})");
@@ -38,7 +36,8 @@ public class EscapeHandler {
Matcher m = ESCAPE_PATTERN.matcher(source);
StringBuffer buf = new StringBuffer();
while (m.find()) {
- m.appendReplacement(buf, String.format("%s", (char) Integer.parseInt(m.group(1), 16)));
+ m.appendReplacement(
+ buf, String.format("%s", (char) Integer.parseInt(m.group(1), 16)));
}
m.appendTail(buf);
return buf.toString();
@@ -47,13 +46,12 @@ public class EscapeHandler {
}
public static void handle(Properties properties) {
- String fieldDelimiter = properties.getProperty(FIELD_DELIMITER_KEY,
- FIELD_DELIMITER_DEFAULT);
+ String fieldDelimiter =
+ properties.getProperty(FIELD_DELIMITER_KEY, FIELD_DELIMITER_DEFAULT);
if (fieldDelimiter.contains(ESCAPE_DELIMITERS_FLAGS)) {
properties.setProperty(FIELD_DELIMITER_KEY, escapeString(fieldDelimiter));
}
- String lineDelimiter = properties.getProperty(LINE_DELIMITER_KEY,
- LINE_DELIMITER_DEFAULT);
+ String lineDelimiter = properties.getProperty(LINE_DELIMITER_KEY, LINE_DELIMITER_DEFAULT);
if (lineDelimiter.contains(ESCAPE_DELIMITERS_FLAGS)) {
properties.setProperty(LINE_DELIMITER_KEY, escapeString(lineDelimiter));
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java
index 28b4a65..814b3ce 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java
@@ -22,7 +22,7 @@ import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
import java.net.URI;
public class HttpGetWithEntity extends HttpEntityEnclosingRequestBase {
- private final static String METHOD_NAME = "GET";
+ private static final String METHOD_NAME = "GET";
@Override
public String getMethod() {
@@ -33,4 +33,4 @@ public class HttpGetWithEntity extends HttpEntityEnclosingRequestBase {
super();
setURI(URI.create(uri));
}
-}
\ No newline at end of file
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java
index b59f2c1..a833e2e 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java
@@ -17,9 +17,10 @@
package org.apache.doris.flink.sink;
+import org.apache.flink.util.Preconditions;
+
import org.apache.commons.codec.binary.Base64;
import org.apache.doris.flink.sink.writer.LoadConstants;
-import org.apache.flink.util.Preconditions;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.client.methods.HttpPut;
@@ -30,13 +31,12 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
-/**
- * Builder for HttpPut.
- */
+/** Builder for HttpPut. */
public class HttpPutBuilder {
String url;
Map<String, String> header;
HttpEntity httpEntity;
+
public HttpPutBuilder() {
header = new HashMap<>();
}
@@ -52,7 +52,7 @@ public class HttpPutBuilder {
}
public HttpPutBuilder addHiddenColumns(boolean add) {
- if(add){
+ if (add) {
header.put("hidden_columns", LoadConstants.DORIS_DELETE_SIGN);
}
return this;
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java
index baf3391..7ad87f6 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java
@@ -22,21 +22,19 @@ import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
-/**
- * util to build http client.
- */
+/** util to build http client. */
public class HttpUtil {
- private final HttpClientBuilder httpClientBuilder = HttpClients
- .custom()
- .setRedirectStrategy(new DefaultRedirectStrategy() {
- @Override
- protected boolean isRedirectable(String method) {
- return true;
- }
- });
+ private final HttpClientBuilder httpClientBuilder =
+ HttpClients.custom()
+ .setRedirectStrategy(
+ new DefaultRedirectStrategy() {
+ @Override
+ protected boolean isRedirectable(String method) {
+ return true;
+ }
+ });
public CloseableHttpClient getHttpClient() {
return httpClientBuilder.build();
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java
index d1815b8..10265b9 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java
@@ -17,13 +17,10 @@
package org.apache.doris.flink.sink;
-/**
- * enum of LoadStatus.
- */
+/** enum of LoadStatus. */
public class LoadStatus {
public static final String SUCCESS = "Success";
public static final String PUBLISH_TIMEOUT = "Publish Timeout";
public static final String LABEL_ALREADY_EXIST = "Label Already Exists";
public static final String FAIL = "Fail";
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java
index 8839a1a..fc23ec9 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java
@@ -19,16 +19,15 @@ package org.apache.doris.flink.sink;
import java.util.regex.Pattern;
-/**
- * util for handle response.
- */
+/** util for handle response. */
public class ResponseUtil {
public static final Pattern LABEL_EXIST_PATTERN =
Pattern.compile("Label \\[(.*)\\] has already been used, relate to txn \\[(\\d+)\\]");
public static final Pattern COMMITTED_PATTERN =
- Pattern.compile("transaction \\[(\\d+)\\] is already \\b(COMMITTED|committed|VISIBLE|visible)\\b, not pre-committed.");
+ Pattern.compile(
+ "transaction \\[(\\d+)\\] is already \\b(COMMITTED|committed|VISIBLE|visible)\\b, not pre-committed.");
public static boolean isCommitted(String msg) {
- return COMMITTED_PATTERN.matcher(msg).find();
+ return COMMITTED_PATTERN.matcher(msg).find();
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java
index 1de6253..297cb18 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java
@@ -18,15 +18,13 @@
package org.apache.doris.flink.sink.batch;
import org.apache.flink.annotation.VisibleForTesting;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
-
-/**
- * buffer to queue
- */
+/** buffer to queue. */
public class BatchRecordBuffer {
private static final Logger LOG = LoggerFactory.getLogger(BatchRecordBuffer.class);
public static final String LINE_SEPARATOR = "\n";
@@ -39,7 +37,7 @@ public class BatchRecordBuffer {
private String database;
private String table;
- public BatchRecordBuffer(){}
+ public BatchRecordBuffer() {}
public BatchRecordBuffer(byte[] lineDelimiter, int bufferSize) {
super();
@@ -57,7 +55,7 @@ public class BatchRecordBuffer {
public void insert(byte[] record) {
ensureCapacity(record.length);
- if(loadBatchFirstRecord){
+ if (loadBatchFirstRecord) {
loadBatchFirstRecord = false;
} else {
this.buffer.put(this.lineDelimiter);
@@ -70,7 +68,7 @@ public class BatchRecordBuffer {
@VisibleForTesting
public void ensureCapacity(int length) {
int lineDelimiterSize = this.lineDelimiter.length;
- if(buffer.remaining() - lineDelimiterSize >= length){
+ if (buffer.remaining() - lineDelimiterSize >= length) {
return;
}
int currentRemain = buffer.remaining();
@@ -87,7 +85,12 @@ public class BatchRecordBuffer {
tmp.put(buffer);
buffer.clear();
buffer = tmp;
- LOG.info("record length {},buffer remain {} ,grow capacity {} to {}", length, currentRemain, currentCapacity, newCapacity);
+ LOG.info(
+ "record length {},buffer remain {} ,grow capacity {} to {}",
+ length,
+ currentRemain,
+ currentCapacity,
+ newCapacity);
}
public String getLabelName() {
@@ -98,21 +101,19 @@ public class BatchRecordBuffer {
this.labelName = labelName;
}
- /**
- * @return true if buffer is empty
- */
+ /** @return true if buffer is empty */
public boolean isEmpty() {
return numOfRecords == 0;
}
public ByteBuffer getData() {
- //change mode
+ // change mode
buffer.flip();
- LOG.debug("flush buffer: {} records, {} bytes",getNumOfRecords(),getBufferSizeBytes());
+ LOG.debug("flush buffer: {} records, {} bytes", getNumOfRecords(), getBufferSizeBytes());
return buffer;
}
- public void clear(){
+ public void clear() {
this.buffer.clear();
this.numOfRecords = 0;
this.bufferSizeBytes = 0;
@@ -120,33 +121,26 @@ public class BatchRecordBuffer {
this.loadBatchFirstRecord = true;
}
- public ByteBuffer getBuffer(){
+ public ByteBuffer getBuffer() {
return buffer;
}
- /**
- * @return Number of records in this buffer
- */
+
+ /** @return Number of records in this buffer */
public int getNumOfRecords() {
return numOfRecords;
}
- /**
- * @return Buffer size in bytes
- */
+ /** @return Buffer size in bytes */
public int getBufferSizeBytes() {
return bufferSizeBytes;
}
- /**
- * @param numOfRecords Updates number of records (Usually by 1)
- */
+ /** @param numOfRecords Updates number of records (Usually by 1) */
public void setNumOfRecords(int numOfRecords) {
this.numOfRecords = numOfRecords;
}
- /**
- * @param bufferSizeBytes Updates sum of size of records present in this buffer (Bytes)
- */
+ /** @param bufferSizeBytes Updates sum of size of records present in this buffer (Bytes) */
public void setBufferSizeBytes(int bufferSizeBytes) {
this.bufferSizeBytes = bufferSizeBytes;
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
index eea67b8..d9a424b 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
@@ -17,13 +17,14 @@
package org.apache.doris.flink.sink.batch;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.util.Preconditions;
+
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
-import org.apache.flink.api.connector.sink2.Sink;
-import org.apache.flink.api.connector.sink2.SinkWriter;
-import org.apache.flink.util.Preconditions;
import java.io.IOException;
@@ -33,10 +34,11 @@ public class DorisBatchSink<IN> implements Sink<IN> {
private final DorisExecutionOptions dorisExecutionOptions;
private final DorisRecordSerializer<IN> serializer;
- public DorisBatchSink(DorisOptions dorisOptions,
- DorisReadOptions dorisReadOptions,
- DorisExecutionOptions dorisExecutionOptions,
- DorisRecordSerializer<IN> serializer) {
+ public DorisBatchSink(
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions dorisExecutionOptions,
+ DorisRecordSerializer<IN> serializer) {
this.dorisOptions = dorisOptions;
this.dorisReadOptions = dorisReadOptions;
this.dorisExecutionOptions = dorisExecutionOptions;
@@ -45,7 +47,13 @@ public class DorisBatchSink<IN> implements Sink<IN> {
@Override
public SinkWriter<IN> createWriter(InitContext initContext) throws IOException {
- DorisBatchWriter<IN> dorisBatchWriter = new DorisBatchWriter<IN>(initContext, serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions);
+ DorisBatchWriter<IN> dorisBatchWriter =
+ new DorisBatchWriter<IN>(
+ initContext,
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ dorisExecutionOptions);
dorisBatchWriter.initializeLoad();
return dorisBatchWriter;
}
@@ -56,6 +64,7 @@ public class DorisBatchSink<IN> implements Sink<IN> {
/**
* build for DorisBatchSink.
+ *
* @param <IN> record type.
*/
public static class Builder<IN> {
@@ -74,7 +83,8 @@ public class DorisBatchSink<IN> implements Sink<IN> {
return this;
}
- public DorisBatchSink.Builder<IN> setDorisExecutionOptions(DorisExecutionOptions dorisExecutionOptions) {
+ public DorisBatchSink.Builder<IN> setDorisExecutionOptions(
+ DorisExecutionOptions dorisExecutionOptions) {
this.dorisExecutionOptions = dorisExecutionOptions;
return this;
}
@@ -88,10 +98,11 @@ public class DorisBatchSink<IN> implements Sink<IN> {
Preconditions.checkNotNull(dorisOptions);
Preconditions.checkNotNull(dorisExecutionOptions);
Preconditions.checkNotNull(serializer);
- if(dorisReadOptions == null) {
+ if (dorisReadOptions == null) {
dorisReadOptions = DorisReadOptions.builder().build();
}
- return new DorisBatchSink<>(dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer);
+ return new DorisBatchSink<>(
+ dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer);
}
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java
index 6adb436..7ca0cda 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java
@@ -17,6 +17,8 @@
package org.apache.doris.flink.sink.batch;
+import org.apache.flink.util.Preconditions;
+
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
import org.apache.doris.flink.cfg.DorisExecutionOptions;
@@ -30,7 +32,6 @@ import org.apache.doris.flink.sink.EscapeHandler;
import org.apache.doris.flink.sink.HttpPutBuilder;
import org.apache.doris.flink.sink.HttpUtil;
import org.apache.doris.flink.sink.writer.LabelGenerator;
-import org.apache.flink.util.Preconditions;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -63,14 +64,13 @@ import static org.apache.doris.flink.sink.LoadStatus.SUCCESS;
import static org.apache.doris.flink.sink.writer.LoadConstants.LINE_DELIMITER_DEFAULT;
import static org.apache.doris.flink.sink.writer.LoadConstants.LINE_DELIMITER_KEY;
-/**
- * async stream load
- **/
+/** async stream load. */
public class DorisBatchStreamLoad implements Serializable {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(DorisBatchStreamLoad.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- private static final List<String> DORIS_SUCCESS_STATUS = new ArrayList<>(Arrays.asList(SUCCESS, PUBLISH_TIMEOUT));
+ private static final List<String> DORIS_SUCCESS_STATUS =
+ new ArrayList<>(Arrays.asList(SUCCESS, PUBLISH_TIMEOUT));
private final LabelGenerator labelGenerator;
private final byte[] lineDelimiter;
private static final String LOAD_URL_PATTERN = "http://%s/api/%s/%s/_stream_load";
@@ -90,50 +90,78 @@ public class DorisBatchStreamLoad implements Serializable {
private CloseableHttpClient httpClient = new HttpUtil().getHttpClient();
private BackendUtil backendUtil;
- public DorisBatchStreamLoad(DorisOptions dorisOptions,
- DorisReadOptions dorisReadOptions,
- DorisExecutionOptions executionOptions,
- LabelGenerator labelGenerator) {
- this.backendUtil = StringUtils.isNotEmpty(dorisOptions.getBenodes()) ? new BackendUtil(
- dorisOptions.getBenodes())
- : new BackendUtil(RestService.getBackendsV2(dorisOptions, dorisReadOptions, LOG));
+ public DorisBatchStreamLoad(
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions executionOptions,
+ LabelGenerator labelGenerator) {
+ this.backendUtil =
+ StringUtils.isNotEmpty(dorisOptions.getBenodes())
+ ? new BackendUtil(dorisOptions.getBenodes())
+ : new BackendUtil(
+ RestService.getBackendsV2(dorisOptions, dorisReadOptions, LOG));
this.hostPort = backendUtil.getAvailableBackend();
this.username = dorisOptions.getUsername();
this.password = dorisOptions.getPassword();
this.loadProps = executionOptions.getStreamLoadProp();
this.labelGenerator = labelGenerator;
- this.lineDelimiter = EscapeHandler.escapeString(loadProps.getProperty(LINE_DELIMITER_KEY, LINE_DELIMITER_DEFAULT)).getBytes();
+ this.lineDelimiter =
+ EscapeHandler.escapeString(
+ loadProps.getProperty(LINE_DELIMITER_KEY, LINE_DELIMITER_DEFAULT))
+ .getBytes();
this.executionOptions = executionOptions;
this.flushQueue = new LinkedBlockingDeque<>(executionOptions.getFlushQueueSize());
- if(StringUtils.isNotBlank(dorisOptions.getTableIdentifier())){
+ if (StringUtils.isNotBlank(dorisOptions.getTableIdentifier())) {
String[] tableInfo = dorisOptions.getTableIdentifier().split("\\.");
- Preconditions.checkState(tableInfo.length == 2, "tableIdentifier input error, the format is database.table");
+ Preconditions.checkState(
+ tableInfo.length == 2,
+ "tableIdentifier input error, the format is database.table");
this.loadUrl = String.format(LOAD_URL_PATTERN, hostPort, tableInfo[0], tableInfo[1]);
}
- this.loadAsyncExecutor= new LoadAsyncExecutor();
- this.loadExecutorService = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1), new DefaultThreadFactory("streamload-executor"), new ThreadPoolExecutor.AbortPolicy());
+ this.loadAsyncExecutor = new LoadAsyncExecutor();
+ this.loadExecutorService =
+ new ThreadPoolExecutor(
+ 1,
+ 1,
+ 0L,
+ TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<>(1),
+ new DefaultThreadFactory("streamload-executor"),
+ new ThreadPoolExecutor.AbortPolicy());
this.started = new AtomicBoolean(true);
this.loadExecutorService.execute(loadAsyncExecutor);
}
/**
* write record into cache.
+ *
* @param record
* @throws IOException
*/
- public synchronized void writeRecord(String database, String table, byte[] record) throws InterruptedException {
+ public synchronized void writeRecord(String database, String table, byte[] record)
+ throws InterruptedException {
checkFlushException();
String bufferKey = getTableIdentifier(database, table);
- BatchRecordBuffer buffer = bufferMap.computeIfAbsent(bufferKey, k -> new BatchRecordBuffer(database, table, this.lineDelimiter, executionOptions.getBufferFlushMaxBytes()));
+ BatchRecordBuffer buffer =
+ bufferMap.computeIfAbsent(
+ bufferKey,
+ k ->
+ new BatchRecordBuffer(
+ database,
+ table,
+ this.lineDelimiter,
+ executionOptions.getBufferFlushMaxBytes()));
buffer.insert(record);
- //When it exceeds 80% of the byteSize,to flush, to avoid triggering bytebuffer expansion
+ // When it exceeds 80% of the byteSize,to flush, to avoid triggering bytebuffer expansion
if (buffer.getBufferSizeBytes() >= executionOptions.getBufferFlushMaxBytes() * 0.8
- || (executionOptions.getBufferFlushMaxRows() != 0 && buffer.getNumOfRecords() >= executionOptions.getBufferFlushMaxRows())) {
- flush(bufferKey,false);
+ || (executionOptions.getBufferFlushMaxRows() != 0
+ && buffer.getNumOfRecords() >= executionOptions.getBufferFlushMaxRows())) {
+ flush(bufferKey, false);
}
}
- public synchronized void flush(String bufferKey, boolean waitUtilDone) throws InterruptedException {
+ public synchronized void flush(String bufferKey, boolean waitUtilDone)
+ throws InterruptedException {
checkFlushException();
if (null == bufferKey) {
for (String key : bufferMap.keySet()) {
@@ -155,9 +183,9 @@ public class DorisBatchStreamLoad implements Serializable {
bufferMap.remove(bufferKey);
}
- private void putRecordToFlushQueue(BatchRecordBuffer buffer){
+ private void putRecordToFlushQueue(BatchRecordBuffer buffer) {
checkFlushException();
- if(!loadThreadAlive){
+ if (!loadThreadAlive) {
throw new RuntimeException("load thread already exit, write was interrupted");
}
try {
@@ -174,7 +202,7 @@ public class DorisBatchStreamLoad implements Serializable {
}
private void waitAsyncLoadFinish() {
- for(int i = 0; i < executionOptions.getFlushQueueSize() + 1 ; i++){
+ for (int i = 0; i < executionOptions.getFlushQueueSize() + 1; i++) {
BatchRecordBuffer empty = new BatchRecordBuffer();
putRecordToFlushQueue(empty);
}
@@ -184,11 +212,11 @@ public class DorisBatchStreamLoad implements Serializable {
return database + "." + table;
}
- public void close(){
- //close async executor
+ public void close() {
+ // close async executor
this.loadExecutorService.shutdown();
this.started.set(false);
- //clear buffer
+ // clear buffer
this.flushQueue.clear();
}
@@ -201,7 +229,7 @@ public class DorisBatchStreamLoad implements Serializable {
BatchRecordBuffer buffer = null;
try {
buffer = flushQueue.poll(2000L, TimeUnit.MILLISECONDS);
- if(buffer == null){
+ if (buffer == null) {
continue;
}
if (buffer.getLabelName() != null) {
@@ -210,7 +238,7 @@ public class DorisBatchStreamLoad implements Serializable {
} catch (Exception e) {
LOG.error("worker running error", e);
exception.set(e);
- //clear queue to avoid writer thread blocking
+ // clear queue to avoid writer thread blocking
flushQueue.clear();
break;
}
@@ -219,15 +247,15 @@ public class DorisBatchStreamLoad implements Serializable {
loadThreadAlive = false;
}
- /**
- * execute stream load
- */
- public void load(String label, BatchRecordBuffer buffer) throws IOException{
+ /** execute stream load. */
+ public void load(String label, BatchRecordBuffer buffer) throws IOException {
refreshLoadUrl(buffer.getDatabase(), buffer.getTable());
ByteBuffer data = buffer.getData();
- ByteArrayEntity entity = new ByteArrayEntity(data.array(), data.arrayOffset(), data.limit());
+ ByteArrayEntity entity =
+ new ByteArrayEntity(data.array(), data.arrayOffset(), data.limit());
HttpPutBuilder putBuilder = new HttpPutBuilder();
- putBuilder.setUrl(loadUrl)
+ putBuilder
+ .setUrl(loadUrl)
.baseAuth(username, password)
.setLabel(label)
.addCommonHeader()
@@ -243,21 +271,27 @@ public class DorisBatchStreamLoad implements Serializable {
if (statusCode == 200 && response.getEntity() != null) {
String loadResult = EntityUtils.toString(response.getEntity());
LOG.info("load Result {}", loadResult);
- RespContent respContent = OBJECT_MAPPER.readValue(loadResult, RespContent.class);
+ RespContent respContent =
+ OBJECT_MAPPER.readValue(loadResult, RespContent.class);
if (!DORIS_SUCCESS_STATUS.contains(respContent.getStatus())) {
- String errMsg = String.format("stream load error: %s, see more in %s", respContent.getMessage(), respContent.getErrorURL());
+ String errMsg =
+ String.format(
+ "stream load error: %s, see more in %s",
+ respContent.getMessage(), respContent.getErrorURL());
throw new DorisBatchLoadException(errMsg);
- }else{
+ } else {
return;
}
}
- LOG.error("stream load failed with {}, reason {}, to retry", hostPort, response.getStatusLine().toString());
- }catch (Exception ex){
+ LOG.error(
+ "stream load failed with {}, reason {}, to retry",
+ hostPort,
+ response.getStatusLine().toString());
+ } catch (Exception ex) {
if (retry == executionOptions.getMaxRetries()) {
throw new DorisBatchLoadException("stream load error: ", ex);
}
LOG.error("stream load error with {}, to retry, cause by", hostPort, ex);
-
}
retry++;
// get available backend retry
@@ -268,7 +302,7 @@ public class DorisBatchStreamLoad implements Serializable {
buffer = null;
}
- private void refreshLoadUrl(String database, String table){
+ private void refreshLoadUrl(String database, String table) {
hostPort = backendUtil.getAvailableBackend();
loadUrl = String.format(LOAD_URL_PATTERN, hostPort, database, table);
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java
index 567758d..43aff7f 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java
@@ -17,17 +17,18 @@
package org.apache.doris.flink.sink.batch;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.doris.flink.sink.writer.LabelGenerator;
import org.apache.doris.flink.sink.writer.serializer.DorisRecord;
import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
-import org.apache.flink.api.connector.sink2.Sink;
-import org.apache.flink.api.connector.sink2.SinkWriter;
-import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.StringUtils;
-import org.apache.flink.util.concurrent.ExecutorThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,21 +52,26 @@ public class DorisBatchWriter<IN> implements SinkWriter<IN> {
private String database;
private String table;
- public DorisBatchWriter(Sink.InitContext initContext,
- DorisRecordSerializer<IN> serializer,
- DorisOptions dorisOptions,
- DorisReadOptions dorisReadOptions,
- DorisExecutionOptions executionOptions) {
- if(!StringUtils.isNullOrWhitespaceOnly(dorisOptions.getTableIdentifier())){
+ public DorisBatchWriter(
+ Sink.InitContext initContext,
+ DorisRecordSerializer<IN> serializer,
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions executionOptions) {
+ if (!StringUtils.isNullOrWhitespaceOnly(dorisOptions.getTableIdentifier())) {
String[] tableInfo = dorisOptions.getTableIdentifier().split("\\.");
- Preconditions.checkState(tableInfo.length == 2, "tableIdentifier input error, the format is database.table");
+ Preconditions.checkState(
+ tableInfo.length == 2,
+ "tableIdentifier input error, the format is database.table");
this.database = tableInfo[0];
this.table = tableInfo[1];
}
LOG.info("labelPrefix " + executionOptions.getLabelPrefix());
this.labelPrefix = executionOptions.getLabelPrefix() + "_" + initContext.getSubtaskId();
this.labelGenerator = new LabelGenerator(labelPrefix, false);
- this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, new ExecutorThreadFactory("stream-load-flush-interval"));
+ this.scheduledExecutorService =
+ new ScheduledThreadPoolExecutor(
+ 1, new ExecutorThreadFactory("stream-load-flush-interval"));
this.serializer = serializer;
this.dorisOptions = dorisOptions;
this.dorisReadOptions = dorisReadOptions;
@@ -74,9 +80,13 @@ public class DorisBatchWriter<IN> implements SinkWriter<IN> {
}
public void initializeLoad() throws IOException {
- this.batchStreamLoad = new DorisBatchStreamLoad(dorisOptions, dorisReadOptions, executionOptions, labelGenerator);
- // when uploading data in streaming mode, we need to regularly detect whether there are exceptions.
- scheduledExecutorService.scheduleWithFixedDelay(this::intervalFlush, flushIntervalMs, flushIntervalMs, TimeUnit.MILLISECONDS);
+ this.batchStreamLoad =
+ new DorisBatchStreamLoad(
+ dorisOptions, dorisReadOptions, executionOptions, labelGenerator);
+ // when uploading data in streaming mode,
+ // we need to regularly detect whether there areexceptions.
+ scheduledExecutorService.scheduleWithFixedDelay(
+ this::intervalFlush, flushIntervalMs, flushIntervalMs, TimeUnit.MILLISECONDS);
}
private void intervalFlush() {
@@ -94,22 +104,23 @@ public class DorisBatchWriter<IN> implements SinkWriter<IN> {
String db = this.database;
String tbl = this.table;
DorisRecord record = serializer.serialize(in);
- if(record == null || record.getRow() == null){
- //ddl or value is null
+ if (record == null || record.getRow() == null) {
+ // ddl or value is null
return;
}
- //multi table load
- if(record.getTableIdentifier() != null){
+ // multi table load
+ if (record.getTableIdentifier() != null) {
db = record.getDatabase();
tbl = record.getTable();
}
batchStreamLoad.writeRecord(db, tbl, record.getRow());
}
+
@Override
public void flush(boolean flush) throws IOException, InterruptedException {
checkFlushException();
LOG.info("checkpoint flush triggered.");
- batchStreamLoad.flush(null, true);
+ batchStreamLoad.flush(null, true);
}
@Override
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java
index 7f4d269..936487d 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java
@@ -22,8 +22,7 @@ public class RecordWithMeta {
private String table;
private String record;
- public RecordWithMeta() {
- }
+ public RecordWithMeta() {}
public RecordWithMeta(String database, String table, String record) {
this.database = database;
@@ -58,5 +57,4 @@ public class RecordWithMeta {
public String getTableIdentifier() {
return this.database + "." + this.table;
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java
index ffcb8af..3959c26 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java
@@ -17,6 +17,8 @@
package org.apache.doris.flink.sink.committer;
+import org.apache.flink.api.connector.sink2.Committer;
+
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
@@ -29,7 +31,6 @@ import org.apache.doris.flink.sink.DorisCommittable;
import org.apache.doris.flink.sink.HttpPutBuilder;
import org.apache.doris.flink.sink.HttpUtil;
import org.apache.doris.flink.sink.ResponseUtil;
-import org.apache.flink.api.connector.sink2.Committer;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPut;
@@ -46,9 +47,7 @@ import java.util.Map;
import static org.apache.doris.flink.sink.LoadStatus.SUCCESS;
-/**
- * The committer to commit transaction.
- */
+/** The committer to commit transaction. */
public class DorisCommitter implements Committer<DorisCommittable>, Closeable {
private static final Logger LOG = LoggerFactory.getLogger(DorisCommitter.class);
private static final String commitPattern = "http://%s/api/%s/_stream_load_2pc";
@@ -60,42 +59,51 @@ public class DorisCommitter implements Committer<DorisCommittable>, Closeable {
int maxRetry;
- public DorisCommitter(DorisOptions dorisOptions, DorisReadOptions dorisReadOptions, int maxRetry) {
+ public DorisCommitter(
+ DorisOptions dorisOptions, DorisReadOptions dorisReadOptions, int maxRetry) {
this(dorisOptions, dorisReadOptions, maxRetry, new HttpUtil().getHttpClient());
}
- public DorisCommitter(DorisOptions dorisOptions, DorisReadOptions dorisReadOptions, int maxRetry, CloseableHttpClient client) {
+ public DorisCommitter(
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ int maxRetry,
+ CloseableHttpClient client) {
this.dorisOptions = dorisOptions;
this.dorisReadOptions = dorisReadOptions;
this.maxRetry = maxRetry;
this.httpClient = client;
- this.backendUtil = StringUtils.isNotEmpty(dorisOptions.getBenodes()) ? new BackendUtil(
- dorisOptions.getBenodes())
- : new BackendUtil(RestService.getBackendsV2(dorisOptions, dorisReadOptions, LOG));
+ this.backendUtil =
+ StringUtils.isNotEmpty(dorisOptions.getBenodes())
+ ? new BackendUtil(dorisOptions.getBenodes())
+ : new BackendUtil(
+ RestService.getBackendsV2(dorisOptions, dorisReadOptions, LOG));
}
@Override
- public void commit(Collection<CommitRequest<DorisCommittable>> requests) throws IOException, InterruptedException {
- for (CommitRequest<DorisCommittable> request: requests) {
+ public void commit(Collection<CommitRequest<DorisCommittable>> requests)
+ throws IOException, InterruptedException {
+ for (CommitRequest<DorisCommittable> request : requests) {
commitTransaction(request.getCommittable());
}
}
private void commitTransaction(DorisCommittable committable) throws IOException {
- //basic params
- HttpPutBuilder builder = new HttpPutBuilder()
- .addCommonHeader()
- .baseAuth(dorisOptions.getUsername(), dorisOptions.getPassword())
- .addTxnId(committable.getTxnID())
- .commit();
+ // basic params
+ HttpPutBuilder builder =
+ new HttpPutBuilder()
+ .addCommonHeader()
+ .baseAuth(dorisOptions.getUsername(), dorisOptions.getPassword())
+ .addTxnId(committable.getTxnID())
+ .commit();
- //hostPort
+ // hostPort
String hostPort = committable.getHostPort();
LOG.info("commit txn {} to host {}", committable.getTxnID(), hostPort);
int retry = 0;
while (retry <= maxRetry) {
- //get latest-url
+ // get latest-url
String url = String.format(commitPattern, hostPort, committable.getDb());
HttpPut httpPut = builder.setUrl(url).setEmptyEntity().build();
@@ -105,10 +113,14 @@ public class DorisCommitter implements Committer<DorisCommittable>, Closeable {
if (200 == statusLine.getStatusCode()) {
if (response.getEntity() != null) {
String loadResult = EntityUtils.toString(response.getEntity());
- Map<String, String> res = jsonMapper.readValue(loadResult, new TypeReference<HashMap<String, String>>() {
- });
- if (!res.get("status").equals(SUCCESS) && !ResponseUtil.isCommitted(res.get("msg"))) {
- throw new DorisRuntimeException("commit transaction failed " + loadResult);
+ Map<String, String> res =
+ jsonMapper.readValue(
+ loadResult,
+ new TypeReference<HashMap<String, String>>() {});
+ if (!res.get("status").equals(SUCCESS)
+ && !ResponseUtil.isCommitted(res.get("msg"))) {
+ throw new DorisRuntimeException(
+ "commit transaction failed " + loadResult);
} else {
LOG.info("load result {}", loadResult);
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java
index 6f1f2a9..5b9901f 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java
@@ -17,11 +17,11 @@
package org.apache.doris.flink.sink.schema;
-import org.apache.doris.flink.catalog.doris.FieldSchema;
+import org.apache.flink.util.StringUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.compress.utils.Lists;
-import org.apache.flink.util.StringUtils;
+import org.apache.doris.flink.catalog.doris.FieldSchema;
import java.util.List;
import java.util.Map;
@@ -35,9 +35,11 @@ public class SchemaChangeHelper {
private static final String ADD_DDL = "ALTER TABLE %s ADD COLUMN %s %s";
private static final String DROP_DDL = "ALTER TABLE %s DROP COLUMN %s";
private static final String RENAME_DDL = "ALTER TABLE %s RENAME COLUMN %s %s";
- private static final String CHECK_COLUMN_EXISTS = "SELECT COLUMN_NAME FROM information_schema.`COLUMNS` WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' AND COLUMN_NAME = '%s'";
+ private static final String CHECK_COLUMN_EXISTS =
+ "SELECT COLUMN_NAME FROM information_schema.`COLUMNS` WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' AND COLUMN_NAME = '%s'";
- public static void compareSchema(Map<String, FieldSchema> updateFiledSchemaMap,
+ public static void compareSchema(
+ Map<String, FieldSchema> updateFiledSchemaMap,
Map<String, FieldSchema> originFieldSchemaMap) {
dropFieldSchemas.clear();
addFieldSchemas.clear();
@@ -59,7 +61,10 @@ public class SchemaChangeHelper {
}
}
- public static List<String> generateRenameDDLSql(String table, String oldColumnName, String newColumnName,
+ public static List<String> generateRenameDDLSql(
+ String table,
+ String oldColumnName,
+ String newColumnName,
Map<String, FieldSchema> originFieldSchemaMap) {
ddlSchemas.clear();
List<String> ddlList = Lists.newArrayList();
@@ -93,7 +98,7 @@ public class SchemaChangeHelper {
return ddlList;
}
- public static String buildAddColumnDDL(String tableIdentifier, FieldSchema fieldSchema){
+ public static String buildAddColumnDDL(String tableIdentifier, FieldSchema fieldSchema) {
String name = fieldSchema.getName();
String type = fieldSchema.getTypeString();
String defaultValue = fieldSchema.getDefaultValue();
@@ -108,15 +113,16 @@ public class SchemaChangeHelper {
return addDDL;
}
- public static String buildDropColumnDDL(String tableIdentifier, String columName){
+ public static String buildDropColumnDDL(String tableIdentifier, String columName) {
return String.format(DROP_DDL, tableIdentifier, columName);
}
- public static String buildRenameColumnDDL(String tableIdentifier, String oldColumnName, String newColumnName){
+ public static String buildRenameColumnDDL(
+ String tableIdentifier, String oldColumnName, String newColumnName) {
return String.format(RENAME_DDL, tableIdentifier, oldColumnName, newColumnName);
}
- public static String buildColumnExistsQuery(String database, String table, String column){
+ public static String buildColumnExistsQuery(String database, String table, String column) {
return String.format(CHECK_COLUMN_EXISTS, database, table, column);
}
@@ -141,5 +147,4 @@ public class SchemaChangeHelper {
return isDropColumn;
}
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java
index 569d440..8860f6a 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java
@@ -17,6 +17,9 @@
package org.apache.doris.flink.sink.schema;
+import org.apache.flink.util.CollectionUtil;
+import org.apache.flink.util.StringUtils;
+
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.codec.binary.Base64;
@@ -28,8 +31,6 @@ import org.apache.doris.flink.exception.DorisSchemaChangeException;
import org.apache.doris.flink.exception.IllegalArgumentException;
import org.apache.doris.flink.rest.RestService;
import org.apache.doris.flink.sink.HttpGetWithEntity;
-import org.apache.flink.util.CollectionUtil;
-import org.apache.flink.util.StringUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
@@ -50,7 +51,8 @@ import java.util.Map;
public class SchemaChangeManager implements Serializable {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(SchemaChangeManager.class);
- private static final String CHECK_SCHEMA_CHANGE_API = "http://%s/api/enable_light_schema_change/%s/%s";
+ private static final String CHECK_SCHEMA_CHANGE_API =
+ "http://%s/api/enable_light_schema_change/%s/%s";
private static final String SCHEMA_CHANGE_API = "http://%s/api/query/default_cluster/%s";
private ObjectMapper objectMapper = new ObjectMapper();
private DorisOptions dorisOptions;
@@ -64,18 +66,24 @@ public class SchemaChangeManager implements Serializable {
return execute(createTableDDL, table.getDatabase());
}
- public boolean addColumn(String database, String table, FieldSchema field) throws IOException, IllegalArgumentException {
- if(checkColumnExists(database, table, field.getName())){
- LOG.warn("The column {} already exists in table {}, no need to add it again", field.getName(), table);
+ public boolean addColumn(String database, String table, FieldSchema field)
+ throws IOException, IllegalArgumentException {
+ if (checkColumnExists(database, table, field.getName())) {
+ LOG.warn(
+ "The column {} already exists in table {}, no need to add it again",
+ field.getName(),
+ table);
return true;
}
String tableIdentifier = getTableIdentifier(database, table);
String addColumnDDL = SchemaChangeHelper.buildAddColumnDDL(tableIdentifier, field);
- return schemaChange(database, table, buildRequestParam(false, field.getName()), addColumnDDL);
+ return schemaChange(
+ database, table, buildRequestParam(false, field.getName()), addColumnDDL);
}
- public boolean dropColumn(String database, String table, String columnName) throws IOException, IllegalArgumentException {
- if(!checkColumnExists(database, table, columnName)){
+ public boolean dropColumn(String database, String table, String columnName)
+ throws IOException, IllegalArgumentException {
+ if (!checkColumnExists(database, table, columnName)) {
LOG.warn("The column {} not exists in table {}, no need to drop", columnName, table);
return true;
}
@@ -84,14 +92,21 @@ public class SchemaChangeManager implements Serializable {
return schemaChange(database, table, buildRequestParam(true, columnName), dropColumnDDL);
}
- public boolean renameColumn(String database, String table, String oldColumnName, String newColumnName) throws IOException, IllegalArgumentException {
+ public boolean renameColumn(
+ String database, String table, String oldColumnName, String newColumnName)
+ throws IOException, IllegalArgumentException {
String tableIdentifier = getTableIdentifier(database, table);
- String renameColumnDDL = SchemaChangeHelper.buildRenameColumnDDL(tableIdentifier, oldColumnName, newColumnName);
- return schemaChange(database, table, buildRequestParam(true, oldColumnName), renameColumnDDL);
+ String renameColumnDDL =
+ SchemaChangeHelper.buildRenameColumnDDL(
+ tableIdentifier, oldColumnName, newColumnName);
+ return schemaChange(
+ database, table, buildRequestParam(true, oldColumnName), renameColumnDDL);
}
- public boolean schemaChange(String database, String table, Map<String, Object> params, String sql) throws IOException, IllegalArgumentException {
- if(checkSchemaChange(database, table, params)){
+ public boolean schemaChange(
+ String database, String table, Map<String, Object> params, String sql)
+ throws IOException, IllegalArgumentException {
+ if (checkSchemaChange(database, table, params)) {
return execute(sql, database);
}
return false;
@@ -104,26 +119,28 @@ public class SchemaChangeManager implements Serializable {
return params;
}
- /**
- * check ddl can do light schema change
- */
- public boolean checkSchemaChange(String database, String table, Map<String, Object> params) throws IOException, IllegalArgumentException {
- if(CollectionUtil.isNullOrEmpty(params)){
+ /** check ddl can do light schema change. */
+ public boolean checkSchemaChange(String database, String table, Map<String, Object> params)
+ throws IOException, IllegalArgumentException {
+ if (CollectionUtil.isNullOrEmpty(params)) {
return false;
}
- String requestUrl = String.format(CHECK_SCHEMA_CHANGE_API,
- RestService.randomEndpoint(dorisOptions.getFenodes(), LOG), database, table);
+ String requestUrl =
+ String.format(
+ CHECK_SCHEMA_CHANGE_API,
+ RestService.randomEndpoint(dorisOptions.getFenodes(), LOG),
+ database,
+ table);
HttpGetWithEntity httpGet = new HttpGetWithEntity(requestUrl);
httpGet.setHeader(HttpHeaders.AUTHORIZATION, authHeader());
httpGet.setEntity(new StringEntity(objectMapper.writeValueAsString(params)));
return handleResponse(httpGet);
}
- /**
- * execute sql in doris
- */
- public boolean execute(String ddl, String database) throws IOException, IllegalArgumentException {
- if(StringUtils.isNullOrWhitespaceOnly(ddl)){
+ /** execute sql in doris. */
+ public boolean execute(String ddl, String database)
+ throws IOException, IllegalArgumentException {
+ if (StringUtils.isNullOrWhitespaceOnly(ddl)) {
return false;
}
LOG.info("Execute SQL: {}", ddl);
@@ -131,11 +148,15 @@ public class SchemaChangeManager implements Serializable {
return handleResponse(httpPost);
}
- public HttpPost buildHttpPost(String ddl, String database) throws IllegalArgumentException, IOException {
+ public HttpPost buildHttpPost(String ddl, String database)
+ throws IllegalArgumentException, IOException {
Map<String, String> param = new HashMap<>();
param.put("stmt", ddl);
- String requestUrl = String.format(SCHEMA_CHANGE_API,
- RestService.randomEndpoint(dorisOptions.getFenodes(), LOG), database);
+ String requestUrl =
+ String.format(
+ SCHEMA_CHANGE_API,
+ RestService.randomEndpoint(dorisOptions.getFenodes(), LOG),
+ database);
HttpPost httpPost = new HttpPost(requestUrl);
httpPost.setHeader(HttpHeaders.AUTHORIZATION, authHeader());
httpPost.setHeader(HttpHeaders.CONTENT_TYPE, "application/json");
@@ -155,21 +176,26 @@ public class SchemaChangeManager implements Serializable {
if (code.equals("0")) {
return true;
} else {
- throw new DorisSchemaChangeException("Failed to schemaChange, response: " + loadResult);
+ throw new DorisSchemaChangeException(
+ "Failed to schemaChange, response: " + loadResult);
}
- } else{
- throw new DorisSchemaChangeException("Failed to schemaChange, status: " + statusCode + ", reason: " + reasonPhrase);
+ } else {
+ throw new DorisSchemaChangeException(
+ "Failed to schemaChange, status: "
+ + statusCode
+ + ", reason: "
+ + reasonPhrase);
}
} catch (Exception e) {
LOG.error("SchemaChange request error,", e);
- throw new DorisSchemaChangeException("SchemaChange request error with " + e.getMessage());
+ throw new DorisSchemaChangeException(
+ "SchemaChange request error with " + e.getMessage());
}
}
- /**
- * When processing a column, determine whether it exists and be idempotent.
- */
- public boolean checkColumnExists(String database, String table, String columnName) throws IllegalArgumentException, IOException {
+ /** When processing a column, determine whether it exists and be idempotent. */
+ public boolean checkColumnExists(String database, String table, String columnName)
+ throws IllegalArgumentException, IOException {
String existsQuery = SchemaChangeHelper.buildColumnExistsQuery(database, table, columnName);
HttpPost httpPost = buildHttpPost(existsQuery, database);
try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
@@ -181,7 +207,7 @@ public class SchemaChangeManager implements Serializable {
String code = responseNode.get("code").asText("-1");
if (code.equals("0")) {
JsonNode data = responseNode.get("data").get("data");
- if(!data.isEmpty()){
+ if (!data.isEmpty()) {
return true;
}
}
@@ -193,11 +219,14 @@ public class SchemaChangeManager implements Serializable {
}
private String authHeader() {
- return "Basic " + new String(Base64.encodeBase64(
- (dorisOptions.getUsername() + ":" + dorisOptions.getPassword()).getBytes(StandardCharsets.UTF_8)));
+ return "Basic "
+ + new String(
+ Base64.encodeBase64(
+ (dorisOptions.getUsername() + ":" + dorisOptions.getPassword())
+ .getBytes(StandardCharsets.UTF_8)));
}
- private String getTableIdentifier(String database, String table){
+ private String getTableIdentifier(String database, String table) {
return String.format("%s.%s", database, table);
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java
index c25928e..f60c886 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java
@@ -19,6 +19,7 @@ package org.apache.doris.flink.sink.writer;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.util.Preconditions;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,10 +29,8 @@ import java.util.concurrent.BlockingDeque;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
-/**
- * Channel of record stream and HTTP data stream.
- */
-public class CacheRecordBuffer extends RecordBuffer{
+/** Channel of record stream and HTTP data stream. */
+public class CacheRecordBuffer extends RecordBuffer {
private static final Logger LOG = LoggerFactory.getLogger(CacheRecordBuffer.class);
BlockingDeque<ByteBuffer> bufferCache;
LinkedBlockingQueue<ByteBuffer> bufferPool;
@@ -43,9 +42,13 @@ public class CacheRecordBuffer extends RecordBuffer{
}
@Override
- public void startBufferData() throws IOException{
- LOG.info("start buffer data, read queue size {}, write queue size {}, buffer cache size {}, buffer pool size {}",
- readQueue.size(), writeQueue.size(), bufferCache.size(), bufferPool.size());
+ public void startBufferData() throws IOException {
+ LOG.info(
+ "start buffer data, read queue size {}, write queue size {}, buffer cache size {}, buffer pool size {}",
+ readQueue.size(),
+ writeQueue.size(),
+ bufferCache.size(),
+ bufferPool.size());
try {
// if the cache have data, that should be restarted from previous error
if (currentReadBuffer != null && currentReadBuffer.limit() != 0) {
@@ -64,7 +67,7 @@ public class CacheRecordBuffer extends RecordBuffer{
throw new IOException(e);
}
}
-
+
@Override
public int read(byte[] buf) throws InterruptedException {
if (currentReadBuffer == null) {
@@ -101,7 +104,7 @@ public class CacheRecordBuffer extends RecordBuffer{
}
}
- private ByteBuffer allocate(){
+ private ByteBuffer allocate() {
ByteBuffer buff = bufferPool.poll();
return buff != null ? buff : ByteBuffer.allocate(bufferCapacity);
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java
index fb904ef..30ff365 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java
@@ -17,6 +17,10 @@
package org.apache.doris.flink.sink.writer;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.doris.flink.cfg.DorisExecutionOptions;
@@ -28,9 +32,6 @@ import org.apache.doris.flink.rest.models.RespContent;
import org.apache.doris.flink.sink.EscapeHandler;
import org.apache.doris.flink.sink.HttpPutBuilder;
import org.apache.doris.flink.sink.ResponseUtil;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.concurrent.ExecutorThreadFactory;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -56,9 +57,7 @@ import static org.apache.doris.flink.sink.ResponseUtil.LABEL_EXIST_PATTERN;
import static org.apache.doris.flink.sink.writer.LoadConstants.LINE_DELIMITER_DEFAULT;
import static org.apache.doris.flink.sink.writer.LoadConstants.LINE_DELIMITER_KEY;
-/**
- * load data to doris.
- **/
+/** load data to doris. */
public class DorisStreamLoad implements Serializable {
private static final Logger LOG = LoggerFactory.getLogger(DorisStreamLoad.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@@ -84,11 +83,12 @@ public class DorisStreamLoad implements Serializable {
private final ExecutorService executorService;
private boolean loadBatchFirstRecord;
- public DorisStreamLoad(String hostPort,
- DorisOptions dorisOptions,
- DorisExecutionOptions executionOptions,
- LabelGenerator labelGenerator,
- CloseableHttpClient httpClient) {
+ public DorisStreamLoad(
+ String hostPort,
+ DorisOptions dorisOptions,
+ DorisExecutionOptions executionOptions,
+ LabelGenerator labelGenerator,
+ CloseableHttpClient httpClient) {
this.hostPort = hostPort;
String[] tableInfo = dorisOptions.getTableIdentifier().split("\\.");
this.db = tableInfo[0];
@@ -102,11 +102,24 @@ public class DorisStreamLoad implements Serializable {
this.streamLoadProp = executionOptions.getStreamLoadProp();
this.enableDelete = executionOptions.getDeletable();
this.httpClient = httpClient;
- this.executorService = new ThreadPoolExecutor(1, 1,
- 0L, TimeUnit.MILLISECONDS,
- new LinkedBlockingQueue<>(), new ExecutorThreadFactory("stream-load-upload"));
- this.recordStream = new RecordStream(executionOptions.getBufferSize(), executionOptions.getBufferCount(), executionOptions.isUseCache());
- lineDelimiter = EscapeHandler.escapeString(streamLoadProp.getProperty(LINE_DELIMITER_KEY, LINE_DELIMITER_DEFAULT)).getBytes();
+ this.executorService =
+ new ThreadPoolExecutor(
+ 1,
+ 1,
+ 0L,
+ TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<>(),
+ new ExecutorThreadFactory("stream-load-upload"));
+ this.recordStream =
+ new RecordStream(
+ executionOptions.getBufferSize(),
+ executionOptions.getBufferCount(),
+ executionOptions.isUseCache());
+ lineDelimiter =
+ EscapeHandler.escapeString(
+ streamLoadProp.getProperty(
+ LINE_DELIMITER_KEY, LINE_DELIMITER_DEFAULT))
+ .getBytes();
loadBatchFirstRecord = true;
}
@@ -134,6 +147,7 @@ public class DorisStreamLoad implements Serializable {
/**
* try to discard pending transactions with labels beginning with labelSuffix.
+ *
* @param labelSuffix
* @param chkID
* @throws Exception
@@ -143,7 +157,8 @@ public class DorisStreamLoad implements Serializable {
LOG.info("abort for labelSuffix {}. start chkId {}.", labelSuffix, chkID);
while (true) {
try {
- // TODO: According to label abort txn. Currently, it can only be aborted based on txnid,
+ // TODO: According to label abort txn. Currently,
+ // it can only be aborted based on txnid,
// so we must first request a streamload based on the label to get the txnid.
String label = labelGenerator.generateTableLabel(startChkID);
HttpPutBuilder builder = new HttpPutBuilder();
@@ -154,17 +169,20 @@ public class DorisStreamLoad implements Serializable {
.setLabel(label)
.setEmptyEntity()
.addProperties(streamLoadProp);
- RespContent respContent = handlePreCommitResponse(httpClient.execute(builder.build()));
+ RespContent respContent =
+ handlePreCommitResponse(httpClient.execute(builder.build()));
Preconditions.checkState("true".equals(respContent.getTwoPhaseCommit()));
if (LABEL_ALREADY_EXIST.equals(respContent.getStatus())) {
// label already exist and job finished
if (JOB_EXIST_FINISHED.equals(respContent.getExistingJobStatus())) {
- throw new DorisException("Load status is " + LABEL_ALREADY_EXIST + " and load job finished, " +
- "change you label prefix or restore from latest savepoint!");
-
+ throw new DorisException(
+ "Load status is "
+ + LABEL_ALREADY_EXIST
+ + " and load job finished, "
+ + "change you label prefix or restore from latest savepoint!");
}
// job not finished, abort.
- Matcher matcher = LABEL_EXIST_PATTERN.matcher(respContent.getMessage());
+ Matcher matcher = LABEL_EXIST_PATTERN.matcher(respContent.getMessage());
if (matcher.find()) {
Preconditions.checkState(label.equals(matcher.group(1)));
long txnId = Long.parseLong(matcher.group(2));
@@ -172,7 +190,10 @@ public class DorisStreamLoad implements Serializable {
abortTransaction(txnId);
} else {
LOG.error("response: {}", respContent.toString());
- throw new DorisException("Load Status is " + LABEL_ALREADY_EXIST + ", but no txnID associated with it!");
+ throw new DorisException(
+ "Load Status is "
+ + LABEL_ALREADY_EXIST
+ + ", but no txnID associated with it!");
}
} else {
LOG.info("abort {} for check label {}.", respContent.getTxnId(), label);
@@ -190,10 +211,11 @@ public class DorisStreamLoad implements Serializable {
/**
* write record into stream.
+ *
* @param record
* @throws IOException
*/
- public void writeRecord(byte[] record) throws IOException{
+ public void writeRecord(byte[] record) throws IOException {
if (loadBatchFirstRecord) {
loadBatchFirstRecord = false;
} else {
@@ -207,7 +229,7 @@ public class DorisStreamLoad implements Serializable {
return recordStream;
}
- public RespContent handlePreCommitResponse(CloseableHttpResponse response) throws Exception{
+ public RespContent handlePreCommitResponse(CloseableHttpResponse response) throws Exception {
final int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200 && response.getEntity() != null) {
String loadResult = EntityUtils.toString(response.getEntity());
@@ -217,12 +239,12 @@ public class DorisStreamLoad implements Serializable {
throw new StreamLoadException("stream load error: " + response.getStatusLine().toString());
}
- public RespContent stopLoad(String label) throws IOException{
+ public RespContent stopLoad(String label) throws IOException {
recordStream.endInput();
LOG.info("table {} stream load stopped for {} on host {}", table, label, hostPort);
Preconditions.checkState(pendingLoadFuture != null);
try {
- return handlePreCommitResponse(pendingLoadFuture.get());
+ return handlePreCommitResponse(pendingLoadFuture.get());
} catch (Exception e) {
throw new DorisRuntimeException(e);
}
@@ -230,6 +252,7 @@ public class DorisStreamLoad implements Serializable {
/**
* start write data for new checkpoint.
+ *
* @param label
* @throws IOException
*/
@@ -240,7 +263,8 @@ public class DorisStreamLoad implements Serializable {
LOG.info("table {} stream load started for {} on host {}", table, label, hostPort);
try {
InputStreamEntity entity = new InputStreamEntity(recordStream);
- putBuilder.setUrl(loadUrlStr)
+ putBuilder
+ .setUrl(loadUrlStr)
.baseAuth(user, passwd)
.addCommonHeader()
.addHiddenColumns(enableDelete)
@@ -248,12 +272,14 @@ public class DorisStreamLoad implements Serializable {
.setEntity(entity)
.addProperties(streamLoadProp);
if (enable2PC) {
- putBuilder.enable2PC();
+ putBuilder.enable2PC();
}
- pendingLoadFuture = executorService.submit(() -> {
- LOG.info("table {} start execute load", table);
- return httpClient.execute(putBuilder.build());
- });
+ pendingLoadFuture =
+ executorService.submit(
+ () -> {
+ LOG.info("table {} start execute load", table);
+ return httpClient.execute(putBuilder.build());
+ });
} catch (Exception e) {
String err = "failed to stream load data with label: " + label;
LOG.warn(err, e);
@@ -274,16 +300,18 @@ public class DorisStreamLoad implements Serializable {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != 200 || response.getEntity() == null) {
LOG.warn("abort transaction response: " + response.getStatusLine().toString());
- throw new DorisRuntimeException("Fail to abort transaction " + txnID + " with url " + abortUrlStr);
+ throw new DorisRuntimeException(
+ "Fail to abort transaction " + txnID + " with url " + abortUrlStr);
}
ObjectMapper mapper = new ObjectMapper();
String loadResult = EntityUtils.toString(response.getEntity());
- Map<String, String> res = mapper.readValue(loadResult, new TypeReference<HashMap<String, String>>(){});
+ Map<String, String> res =
+ mapper.readValue(loadResult, new TypeReference<HashMap<String, String>>() {});
if (!SUCCESS.equals(res.get("status"))) {
if (ResponseUtil.isCommitted(res.get("msg"))) {
- throw new DorisException("try abort committed transaction, " +
- "do you recover from old savepoint?");
+ throw new DorisException(
+ "try abort committed transaction, " + "do you recover from old savepoint?");
}
LOG.warn("Fail to abort transaction. txnId: {}, error: {}", txnID, res.get("msg"));
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java
index 0033ad0..3fec941 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java
@@ -17,6 +17,13 @@
package org.apache.doris.flink.sink.writer;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.sink2.StatefulSink;
+import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
+import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
@@ -29,12 +36,6 @@ import org.apache.doris.flink.sink.DorisCommittable;
import org.apache.doris.flink.sink.HttpUtil;
import org.apache.doris.flink.sink.writer.serializer.DorisRecord;
import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.connector.sink2.Sink;
-import org.apache.flink.api.connector.sink2.StatefulSink;
-import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
-import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
-import org.apache.flink.util.concurrent.ExecutorThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,16 +56,19 @@ import static org.apache.doris.flink.sink.LoadStatus.SUCCESS;
/**
* Doris Writer will load data to doris.
+ *
* @param <IN>
*/
-public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, DorisWriterState>,
- TwoPhaseCommittingSink.PrecommittingSinkWriter<IN, DorisCommittable> {
+public class DorisWriter<IN>
+ implements StatefulSink.StatefulSinkWriter<IN, DorisWriterState>,
+ TwoPhaseCommittingSink.PrecommittingSinkWriter<IN, DorisCommittable> {
private static final Logger LOG = LoggerFactory.getLogger(DorisWriter.class);
- private static final List<String> DORIS_SUCCESS_STATUS = new ArrayList<>(Arrays.asList(SUCCESS, PUBLISH_TIMEOUT));
+ private static final List<String> DORIS_SUCCESS_STATUS =
+ new ArrayList<>(Arrays.asList(SUCCESS, PUBLISH_TIMEOUT));
private final long lastCheckpointId;
private long curCheckpointId;
private Map<String, DorisStreamLoad> dorisStreamLoadMap = new ConcurrentHashMap<>();
- private Map<String, LabelGenerator> labelGeneratorMap = new ConcurrentHashMap<>();;
+ private Map<String, LabelGenerator> labelGeneratorMap = new ConcurrentHashMap<>();
volatile boolean globalLoading;
private Map<String, Boolean> loadingMap = new ConcurrentHashMap<>();
private final DorisOptions dorisOptions;
@@ -79,12 +83,13 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
private transient volatile Exception loadException = null;
private BackendUtil backendUtil;
- public DorisWriter(Sink.InitContext initContext,
- Collection<DorisWriterState> state,
- DorisRecordSerializer<IN> serializer,
- DorisOptions dorisOptions,
- DorisReadOptions dorisReadOptions,
- DorisExecutionOptions executionOptions) {
+ public DorisWriter(
+ Sink.InitContext initContext,
+ Collection<DorisWriterState> state,
+ DorisRecordSerializer<IN> serializer,
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions executionOptions) {
this.lastCheckpointId =
initContext
.getRestoredCheckpointId()
@@ -94,7 +99,8 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
LOG.info("labelPrefix " + executionOptions.getLabelPrefix());
this.labelPrefix = executionOptions.getLabelPrefix();
this.subtaskId = initContext.getSubtaskId();
- this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, new ExecutorThreadFactory("stream-load-check"));
+ this.scheduledExecutorService =
+ new ScheduledThreadPoolExecutor(1, new ExecutorThreadFactory("stream-load-check"));
this.serializer = serializer;
this.dorisOptions = dorisOptions;
this.dorisReadOptions = dorisReadOptions;
@@ -108,7 +114,7 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
public void initializeLoad(Collection<DorisWriterState> state) {
this.backendUtil = BackendUtil.getInstance(dorisOptions, dorisReadOptions, LOG);
try {
- if(executionOptions.enabled2PC()) {
+ if (executionOptions.enabled2PC()) {
abortLingeringTransactions(state);
}
} catch (Exception e) {
@@ -117,35 +123,42 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
}
// get main work thread.
executorThread = Thread.currentThread();
- // when uploading data in streaming mode, we need to regularly detect whether there are exceptions.
- scheduledExecutorService.scheduleWithFixedDelay(this::checkDone, 200, intervalTime, TimeUnit.MILLISECONDS);
+ // when uploading data in streaming mode,
+ // we need to regularly detect whether there are exceptions.
+ scheduledExecutorService.scheduleWithFixedDelay(
+ this::checkDone, 200, intervalTime, TimeUnit.MILLISECONDS);
}
- private void abortLingeringTransactions(Collection<DorisWriterState> recoveredStates) throws Exception {
+ private void abortLingeringTransactions(Collection<DorisWriterState> recoveredStates)
+ throws Exception {
List<String> alreadyAborts = new ArrayList<>();
- //abort label in state
- for(DorisWriterState state : recoveredStates){
- // Todo: When the sink parallelism is reduced,
- // the txn of the redundant task before aborting is also needed.
- if(!state.getLabelPrefix().equals(labelPrefix)){
- LOG.warn("Label prefix from previous execution {} has changed to {}.", state.getLabelPrefix(), executionOptions.getLabelPrefix());
- }
- if (state.getDatabase() == null || state.getTable() == null) {
- LOG.warn("Transactions cannot be aborted when restore because the last used flink-doris-connector version less than 1.5.0.");
- continue;
- }
- String key = state.getDatabase() + "." + state.getTable();
- DorisStreamLoad streamLoader = getStreamLoader(key);
- streamLoader.abortPreCommit(state.getLabelPrefix(), curCheckpointId);
- alreadyAborts.add(state.getLabelPrefix());
+ // abort label in state
+ for (DorisWriterState state : recoveredStates) {
+ // Todo: When the sink parallelism is reduced,
+ // the txn of the redundant task before aborting is also needed.
+ if (!state.getLabelPrefix().equals(labelPrefix)) {
+ LOG.warn(
+ "Label prefix from previous execution {} has changed to {}.",
+ state.getLabelPrefix(),
+ executionOptions.getLabelPrefix());
+ }
+ if (state.getDatabase() == null || state.getTable() == null) {
+ LOG.warn(
+ "Transactions cannot be aborted when restore because the last used flink-doris-connector version less than 1.5.0.");
+ continue;
+ }
+ String key = state.getDatabase() + "." + state.getTable();
+ DorisStreamLoad streamLoader = getStreamLoader(key);
+ streamLoader.abortPreCommit(state.getLabelPrefix(), curCheckpointId);
+ alreadyAborts.add(state.getLabelPrefix());
}
// TODO: In a multi-table scenario, if do not restore from checkpoint,
// when modify labelPrefix at startup, we cannot abort the previous label.
- if(!alreadyAborts.contains(labelPrefix)
+ if (!alreadyAborts.contains(labelPrefix)
&& StringUtils.isNotEmpty(dorisOptions.getTableIdentifier())
- && StringUtils.isNotEmpty(labelPrefix)){
- //abort current labelPrefix
+ && StringUtils.isNotEmpty(labelPrefix)) {
+ // abort current labelPrefix
DorisStreamLoad streamLoader = getStreamLoader(dorisOptions.getTableIdentifier());
streamLoader.abortPreCommit(labelPrefix, curCheckpointId);
}
@@ -157,17 +170,17 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
String tableKey = dorisOptions.getTableIdentifier();
DorisRecord record = serializer.serialize(in);
- if(record == null || record.getRow() == null){
- //ddl or value is null
+ if (record == null || record.getRow() == null) {
+ // ddl or value is null
return;
}
- //multi table load
- if(record.getTableIdentifier() != null){
+ // multi table load
+ if (record.getTableIdentifier() != null) {
tableKey = record.getTableIdentifier();
}
DorisStreamLoad streamLoader = getStreamLoader(tableKey);
- if(!loadingMap.containsKey(tableKey)) {
+ if (!loadingMap.containsKey(tableKey)) {
// start stream load only when there has data
LabelGenerator labelGenerator = getLabelGenerator(tableKey);
String currentLabel = labelGenerator.generateTableLabel(curCheckpointId);
@@ -180,13 +193,13 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
@Override
public void flush(boolean flush) throws IOException, InterruptedException {
- //No action is triggered, everything is in the precommit method
+ // No action is triggered, everything is in the precommit method
}
@Override
public Collection<DorisCommittable> prepareCommit() throws IOException, InterruptedException {
// Verify whether data is written during a checkpoint
- if(!globalLoading && loadingMap.values().stream().noneMatch(Boolean::booleanValue)){
+ if (!globalLoading && loadingMap.values().stream().noneMatch(Boolean::booleanValue)) {
return Collections.emptyList();
}
// disable exception checker before stop load.
@@ -194,9 +207,9 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
// submit stream load http request
List<DorisCommittable> committableList = new ArrayList<>();
- for(Map.Entry<String, DorisStreamLoad> streamLoader : dorisStreamLoadMap.entrySet()){
+ for (Map.Entry<String, DorisStreamLoad> streamLoader : dorisStreamLoadMap.entrySet()) {
String tableIdentifier = streamLoader.getKey();
- if(!loadingMap.getOrDefault(tableIdentifier, false)){
+ if (!loadingMap.getOrDefault(tableIdentifier, false)) {
LOG.debug("skip table {}, no data need to load.", tableIdentifier);
continue;
}
@@ -205,12 +218,19 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
String currentLabel = labelGenerator.generateTableLabel(curCheckpointId);
RespContent respContent = dorisStreamLoad.stopLoad(currentLabel);
if (!DORIS_SUCCESS_STATUS.contains(respContent.getStatus())) {
- String errMsg = String.format("tabel {} stream load error: %s, see more in %s", tableIdentifier, respContent.getMessage(), respContent.getErrorURL());
+ String errMsg =
+ String.format(
+ "tabel {} stream load error: %s, see more in %s",
+ tableIdentifier,
+ respContent.getMessage(),
+ respContent.getErrorURL());
throw new DorisRuntimeException(errMsg);
}
- if(executionOptions.enabled2PC()){
+ if (executionOptions.enabled2PC()) {
long txnId = respContent.getTxnId();
- committableList.add(new DorisCommittable(dorisStreamLoad.getHostPort(), dorisStreamLoad.getDb(), txnId));
+ committableList.add(
+ new DorisCommittable(
+ dorisStreamLoad.getHostPort(), dorisStreamLoad.getDb(), txnId));
}
}
// clean loadingMap
@@ -221,40 +241,51 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
@Override
public List<DorisWriterState> snapshotState(long checkpointId) throws IOException {
List<DorisWriterState> writerStates = new ArrayList<>();
- for(DorisStreamLoad dorisStreamLoad : dorisStreamLoadMap.values()){
- //Dynamic refresh backend
+ for (DorisStreamLoad dorisStreamLoad : dorisStreamLoadMap.values()) {
+ // Dynamic refresh backend
dorisStreamLoad.setHostPort(backendUtil.getAvailableBackend());
- DorisWriterState writerState = new DorisWriterState(labelPrefix, dorisStreamLoad.getDb(), dorisStreamLoad.getTable(), subtaskId);
+ DorisWriterState writerState =
+ new DorisWriterState(
+ labelPrefix,
+ dorisStreamLoad.getDb(),
+ dorisStreamLoad.getTable(),
+ subtaskId);
writerStates.add(writerState);
}
this.curCheckpointId = checkpointId + 1;
return writerStates;
}
- private LabelGenerator getLabelGenerator(String tableKey){
- return labelGeneratorMap.computeIfAbsent(tableKey, v-> new LabelGenerator(labelPrefix, executionOptions.enabled2PC(), tableKey, subtaskId));
+ private LabelGenerator getLabelGenerator(String tableKey) {
+ return labelGeneratorMap.computeIfAbsent(
+ tableKey,
+ v ->
+ new LabelGenerator(
+ labelPrefix, executionOptions.enabled2PC(), tableKey, subtaskId));
}
- private DorisStreamLoad getStreamLoader(String tableKey){
+ private DorisStreamLoad getStreamLoader(String tableKey) {
LabelGenerator labelGenerator = getLabelGenerator(tableKey);
dorisOptions.setTableIdentifier(tableKey);
- return dorisStreamLoadMap.computeIfAbsent(tableKey, v -> new DorisStreamLoad(backendUtil.getAvailableBackend(),
- dorisOptions,
- executionOptions,
- labelGenerator,
- new HttpUtil().getHttpClient()));
+ return dorisStreamLoadMap.computeIfAbsent(
+ tableKey,
+ v ->
+ new DorisStreamLoad(
+ backendUtil.getAvailableBackend(),
+ dorisOptions,
+ executionOptions,
+ labelGenerator,
+ new HttpUtil().getHttpClient()));
}
- /**
- * Check the streamload http request regularly
- */
+ /** Check the streamload http request regularly. */
private void checkDone() {
- for(Map.Entry<String, DorisStreamLoad> streamLoadMap : dorisStreamLoadMap.entrySet()){
+ for (Map.Entry<String, DorisStreamLoad> streamLoadMap : dorisStreamLoadMap.entrySet()) {
checkAllDone(streamLoadMap.getKey(), streamLoadMap.getValue());
}
}
- private void checkAllDone(String tableIdentifier, DorisStreamLoad dorisStreamLoad){
+ private void checkAllDone(String tableIdentifier, DorisStreamLoad dorisStreamLoad) {
// the load future is done and checked in prepareCommit().
// this will check error while loading.
LOG.debug("start timer checker, interval {} ms", intervalTime);
@@ -278,23 +309,32 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
dorisStreamLoad.abortPreCommit(labelPrefix, curCheckpointId);
}
// start a new txn(stream load)
- LOG.info("getting exception, breakpoint resume for checkpoint ID: {}, table {}", curCheckpointId, tableIdentifier);
+ LOG.info(
+ "getting exception, breakpoint resume for checkpoint ID: {}, table {}",
+ curCheckpointId,
+ tableIdentifier);
LabelGenerator labelGenerator = getLabelGenerator(tableIdentifier);
- dorisStreamLoad.startLoad(labelGenerator.generateTableLabel(curCheckpointId), true);
+ dorisStreamLoad.startLoad(
+ labelGenerator.generateTableLabel(curCheckpointId), true);
} catch (Exception e) {
throw new DorisRuntimeException(e);
}
} else {
String errorMsg;
try {
- RespContent content = dorisStreamLoad.handlePreCommitResponse(dorisStreamLoad.getPendingLoadFuture().get());
+ RespContent content =
+ dorisStreamLoad.handlePreCommitResponse(
+ dorisStreamLoad.getPendingLoadFuture().get());
errorMsg = content.getMessage();
} catch (Exception e) {
errorMsg = e.getMessage();
}
loadException = new StreamLoadException(errorMsg);
- LOG.error("table {} stream load finished unexpectedly, interrupt worker thread! {}", tableIdentifier, errorMsg);
+ LOG.error(
+ "table {} stream load finished unexpectedly, interrupt worker thread! {}",
+ tableIdentifier,
+ errorMsg);
// set the executor thread interrupted in case blocking in write data.
executorThread.interrupt();
}
@@ -325,10 +365,9 @@ public class DorisWriter<IN> implements StatefulSink.StatefulSinkWriter<IN, Dori
scheduledExecutorService.shutdownNow();
}
if (dorisStreamLoadMap != null && !dorisStreamLoadMap.isEmpty()) {
- for(DorisStreamLoad dorisStreamLoad : dorisStreamLoadMap.values()){
+ for (DorisStreamLoad dorisStreamLoad : dorisStreamLoadMap.values()) {
dorisStreamLoad.close();
}
}
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java
index a08e58d..112d752 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java
@@ -19,14 +19,13 @@ package org.apache.doris.flink.sink.writer;
import java.util.Objects;
-/**
- * hold state for DorisWriter.
- */
+/** hold state for DorisWriter. */
public class DorisWriterState {
String labelPrefix;
String database;
String table;
int subtaskId;
+
public DorisWriterState(String labelPrefix) {
this.labelPrefix = labelPrefix;
}
@@ -76,11 +75,18 @@ public class DorisWriterState {
@Override
public String toString() {
- return "DorisWriterState{" +
- "labelPrefix='" + labelPrefix + '\'' +
- ", database='" + database + '\'' +
- ", table='" + table + '\'' +
- ", subtaskId=" + subtaskId +
- '}';
+ return "DorisWriterState{"
+ + "labelPrefix='"
+ + labelPrefix
+ + '\''
+ + ", database='"
+ + database
+ + '\''
+ + ", table='"
+ + table
+ + '\''
+ + ", subtaskId="
+ + subtaskId
+ + '}';
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java
index 97f5cc1..ad055d1 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java
@@ -25,12 +25,11 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-/**
- * Serializer for DorisWriterState.
- */
+/** Serializer for DorisWriterState. */
public class DorisWriterStateSerializer implements SimpleVersionedSerializer<DorisWriterState> {
private static final int VERSION = 2;
+
@Override
public int getVersion() {
return VERSION;
@@ -39,7 +38,7 @@ public class DorisWriterStateSerializer implements SimpleVersionedSerializer<Dor
@Override
public byte[] serialize(DorisWriterState dorisWriterState) throws IOException {
try (final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- final DataOutputStream out = new DataOutputStream(baos)) {
+ final DataOutputStream out = new DataOutputStream(baos)) {
out.writeUTF(dorisWriterState.getLabelPrefix());
out.writeUTF(dorisWriterState.getDatabase());
out.writeUTF(dorisWriterState.getTable());
@@ -52,16 +51,16 @@ public class DorisWriterStateSerializer implements SimpleVersionedSerializer<Dor
@Override
public DorisWriterState deserialize(int version, byte[] serialized) throws IOException {
try (final ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
- final DataInputStream in = new DataInputStream(bais)) {
- String labelPrefix = in.readUTF();
- if(version == 1){
- return new DorisWriterState(labelPrefix);
- }else {
- final String database = in.readUTF();
- final String table = in.readUTF();
- final int subtaskId = in.readInt();
- return new DorisWriterState(labelPrefix, database, table, subtaskId);
- }
+ final DataInputStream in = new DataInputStream(bais)) {
+ String labelPrefix = in.readUTF();
+ if (version == 1) {
+ return new DorisWriterState(labelPrefix);
+ } else {
+ final String database = in.readUTF();
+ final String table = in.readUTF();
+ final int subtaskId = in.readInt();
+ return new DorisWriterState(labelPrefix, database, table, subtaskId);
+ }
}
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java
index 60f5595..312dd8f 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java
@@ -14,15 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.sink.writer;
import org.apache.flink.util.Preconditions;
import java.util.UUID;
-/**
- * Generator label for stream load.
- */
+/** Generator label for stream load. */
public class LabelGenerator {
private String labelPrefix;
private boolean enable2PC;
@@ -34,7 +33,8 @@ public class LabelGenerator {
this.enable2PC = enable2PC;
}
- public LabelGenerator(String labelPrefix, boolean enable2PC, String tableIdentifier, int subtaskId) {
+ public LabelGenerator(
+ String labelPrefix, boolean enable2PC, String tableIdentifier, int subtaskId) {
this(labelPrefix, enable2PC);
// The label of stream load can not contain `.`
this.tableIdentifier = tableIdentifier.replace(".", "_");
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java
index d38f960..7b0d1d0 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java
@@ -17,9 +17,7 @@
package org.apache.doris.flink.sink.writer;
-/**
- * Constants for load.
- */
+/** Constants for load. */
public class LoadConstants {
public static final String COLUMNS_KEY = "columns";
public static final String FIELD_DELIMITER_KEY = "column_separator";
@@ -32,5 +30,4 @@ public class LoadConstants {
public static final String NULL_VALUE = "\\N";
public static final String DORIS_DELETE_SIGN = "__DORIS_DELETE_SIGN__";
public static final String READ_JSON_BY_LINE = "read_json_by_line";
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java
index e5259cb..c84e103 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java
@@ -18,6 +18,7 @@
package org.apache.doris.flink.sink.writer;
import org.apache.flink.util.Preconditions;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -27,9 +28,7 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;
-/**
- * Channel of record stream and HTTP data stream.
- */
+/** Channel of record stream and HTTP data stream. */
public class RecordBuffer {
private static final Logger LOG = LoggerFactory.getLogger(RecordBuffer.class);
BlockingQueue<ByteBuffer> writeQueue;
@@ -52,17 +51,20 @@ public class RecordBuffer {
this.queueSize = queueSize;
}
- public void startBufferData() throws IOException{
- LOG.info("start buffer data, read queue size {}, write queue size {}", readQueue.size(), writeQueue.size());
+ public void startBufferData() throws IOException {
+ LOG.info(
+ "start buffer data, read queue size {}, write queue size {}",
+ readQueue.size(),
+ writeQueue.size());
Preconditions.checkState(readQueue.size() == 0);
Preconditions.checkState(writeQueue.size() == queueSize);
- for (ByteBuffer byteBuffer: writeQueue) {
+ for (ByteBuffer byteBuffer : writeQueue) {
Preconditions.checkState(byteBuffer.position() == 0);
Preconditions.checkState(byteBuffer.remaining() == bufferCapacity);
}
}
- public void stopBufferData() throws IOException{
+ public void stopBufferData() throws IOException {
try {
// add Empty buffer as finish flag.
boolean isEmpty = false;
@@ -84,7 +86,7 @@ public class RecordBuffer {
}
}
- public void write(byte[] buf) throws InterruptedException{
+ public void write(byte[] buf) throws InterruptedException {
int wPos = 0;
do {
if (currentWriteBuffer == null) {
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java
index baf68bf..693967a 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java
@@ -20,9 +20,7 @@ package org.apache.doris.flink.sink.writer;
import java.io.IOException;
import java.io.InputStream;
-/**
- * Record Stream for writing record.
- */
+/** Record Stream for writing record. */
public class RecordStream extends InputStream {
private final RecordBuffer recordBuffer;
@@ -34,7 +32,7 @@ public class RecordStream extends InputStream {
public RecordStream(int bufferSize, int bufferCount, boolean useCache) {
if (useCache) {
this.recordBuffer = new CacheRecordBuffer(bufferSize, bufferCount);
- }else {
+ } else {
this.recordBuffer = new RecordBuffer(bufferSize, bufferCount);
}
}
@@ -42,12 +40,12 @@ public class RecordStream extends InputStream {
public void startInput(boolean isResume) throws IOException {
// if resume from breakpoint, do not recycle cache buffer
if (!isResume && recordBuffer instanceof CacheRecordBuffer) {
- ((CacheRecordBuffer)recordBuffer).recycleCache();
+ ((CacheRecordBuffer) recordBuffer).recycleCache();
}
recordBuffer.startBufferData();
}
- public void endInput() throws IOException{
+ public void endInput() throws IOException {
recordBuffer.stopBufferData();
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java
index fc2f00c..d15f07c 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.sink.writer.serializer;
import java.io.Serializable;
@@ -23,8 +24,7 @@ public class DorisRecord implements Serializable {
private String table;
private byte[] row;
- public DorisRecord() {
- }
+ public DorisRecord() {}
public DorisRecord(String database, String table, byte[] row) {
this.database = database;
@@ -32,8 +32,8 @@ public class DorisRecord implements Serializable {
this.row = row;
}
- public String getTableIdentifier(){
- if(database == null || table == null){
+ public String getTableIdentifier() {
+ if (database == null || table == null) {
return null;
}
return database + "." + table;
@@ -68,9 +68,9 @@ public class DorisRecord implements Serializable {
}
public static DorisRecord of(String tableIdentifier, byte[] row) {
- if(tableIdentifier != null) {
+ if (tableIdentifier != null) {
String[] dbTbl = tableIdentifier.split("\\.");
- if(dbTbl.length == 2){
+ if (dbTbl.length == 2) {
String database = dbTbl[0];
String table = dbTbl[1];
return new DorisRecord(database, table, row);
@@ -82,5 +82,4 @@ public class DorisRecord implements Serializable {
public static DorisRecord of(byte[] row) {
return new DorisRecord(null, null, row);
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java
index 8c699f6..c1135fa 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java
@@ -22,16 +22,17 @@ import java.io.Serializable;
/**
* How to serialize the record to bytes.
+ *
* @param <T>
*/
public interface DorisRecordSerializer<T> extends Serializable {
/**
* define how to convert record into byte array.
+ *
* @param record
* @return [tableIdentifer,byte array]
* @throws IOException
*/
DorisRecord serialize(T record) throws IOException;
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java
index 25d06ef..cdbd482 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java
@@ -17,6 +17,12 @@
package org.apache.doris.flink.sink.writer.serializer;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.CollectionUtil;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -40,11 +46,6 @@ import org.apache.doris.flink.tools.cdc.mysql.MysqlType;
import org.apache.doris.flink.tools.cdc.oracle.OracleType;
import org.apache.doris.flink.tools.cdc.postgres.PostgresType;
import org.apache.doris.flink.tools.cdc.sqlserver.SqlServerType;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.CollectionUtil;
-import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -71,11 +72,14 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
private static final String OP_CREATE = "c"; // insert
private static final String OP_UPDATE = "u"; // update
private static final String OP_DELETE = "d"; // delete
- public static final String EXECUTE_DDL = "ALTER TABLE %s %s COLUMN %s %s"; // alter table tbl add cloumn aca int
- private static final String addDropDDLRegex
- = "ALTER\\s+TABLE\\s+[^\\s]+\\s+(ADD|DROP)\\s+(COLUMN\\s+)?([^\\s]+)(\\s+([^\\s]+))?.*";
- private static final Pattern renameDDLPattern = Pattern.compile(
- "ALTER\\s+TABLE\\s+(\\w+)\\s+RENAME\\s+COLUMN\\s+(\\w+)\\s+TO\\s+(\\w+)", Pattern.CASE_INSENSITIVE);
+ public static final String EXECUTE_DDL =
+ "ALTER TABLE %s %s COLUMN %s %s"; // alter table tbl add cloumn aca int
+ private static final String addDropDDLRegex =
+ "ALTER\\s+TABLE\\s+[^\\s]+\\s+(ADD|DROP)\\s+(COLUMN\\s+)?([^\\s]+)(\\s+([^\\s]+))?.*";
+ private static final Pattern renameDDLPattern =
+ Pattern.compile(
+ "ALTER\\s+TABLE\\s+(\\w+)\\s+RENAME\\s+COLUMN\\s+(\\w+)\\s+TO\\s+(\\w+)",
+ Pattern.CASE_INSENSITIVE);
private final Pattern addDropDDLPattern;
private DorisOptions dorisOptions;
private ObjectMapper objectMapper = new ObjectMapper();
@@ -97,13 +101,17 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
private Map<String, String> tableProperties;
private String targetDatabase;
- public JsonDebeziumSchemaSerializer(DorisOptions dorisOptions,
+ public JsonDebeziumSchemaSerializer(
+ DorisOptions dorisOptions,
Pattern pattern,
String sourceTableName,
boolean newSchemaChange) {
this.dorisOptions = dorisOptions;
- this.addDropDDLPattern = pattern == null ? Pattern.compile(addDropDDLRegex, Pattern.CASE_INSENSITIVE) : pattern;
- if(!StringUtils.isNullOrWhitespaceOnly(dorisOptions.getTableIdentifier())){
+ this.addDropDDLPattern =
+ pattern == null
+ ? Pattern.compile(addDropDDLRegex, Pattern.CASE_INSENSITIVE)
+ : pattern;
+ if (!StringUtils.isNullOrWhitespaceOnly(dorisOptions.getTableIdentifier())) {
String[] tableInfo = dorisOptions.getTableIdentifier().split("\\.");
this.database = tableInfo[0];
this.table = tableInfo[1];
@@ -119,20 +127,24 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
this.schemaChangeManager = new SchemaChangeManager(dorisOptions);
}
- public JsonDebeziumSchemaSerializer(DorisOptions dorisOptions,
+ public JsonDebeziumSchemaSerializer(
+ DorisOptions dorisOptions,
Pattern pattern,
String sourceTableName,
boolean newSchemaChange,
DorisExecutionOptions executionOptions) {
this(dorisOptions, pattern, sourceTableName, newSchemaChange);
if (executionOptions != null) {
- this.lineDelimiter = executionOptions.getStreamLoadProp()
- .getProperty(LINE_DELIMITER_KEY, LINE_DELIMITER_DEFAULT);
+ this.lineDelimiter =
+ executionOptions
+ .getStreamLoadProp()
+ .getProperty(LINE_DELIMITER_KEY, LINE_DELIMITER_DEFAULT);
this.ignoreUpdateBefore = executionOptions.getIgnoreUpdateBefore();
}
}
- public JsonDebeziumSchemaSerializer(DorisOptions dorisOptions,
+ public JsonDebeziumSchemaSerializer(
+ DorisOptions dorisOptions,
Pattern pattern,
String sourceTableName,
boolean newSchemaChange,
@@ -165,11 +177,14 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
initOriginFieldSchema(recordRoot);
}
- //Filter out table records that are not in tableMapping
+ // Filter out table records that are not in tableMapping
String cdcTableIdentifier = getCdcTableIdentifier(recordRoot);
String dorisTableIdentifier = getDorisTableIdentifier(cdcTableIdentifier);
- if(StringUtils.isNullOrWhitespaceOnly(dorisTableIdentifier)){
- LOG.warn("filter table {}, because it is not listened, record detail is {}", cdcTableIdentifier, record);
+ if (StringUtils.isNullOrWhitespaceOnly(dorisTableIdentifier)) {
+ LOG.warn(
+ "filter table {}, because it is not listened, record detail is {}",
+ cdcTableIdentifier,
+ record);
return null;
}
@@ -191,11 +206,13 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
return null;
}
- return DorisRecord.of(dorisTableIdentifier, objectMapper.writeValueAsString(valueMap).getBytes(StandardCharsets.UTF_8));
+ return DorisRecord.of(
+ dorisTableIdentifier,
+ objectMapper.writeValueAsString(valueMap).getBytes(StandardCharsets.UTF_8));
}
/**
- * Change the update event into two
+ * Change the update event into two.
*
* @param recordRoot
* @return
@@ -206,8 +223,7 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
// convert delete
Map<String, Object> beforeRow = extractBeforeRow(recordRoot);
addDeleteSign(beforeRow, true);
- updateRow.append(objectMapper.writeValueAsString(beforeRow))
- .append(this.lineDelimiter);
+ updateRow.append(objectMapper.writeValueAsString(beforeRow)).append(this.lineDelimiter);
}
// convert insert
@@ -225,22 +241,22 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
}
EventType eventType = extractEventType(recordRoot);
- if(eventType == null){
+ if (eventType == null) {
return false;
}
- if(eventType.equals(EventType.CREATE)){
+ if (eventType.equals(EventType.CREATE)) {
TableSchema tableSchema = extractCreateTableSchema(recordRoot);
status = schemaChangeManager.createTable(tableSchema);
- if(status){
+ if (status) {
String cdcTbl = getCdcTableIdentifier(recordRoot);
String dorisTbl = getCreateTableIdentifier(recordRoot);
tableMapping.put(cdcTbl, dorisTbl);
LOG.info("create table ddl status: {}", status);
}
- } else if (eventType.equals(EventType.ALTER)){
+ } else if (eventType.equals(EventType.ALTER)) {
// db,table
Tuple2<String, String> tuple = getDorisTableTuple(recordRoot);
- if(tuple == null){
+ if (tuple == null) {
return false;
}
List<String> ddlSqlList = extractDDLList(recordRoot);
@@ -256,7 +272,7 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
status = doSchemaChange && schemaChangeManager.execute(ddlSql, tuple.f0);
LOG.info("schema change status:{}, ddl:{}", status, ddlSql);
}
- } else{
+ } else {
LOG.info("Unsupported event type {}", eventType);
}
} catch (Exception ex) {
@@ -268,18 +284,16 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
protected JsonNode extractTableChange(JsonNode record) throws JsonProcessingException {
JsonNode historyRecord = extractHistoryRecord(record);
JsonNode tableChanges = historyRecord.get("tableChanges");
- if(!Objects.isNull(tableChanges)){
+ if (!Objects.isNull(tableChanges)) {
JsonNode tableChange = tableChanges.get(0);
return tableChange;
}
return null;
}
- /**
- * Parse Alter Event
- */
+ /** Parse Alter Event. */
@VisibleForTesting
- public List<String> extractDDLList(JsonNode record) throws IOException{
+ public List<String> extractDDLList(JsonNode record) throws IOException {
String dorisTable = getDorisTableIdentifier(record);
JsonNode historyRecord = extractHistoryRecord(record);
String ddl = extractJsonNode(historyRecord, "ddl");
@@ -290,7 +304,9 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
JsonNode columns = tableChange.get("table").get("columns");
if (firstSchemaChange) {
- sourceConnector = SourceConnector.valueOf(record.get("source").get("connector").asText().toUpperCase());
+ sourceConnector =
+ SourceConnector.valueOf(
+ record.get("source").get("connector").asText().toUpperCase());
fillOriginSchema(columns);
}
@@ -320,12 +336,14 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
@VisibleForTesting
public TableSchema extractCreateTableSchema(JsonNode record) throws JsonProcessingException {
- if(sourceConnector == null){
- sourceConnector = SourceConnector.valueOf(record.get("source").get("connector").asText().toUpperCase());
+ if (sourceConnector == null) {
+ sourceConnector =
+ SourceConnector.valueOf(
+ record.get("source").get("connector").asText().toUpperCase());
}
String dorisTable = getCreateTableIdentifier(record);
- JsonNode tableChange = extractTableChange(record);
+ JsonNode tableChange = extractTableChange(record);
JsonNode pkColumns = tableChange.get("table").get("primaryKeyColumnNames");
JsonNode columns = tableChange.get("table").get("columns");
JsonNode comment = tableChange.get("table").get("comment");
@@ -335,7 +353,7 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
buildFieldSchema(field, column);
}
List<String> pkList = new ArrayList<>();
- for(JsonNode column : pkColumns){
+ for (JsonNode column : pkColumns) {
String fieldName = column.asText();
pkList.add(fieldName);
}
@@ -354,11 +372,12 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
return tableSchema;
}
- private List<String> buildDistributeKeys(List<String> primaryKeys, Map<String, FieldSchema> fields) {
+ private List<String> buildDistributeKeys(
+ List<String> primaryKeys, Map<String, FieldSchema> fields) {
if (!CollectionUtil.isNullOrEmpty(primaryKeys)) {
return primaryKeys;
}
- if(!fields.isEmpty()){
+ if (!fields.isEmpty()) {
Map.Entry<String, FieldSchema> firstField = fields.entrySet().iterator().next();
return Collections.singletonList(firstField.getKey());
}
@@ -369,6 +388,7 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
public void setOriginFieldSchemaMap(Map<String, FieldSchema> originFieldSchemaMap) {
this.originFieldSchemaMap = originFieldSchemaMap;
}
+
@VisibleForTesting
public boolean schemaChange(JsonNode recordRoot) {
boolean status = false;
@@ -378,7 +398,7 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
}
// db,table
Tuple2<String, String> tuple = getDorisTableTuple(recordRoot);
- if(tuple == null){
+ if (tuple == null) {
return false;
}
@@ -397,9 +417,7 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
return status;
}
- /**
- * When cdc synchronizes multiple tables, it will capture multiple table schema changes
- */
+ /** When cdc synchronizes multiple tables, it will capture multiple table schema changes. */
protected boolean checkTable(JsonNode recordRoot) {
String db = extractDatabase(recordRoot);
String tbl = extractTable(recordRoot);
@@ -407,64 +425,62 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
return sourceTableName.equals(dbTbl);
}
- public String getCdcTableIdentifier(JsonNode record){
+ public String getCdcTableIdentifier(JsonNode record) {
String db = extractJsonNode(record.get("source"), "db");
String schema = extractJsonNode(record.get("source"), "schema");
String table = extractJsonNode(record.get("source"), "table");
return SourceSchema.getString(db, schema, table);
}
- public String getCreateTableIdentifier(JsonNode record){
+ public String getCreateTableIdentifier(JsonNode record) {
String table = extractJsonNode(record.get("source"), "table");
return targetDatabase + "." + table;
}
- public String getDorisTableIdentifier(String cdcTableIdentifier){
- if(!StringUtils.isNullOrWhitespaceOnly(dorisOptions.getTableIdentifier())){
+ public String getDorisTableIdentifier(String cdcTableIdentifier) {
+ if (!StringUtils.isNullOrWhitespaceOnly(dorisOptions.getTableIdentifier())) {
return dorisOptions.getTableIdentifier();
}
- if(!CollectionUtil.isNullOrEmpty(tableMapping)
+ if (!CollectionUtil.isNullOrEmpty(tableMapping)
&& !StringUtils.isNullOrWhitespaceOnly(cdcTableIdentifier)
- && tableMapping.get(cdcTableIdentifier) != null){
+ && tableMapping.get(cdcTableIdentifier) != null) {
return tableMapping.get(cdcTableIdentifier);
}
return null;
}
- protected String getDorisTableIdentifier(JsonNode record){
+ protected String getDorisTableIdentifier(JsonNode record) {
String identifier = getCdcTableIdentifier(record);
return getDorisTableIdentifier(identifier);
}
- protected Tuple2<String, String> getDorisTableTuple(JsonNode record){
+ protected Tuple2<String, String> getDorisTableTuple(JsonNode record) {
String identifier = getDorisTableIdentifier(record);
- if(StringUtils.isNullOrWhitespaceOnly(identifier)){
+ if (StringUtils.isNullOrWhitespaceOnly(identifier)) {
return null;
}
String[] tableInfo = identifier.split("\\.");
- if(tableInfo.length != 2){
+ if (tableInfo.length != 2) {
return null;
}
return Tuple2.of(tableInfo[0], tableInfo[1]);
}
- private boolean checkSchemaChange(String database, String table, String ddl) throws IOException, IllegalArgumentException {
+ private boolean checkSchemaChange(String database, String table, String ddl)
+ throws IOException, IllegalArgumentException {
Map<String, Object> param = buildRequestParam(ddl);
return schemaChangeManager.checkSchemaChange(database, table, param);
}
- private boolean checkSchemaChange(String database, String table, DDLSchema ddlSchema) throws IOException, IllegalArgumentException {
- Map<String, Object> param = SchemaChangeManager.buildRequestParam(ddlSchema.isDropColumn(), ddlSchema.getColumnName());
+ private boolean checkSchemaChange(String database, String table, DDLSchema ddlSchema)
+ throws IOException, IllegalArgumentException {
+ Map<String, Object> param =
+ SchemaChangeManager.buildRequestParam(
+ ddlSchema.isDropColumn(), ddlSchema.getColumnName());
return schemaChangeManager.checkSchemaChange(database, table, param);
}
- /**
- * Build param
- * {
- * "isDropColumn": true,
- * "columnName" : "column"
- * }
- */
+ /** Build param { "isDropColumn": true, "columnName" : "column" }. */
protected Map<String, Object> buildRequestParam(String ddl) {
Map<String, Object> params = new HashMap<>();
Matcher matcher = addDropDDLPattern.matcher(ddl);
@@ -490,26 +506,25 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
return extractJsonNode(record.get("source"), "table");
}
- /**
- * Parse event type
- */
+ /** Parse event type. */
protected EventType extractEventType(JsonNode record) throws JsonProcessingException {
JsonNode tableChange = extractTableChange(record);
- if(tableChange == null || tableChange.get("type") == null){
+ if (tableChange == null || tableChange.get("type") == null) {
return null;
}
String type = tableChange.get("type").asText();
- if(EventType.ALTER.toString().equalsIgnoreCase(type)){
+ if (EventType.ALTER.toString().equalsIgnoreCase(type)) {
return EventType.ALTER;
- }else if(EventType.CREATE.toString().equalsIgnoreCase(type)){
+ } else if (EventType.CREATE.toString().equalsIgnoreCase(type)) {
return EventType.CREATE;
}
return null;
}
private String extractJsonNode(JsonNode record, String key) {
- return record != null && record.get(key) != null &&
- !(record.get(key) instanceof NullNode) ? record.get(key).asText() : null;
+ return record != null && record.get(key) != null && !(record.get(key) instanceof NullNode)
+ ? record.get(key).asText()
+ : null;
}
private Map<String, Object> extractBeforeRow(JsonNode record) {
@@ -521,8 +536,8 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
}
private Map<String, Object> extractRow(JsonNode recordRow) {
- Map<String, Object> recordMap = objectMapper.convertValue(recordRow, new TypeReference<Map<String, Object>>() {
- });
+ Map<String, Object> recordMap =
+ objectMapper.convertValue(recordRow, new TypeReference<Map<String, Object>>() {});
return recordMap != null ? recordMap : new HashMap<>();
}
@@ -530,7 +545,8 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
if (record != null && record.has("historyRecord")) {
return objectMapper.readTree(record.get("historyRecord").asText());
}
- // The ddl passed by some scenes will not be included in the historyRecord, such as DebeziumSourceFunction
+ // The ddl passed by some scenes will not be included in the historyRecord, such as
+ // DebeziumSourceFunction
return record;
}
@@ -561,7 +577,8 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
String fieldName = column.get("name").asText();
if (originFieldSchemaMap.containsKey(fieldName)) {
String dorisTypeName = buildDorisTypeName(column);
- String defaultValue = handleDefaultValue(extractJsonNode(column, "defaultValueExpression"));
+ String defaultValue =
+ handleDefaultValue(extractJsonNode(column, "defaultValueExpression"));
String comment = extractJsonNode(column, "comment");
FieldSchema fieldSchema = originFieldSchemaMap.get(fieldName);
fieldSchema.setName(fieldName);
@@ -571,8 +588,10 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
}
}
} else {
- LOG.error("Current schema change failed! You need to ensure that "
- + "there is data in the table." + dorisOptions.getTableIdentifier());
+ LOG.error(
+ "Current schema change failed! You need to ensure that "
+ + "there is data in the table."
+ + dorisOptions.getTableIdentifier());
originFieldSchemaMap = new LinkedHashMap<>();
columns.forEach(column -> buildFieldSchema(originFieldSchemaMap, column));
}
@@ -585,7 +604,8 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
String dorisTypeName = buildDorisTypeName(column);
String defaultValue = handleDefaultValue(extractJsonNode(column, "defaultValueExpression"));
String comment = extractJsonNode(column, "comment");
- filedSchemaMap.put(fieldName, new FieldSchema(fieldName, dorisTypeName, defaultValue, comment));
+ filedSchemaMap.put(
+ fieldName, new FieldSchema(fieldName, dorisTypeName, defaultValue, comment));
}
@VisibleForTesting
@@ -618,7 +638,8 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
if (StringUtils.isNullOrWhitespaceOnly(defaultValue)) {
return null;
}
- // Due to historical reasons, doris needs to add quotes to the default value of the new column
+ // Due to historical reasons, doris needs to add quotes to the default value of the new
+ // column
// For example in mysql: alter table add column c1 int default 100
// In Doris: alter table add column c1 int default '100'
if (Pattern.matches("['\"].*?['\"]", defaultValue)) {
@@ -636,7 +657,8 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
if (CollectionUtils.isEmpty(columnNameSet)) {
columnNameSet = extractBeforeRow(recordRoot).keySet();
}
- columnNameSet.forEach(columnName -> originFieldSchemaMap.put(columnName, new FieldSchema()));
+ columnNameSet.forEach(
+ columnName -> originFieldSchemaMap.put(columnName, new FieldSchema()));
firstLoad = false;
}
@@ -659,9 +681,7 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
return new JsonDebeziumSchemaSerializer.Builder();
}
- /**
- * Builder for JsonDebeziumSchemaSerializer.
- */
+ /** Builder for JsonDebeziumSchemaSerializer. */
public static class Builder {
private DorisOptions dorisOptions;
private Pattern addDropDDLPattern;
@@ -713,8 +733,15 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
}
public JsonDebeziumSchemaSerializer build() {
- return new JsonDebeziumSchemaSerializer(dorisOptions, addDropDDLPattern, sourceTableName, newSchemaChange,
- executionOptions, tableMapping, tableProperties, targetDatabase);
+ return new JsonDebeziumSchemaSerializer(
+ dorisOptions,
+ addDropDDLPattern,
+ sourceTableName,
+ newSchemaChange,
+ executionOptions,
+ tableMapping,
+ tableProperties,
+ targetDatabase);
}
}
@@ -733,6 +760,5 @@ public class JsonDebeziumSchemaSerializer implements DorisRecordSerializer<Strin
}
return type;
-
}
-}
\ No newline at end of file
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java
index 74e2853..303ef4d 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java
@@ -30,13 +30,19 @@ public class RecordWithMetaSerializer implements DorisRecordSerializer<RecordWit
@Override
public DorisRecord serialize(RecordWithMeta record) throws IOException {
- if(StringUtils.isBlank(record.getTable())
+ if (StringUtils.isBlank(record.getTable())
|| StringUtils.isBlank(record.getDatabase())
- || record.getRecord() == null){
- LOG.warn("Record or meta format is incorrect, ignore record db:{}, table:{}, row:{}",
- record.getDatabase(), record.getTable(), record.getRecord());
+ || record.getRecord() == null) {
+ LOG.warn(
+ "Record or meta format is incorrect, ignore record db:{}, table:{}, row:{}",
+ record.getDatabase(),
+ record.getTable(),
+ record.getRecord());
return null;
}
- return DorisRecord.of(record.getDatabase(), record.getTable(), record.getRecord().getBytes(StandardCharsets.UTF_8));
+ return DorisRecord.of(
+ record.getDatabase(),
+ record.getTable(),
+ record.getRecord().getBytes(StandardCharsets.UTF_8));
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java
index a9830a5..2615fb9 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java
@@ -17,14 +17,15 @@
package org.apache.doris.flink.sink.writer.serializer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.doris.flink.deserialization.converter.DorisRowConverter;
-import org.apache.doris.flink.sink.EscapeHandler;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.DataType;
import org.apache.flink.types.RowKind;
import org.apache.flink.util.Preconditions;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.doris.flink.deserialization.converter.DorisRowConverter;
+import org.apache.doris.flink.sink.EscapeHandler;
+
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
@@ -36,9 +37,7 @@ import static org.apache.doris.flink.sink.writer.LoadConstants.DORIS_DELETE_SIGN
import static org.apache.doris.flink.sink.writer.LoadConstants.JSON;
import static org.apache.doris.flink.sink.writer.LoadConstants.NULL_VALUE;
-/**
- * Serializer for RowData.
- */
+/** Serializer for RowData. */
public class RowDataSerializer implements DorisRecordSerializer<RowData> {
String[] fieldNames;
String type;
@@ -47,7 +46,12 @@ public class RowDataSerializer implements DorisRecordSerializer<RowData> {
private final boolean enableDelete;
private final DorisRowConverter rowConverter;
- private RowDataSerializer(String[] fieldNames, DataType[] dataTypes, String type, String fieldDelimiter, boolean enableDelete) {
+ private RowDataSerializer(
+ String[] fieldNames,
+ DataType[] dataTypes,
+ String type,
+ String fieldDelimiter,
+ boolean enableDelete) {
this.fieldNames = fieldNames;
this.type = type;
this.fieldDelimiter = fieldDelimiter;
@@ -59,7 +63,7 @@ public class RowDataSerializer implements DorisRecordSerializer<RowData> {
}
@Override
- public DorisRecord serialize(RowData record) throws IOException{
+ public DorisRecord serialize(RowData record) throws IOException {
int maxIndex = Math.min(record.getArity(), fieldNames.length);
String valString;
if (JSON.equals(type)) {
@@ -116,9 +120,7 @@ public class RowDataSerializer implements DorisRecordSerializer<RowData> {
return new Builder();
}
- /**
- * Builder for RowDataSerializer.
- */
+ /** Builder for RowDataSerializer. */
public static class Builder {
private String[] fieldNames;
private DataType[] dataTypes;
@@ -152,7 +154,8 @@ public class RowDataSerializer implements DorisRecordSerializer<RowData> {
}
public RowDataSerializer build() {
- Preconditions.checkState(CSV.equals(type) && fieldDelimiter != null || JSON.equals(type));
+ Preconditions.checkState(
+ CSV.equals(type) && fieldDelimiter != null || JSON.equals(type));
Preconditions.checkNotNull(dataTypes);
Preconditions.checkNotNull(fieldNames);
return new RowDataSerializer(fieldNames, dataTypes, type, fieldDelimiter, deletable);
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java
index a3cf61c..3d7acd2 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java
@@ -30,31 +30,35 @@ import static org.apache.doris.flink.sink.writer.LoadConstants.CSV;
import static org.apache.doris.flink.sink.writer.LoadConstants.JSON;
/**
- * Serializer for {@link Row}.
- * Quick way to support RowSerializer on existing code
- * TODO: support original Doris to Row serializer
+ * Serializer for {@link Row}. Quick way to support RowSerializer on existing code.
+ *
+ * <p>TODO: support original Doris to Row serializer
*/
public class RowSerializer implements DorisRecordSerializer<Row> {
- /**
- * converter {@link Row} to {@link RowData}
- */
+ /** converter {@link Row} to {@link RowData}. */
private final RowRowConverter rowRowConverter;
+
private final RowDataSerializer rowDataSerializer;
- private RowSerializer(String[] fieldNames, DataType[] dataTypes, String type, String fieldDelimiter,
- boolean enableDelete) {
+ private RowSerializer(
+ String[] fieldNames,
+ DataType[] dataTypes,
+ String type,
+ String fieldDelimiter,
+ boolean enableDelete) {
this.rowRowConverter = RowRowConverter.create(DataTypes.ROW(dataTypes));
- this.rowDataSerializer = RowDataSerializer.builder()
- .setFieldNames(fieldNames)
- .setFieldType(dataTypes)
- .setType(type)
- .setFieldDelimiter(fieldDelimiter)
- .enableDelete(enableDelete)
- .build();
+ this.rowDataSerializer =
+ RowDataSerializer.builder()
+ .setFieldNames(fieldNames)
+ .setFieldType(dataTypes)
+ .setType(type)
+ .setFieldDelimiter(fieldDelimiter)
+ .enableDelete(enableDelete)
+ .build();
}
@Override
- public DorisRecord serialize(Row record) throws IOException{
+ public DorisRecord serialize(Row record) throws IOException {
RowData rowDataRecord = this.rowRowConverter.toInternal(record);
return this.rowDataSerializer.serialize(rowDataRecord);
}
@@ -63,9 +67,7 @@ public class RowSerializer implements DorisRecordSerializer<Row> {
return new Builder();
}
- /**
- * Builder for RowSerializer.
- */
+ /** Builder for RowSerializer. */
public static class Builder {
private String[] fieldNames;
private DataType[] dataTypes;
@@ -99,7 +101,8 @@ public class RowSerializer implements DorisRecordSerializer<Row> {
}
public RowSerializer build() {
- Preconditions.checkState(CSV.equals(type) && fieldDelimiter != null || JSON.equals(type));
+ Preconditions.checkState(
+ CSV.equals(type) && fieldDelimiter != null || JSON.equals(type));
Preconditions.checkNotNull(dataTypes);
Preconditions.checkNotNull(fieldNames);
return new RowSerializer(fieldNames, dataTypes, type, fieldDelimiter, deletable);
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java
index aa2ad85..4933385 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java
@@ -17,14 +17,10 @@
package org.apache.doris.flink.sink.writer.serializer;
-import org.apache.flink.api.java.tuple.Tuple2;
-
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-/**
- * String Serializer.
- */
+/** String Serializer. */
public class SimpleStringSerializer implements DorisRecordSerializer<String> {
@Override
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java
index 546dc6e..e7b5ce7 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java
@@ -39,75 +39,107 @@ public class DorisConfigOptions {
public static final String IDENTIFIER = "doris";
// common option
- public static final ConfigOption<String> FENODES = ConfigOptions.key("fenodes").stringType().noDefaultValue().withDescription("doris fe http address.");
- public static final ConfigOption<String> BENODES = ConfigOptions.key("benodes").stringType().noDefaultValue().withDescription("doris be http address.");
- public static final ConfigOption<String> TABLE_IDENTIFIER = ConfigOptions.key("table.identifier").stringType().noDefaultValue().withDescription("the doris table name.");
- public static final ConfigOption<String> USERNAME = ConfigOptions.key("username").stringType().noDefaultValue().withDescription("the doris user name.");
- public static final ConfigOption<String> PASSWORD = ConfigOptions.key("password").stringType().noDefaultValue().withDescription("the doris password.");
- public static final ConfigOption<String> JDBC_URL = ConfigOptions.key("jdbc-url").stringType().noDefaultValue().withDescription("doris jdbc url address.");
- public static final ConfigOption<Boolean> AUTO_REDIRECT = ConfigOptions.key("auto-redirect").booleanType().defaultValue(false).withDescription("Use automatic redirection of fe without explicitly obtaining the be list");
+ public static final ConfigOption<String> FENODES =
+ ConfigOptions.key("fenodes")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("doris fe http address.");
+ public static final ConfigOption<String> BENODES =
+ ConfigOptions.key("benodes")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("doris be http address.");
+ public static final ConfigOption<String> TABLE_IDENTIFIER =
+ ConfigOptions.key("table.identifier")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("the doris table name.");
+ public static final ConfigOption<String> USERNAME =
+ ConfigOptions.key("username")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("the doris user name.");
+ public static final ConfigOption<String> PASSWORD =
+ ConfigOptions.key("password")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("the doris password.");
+ public static final ConfigOption<String> JDBC_URL =
+ ConfigOptions.key("jdbc-url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("doris jdbc url address.");
+ public static final ConfigOption<Boolean> AUTO_REDIRECT =
+ ConfigOptions.key("auto-redirect")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Use automatic redirection of fe without explicitly obtaining the be list");
// source config options
- public static final ConfigOption<String> DORIS_READ_FIELD = ConfigOptions
- .key("doris.read.field")
- .stringType()
- .noDefaultValue()
- .withDescription("List of column names in the Doris table, separated by commas");
- public static final ConfigOption<String> DORIS_FILTER_QUERY = ConfigOptions
- .key("doris.filter.query")
- .stringType()
- .noDefaultValue()
- .withDescription("Filter expression of the query, which is transparently transmitted to Doris. Doris uses this expression to complete source-side data filtering");
- public static final ConfigOption<Integer> DORIS_TABLET_SIZE = ConfigOptions
- .key("doris.request.tablet.size")
- .intType()
- .defaultValue(DORIS_TABLET_SIZE_DEFAULT)
- .withDescription("");
- public static final ConfigOption<Integer> DORIS_REQUEST_CONNECT_TIMEOUT_MS = ConfigOptions
- .key("doris.request.connect.timeout.ms")
- .intType()
- .defaultValue(DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT)
- .withDescription("");
- public static final ConfigOption<Integer> DORIS_REQUEST_READ_TIMEOUT_MS = ConfigOptions
- .key("doris.request.read.timeout.ms")
- .intType()
- .defaultValue(DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT)
- .withDescription("");
- public static final ConfigOption<Integer> DORIS_REQUEST_QUERY_TIMEOUT_S = ConfigOptions
- .key("doris.request.query.timeout.s")
- .intType()
- .defaultValue(DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT)
- .withDescription("");
- public static final ConfigOption<Integer> DORIS_REQUEST_RETRIES = ConfigOptions
- .key("doris.request.retries")
- .intType()
- .defaultValue(DORIS_REQUEST_RETRIES_DEFAULT)
- .withDescription("");
- public static final ConfigOption<Boolean> DORIS_DESERIALIZE_ARROW_ASYNC = ConfigOptions
- .key("doris.deserialize.arrow.async")
- .booleanType()
- .defaultValue(DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT)
- .withDescription("");
- public static final ConfigOption<Integer> DORIS_DESERIALIZE_QUEUE_SIZE = ConfigOptions
- .key("doris.deserialize.queue.size")
- .intType()
- .defaultValue(DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT)
- .withDescription("");
- public static final ConfigOption<Integer> DORIS_BATCH_SIZE = ConfigOptions
- .key("doris.batch.size")
- .intType()
- .defaultValue(DORIS_BATCH_SIZE_DEFAULT)
- .withDescription("");
- public static final ConfigOption<Long> DORIS_EXEC_MEM_LIMIT = ConfigOptions
- .key("doris.exec.mem.limit")
- .longType()
- .defaultValue(DORIS_EXEC_MEM_LIMIT_DEFAULT)
- .withDescription("");
- public static final ConfigOption<Boolean> SOURCE_USE_OLD_API = ConfigOptions
- .key("source.use-old-api")
- .booleanType()
- .defaultValue(false)
- .withDescription("Whether to read data using the new interface defined according to the FLIP-27 specification,default false");
+ public static final ConfigOption<String> DORIS_READ_FIELD =
+ ConfigOptions.key("doris.read.field")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "List of column names in the Doris table, separated by commas");
+ public static final ConfigOption<String> DORIS_FILTER_QUERY =
+ ConfigOptions.key("doris.filter.query")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Filter expression of the query, which is transparently transmitted to Doris. Doris uses this expression to complete source-side data filtering");
+ public static final ConfigOption<Integer> DORIS_TABLET_SIZE =
+ ConfigOptions.key("doris.request.tablet.size")
+ .intType()
+ .defaultValue(DORIS_TABLET_SIZE_DEFAULT)
+ .withDescription("");
+ public static final ConfigOption<Integer> DORIS_REQUEST_CONNECT_TIMEOUT_MS =
+ ConfigOptions.key("doris.request.connect.timeout.ms")
+ .intType()
+ .defaultValue(DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT)
+ .withDescription("");
+ public static final ConfigOption<Integer> DORIS_REQUEST_READ_TIMEOUT_MS =
+ ConfigOptions.key("doris.request.read.timeout.ms")
+ .intType()
+ .defaultValue(DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT)
+ .withDescription("");
+ public static final ConfigOption<Integer> DORIS_REQUEST_QUERY_TIMEOUT_S =
+ ConfigOptions.key("doris.request.query.timeout.s")
+ .intType()
+ .defaultValue(DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT)
+ .withDescription("");
+ public static final ConfigOption<Integer> DORIS_REQUEST_RETRIES =
+ ConfigOptions.key("doris.request.retries")
+ .intType()
+ .defaultValue(DORIS_REQUEST_RETRIES_DEFAULT)
+ .withDescription("");
+ public static final ConfigOption<Boolean> DORIS_DESERIALIZE_ARROW_ASYNC =
+ ConfigOptions.key("doris.deserialize.arrow.async")
+ .booleanType()
+ .defaultValue(DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT)
+ .withDescription("");
+ public static final ConfigOption<Integer> DORIS_DESERIALIZE_QUEUE_SIZE =
+ ConfigOptions.key("doris.deserialize.queue.size")
+ .intType()
+ .defaultValue(DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT)
+ .withDescription("");
+ public static final ConfigOption<Integer> DORIS_BATCH_SIZE =
+ ConfigOptions.key("doris.batch.size")
+ .intType()
+ .defaultValue(DORIS_BATCH_SIZE_DEFAULT)
+ .withDescription("");
+ public static final ConfigOption<Long> DORIS_EXEC_MEM_LIMIT =
+ ConfigOptions.key("doris.exec.mem.limit")
+ .longType()
+ .defaultValue(DORIS_EXEC_MEM_LIMIT_DEFAULT)
+ .withDescription("");
+ public static final ConfigOption<Boolean> SOURCE_USE_OLD_API =
+ ConfigOptions.key("source.use-old-api")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Whether to read data using the new interface defined according to the FLIP-27 specification,default false");
// Lookup options
public static final ConfigOption<Long> LOOKUP_CACHE_MAX_ROWS =
@@ -135,7 +167,8 @@ public class DorisConfigOptions {
ConfigOptions.key("lookup.jdbc.read.batch.size")
.intType()
.defaultValue(128)
- .withDescription("when dimension table query, save the maximum number of batches.");
+ .withDescription(
+ "when dimension table query, save the maximum number of batches.");
public static final ConfigOption<Integer> LOOKUP_JDBC_READ_BATCH_QUEUE_SIZE =
ConfigOptions.key("lookup.jdbc.read.batch.queue-size")
@@ -147,7 +180,8 @@ public class DorisConfigOptions {
ConfigOptions.key("lookup.jdbc.read.thread-size")
.intType()
.defaultValue(3)
- .withDescription("the number of threads for dimension table query, each query occupies a JDBC connection");
+ .withDescription(
+ "the number of threads for dimension table query, each query occupies a JDBC connection");
public static final ConfigOption<Boolean> LOOKUP_JDBC_ASYNC =
ConfigOptions.key("lookup.jdbc.async")
@@ -155,91 +189,93 @@ public class DorisConfigOptions {
.defaultValue(false)
.withDescription("whether to set async lookup");
-
// sink config options
- public static final ConfigOption<Boolean> SINK_ENABLE_2PC = ConfigOptions
- .key("sink.enable-2pc")
- .booleanType()
- .defaultValue(true)
- .withDescription("enable 2PC while loading");
+ public static final ConfigOption<Boolean> SINK_ENABLE_2PC =
+ ConfigOptions.key("sink.enable-2pc")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription("enable 2PC while loading");
- public static final ConfigOption<Integer> SINK_CHECK_INTERVAL = ConfigOptions
- .key("sink.check-interval")
- .intType()
- .defaultValue(10000)
- .withDescription("check exception with the interval while loading");
- public static final ConfigOption<Integer> SINK_MAX_RETRIES = ConfigOptions
- .key("sink.max-retries")
- .intType()
- .defaultValue(3)
- .withDescription("the max retry times if writing records to database failed.");
- public static final ConfigOption<Integer> SINK_BUFFER_SIZE = ConfigOptions
- .key("sink.buffer-size")
- .intType()
- .defaultValue(1024 * 1024)
- .withDescription("the buffer size to cache data for stream load.");
- public static final ConfigOption<Integer> SINK_BUFFER_COUNT = ConfigOptions
- .key("sink.buffer-count")
- .intType()
- .defaultValue(3)
- .withDescription("the buffer count to cache data for stream load.");
- public static final ConfigOption<String> SINK_LABEL_PREFIX = ConfigOptions
- .key("sink.label-prefix")
- .stringType()
- .defaultValue("")
- .withDescription("the unique label prefix.");
- public static final ConfigOption<Boolean> SINK_ENABLE_DELETE = ConfigOptions
- .key("sink.enable-delete")
- .booleanType()
- .defaultValue(true)
- .withDescription("whether to enable the delete function");
+ public static final ConfigOption<Integer> SINK_CHECK_INTERVAL =
+ ConfigOptions.key("sink.check-interval")
+ .intType()
+ .defaultValue(10000)
+ .withDescription("check exception with the interval while loading");
+ public static final ConfigOption<Integer> SINK_MAX_RETRIES =
+ ConfigOptions.key("sink.max-retries")
+ .intType()
+ .defaultValue(3)
+ .withDescription("the max retry times if writing records to database failed.");
+ public static final ConfigOption<Integer> SINK_BUFFER_SIZE =
+ ConfigOptions.key("sink.buffer-size")
+ .intType()
+ .defaultValue(1024 * 1024)
+ .withDescription("the buffer size to cache data for stream load.");
+ public static final ConfigOption<Integer> SINK_BUFFER_COUNT =
+ ConfigOptions.key("sink.buffer-count")
+ .intType()
+ .defaultValue(3)
+ .withDescription("the buffer count to cache data for stream load.");
+ public static final ConfigOption<String> SINK_LABEL_PREFIX =
+ ConfigOptions.key("sink.label-prefix")
+ .stringType()
+ .defaultValue("")
+ .withDescription("the unique label prefix.");
+ public static final ConfigOption<Boolean> SINK_ENABLE_DELETE =
+ ConfigOptions.key("sink.enable-delete")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription("whether to enable the delete function");
public static final ConfigOption<Integer> SINK_PARALLELISM = FactoryUtil.SINK_PARALLELISM;
+ public static final ConfigOption<Boolean> SINK_ENABLE_BATCH_MODE =
+ ConfigOptions.key("sink.enable.batch-mode")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Whether to enable batch write mode");
- public static final ConfigOption<Boolean> SINK_ENABLE_BATCH_MODE = ConfigOptions
- .key("sink.enable.batch-mode")
- .booleanType()
- .defaultValue(false)
- .withDescription("Whether to enable batch write mode");
-
- public static final ConfigOption<Integer> SINK_FLUSH_QUEUE_SIZE = ConfigOptions
- .key("sink.flush.queue-size")
- .intType()
- .defaultValue(2)
- .withDescription("Queue length for async stream load, default is 2");
+ public static final ConfigOption<Integer> SINK_FLUSH_QUEUE_SIZE =
+ ConfigOptions.key("sink.flush.queue-size")
+ .intType()
+ .defaultValue(2)
+ .withDescription("Queue length for async stream load, default is 2");
- public static final ConfigOption<Integer> SINK_BUFFER_FLUSH_MAX_ROWS = ConfigOptions
- .key("sink.buffer-flush.max-rows")
- .intType()
- .defaultValue(50000)
- .withDescription("The maximum number of flush items in each batch, the default is 5w");
+ public static final ConfigOption<Integer> SINK_BUFFER_FLUSH_MAX_ROWS =
+ ConfigOptions.key("sink.buffer-flush.max-rows")
+ .intType()
+ .defaultValue(50000)
+ .withDescription(
+ "The maximum number of flush items in each batch, the default is 5w");
- public static final ConfigOption<Integer> SINK_BUFFER_FLUSH_MAX_BYTES = ConfigOptions
- .key("sink.buffer-flush.max-bytes")
- .intType()
- .defaultValue(10 * 1024 * 1024)
- .withDescription("The maximum number of bytes flushed in each batch, the default is 10MB");
+ public static final ConfigOption<Integer> SINK_BUFFER_FLUSH_MAX_BYTES =
+ ConfigOptions.key("sink.buffer-flush.max-bytes")
+ .intType()
+ .defaultValue(10 * 1024 * 1024)
+ .withDescription(
+ "The maximum number of bytes flushed in each batch, the default is 10MB");
- public static final ConfigOption<Duration> SINK_BUFFER_FLUSH_INTERVAL = ConfigOptions
- .key("sink.buffer-flush.interval")
- .durationType()
- .defaultValue(Duration.ofSeconds(10))
- .withDescription("the flush interval mills, over this time, asynchronous threads will flush data. The " +
- "default value is 10s.");
+ public static final ConfigOption<Duration> SINK_BUFFER_FLUSH_INTERVAL =
+ ConfigOptions.key("sink.buffer-flush.interval")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(10))
+ .withDescription(
+ "the flush interval mills, over this time, asynchronous threads will flush data. The "
+ + "default value is 10s.");
- public static final ConfigOption<Boolean> SINK_IGNORE_UPDATE_BEFORE = ConfigOptions
- .key("sink.ignore.update-before")
- .booleanType()
- .defaultValue(true)
- .withDescription("In the CDC scenario, when the primary key of the upstream is inconsistent with that of the downstream, the update-before data needs to be passed to the downstream as deleted data, otherwise the data cannot be deleted.\n" +
- "The default is to ignore, that is, perform upsert semantics.");
+ public static final ConfigOption<Boolean> SINK_IGNORE_UPDATE_BEFORE =
+ ConfigOptions.key("sink.ignore.update-before")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription(
+ "In the CDC scenario, when the primary key of the upstream is inconsistent with that of the downstream, the update-before data needs to be passed to the downstream as deleted data, otherwise the data cannot be deleted.\n"
+ + "The default is to ignore, that is, perform upsert semantics.");
- public static final ConfigOption<Boolean> SINK_USE_CACHE = ConfigOptions
- .key("sink.use-cache")
- .booleanType()
- .defaultValue(false)
- .withDescription("Whether to use buffer cache for breakpoint resume");
+ public static final ConfigOption<Boolean> SINK_USE_CACHE =
+ ConfigOptions.key("sink.use-cache")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Whether to use buffer cache for breakpoint resume");
// Prefix for Doris StreamLoad specific properties.
public static final String STREAM_LOAD_PROP_PREFIX = "sink.properties.";
@@ -255,5 +291,4 @@ public class DorisConfigOptions {
}
return streamLoadProp;
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
index 521d741..49ec868 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
@@ -14,12 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.table;
-import org.apache.doris.flink.cfg.DorisExecutionOptions;
-import org.apache.doris.flink.cfg.DorisLookupOptions;
-import org.apache.doris.flink.cfg.DorisOptions;
-import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.table.api.TableSchema;
@@ -31,6 +28,11 @@ import org.apache.flink.table.factories.FactoryUtil;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.utils.TableSchemaUtils;
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+
import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
@@ -80,14 +82,14 @@ import static org.apache.doris.flink.table.DorisConfigOptions.STREAM_LOAD_PROP_P
import static org.apache.doris.flink.table.DorisConfigOptions.TABLE_IDENTIFIER;
import static org.apache.doris.flink.table.DorisConfigOptions.USERNAME;
-
/**
* The {@link DorisDynamicTableFactory} translates the catalog table to a table source.
*
* <p>Because the table source requires a decoding format, we are discovering the format using the
* provided {@link FactoryUtil} for convenience.
*/
-public final class DorisDynamicTableFactory implements DynamicTableSourceFactory, DynamicTableSinkFactory {
+public final class DorisDynamicTableFactory
+ implements DynamicTableSourceFactory, DynamicTableSinkFactory {
@Override
public String factoryIdentifier() {
@@ -158,14 +160,17 @@ public final class DorisDynamicTableFactory implements DynamicTableSourceFactory
public DynamicTableSource createDynamicTableSource(Context context) {
// either implement your custom validation logic here ...
// or use the provided helper utility
- final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
+ final FactoryUtil.TableFactoryHelper helper =
+ FactoryUtil.createTableFactoryHelper(this, context);
// validate all options
helper.validateExcept(STREAM_LOAD_PROP_PREFIX);
// get the validated options
final ReadableConfig options = helper.getOptions();
// derive the produced data type (excluding computed columns) from the catalog table
- final DataType producedDataType = context.getCatalogTable().getSchema().toPhysicalRowDataType();
- TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());
+ final DataType producedDataType =
+ context.getCatalogTable().getSchema().toPhysicalRowDataType();
+ TableSchema physicalSchema =
+ TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());
// create and return dynamic table source
return new DorisDynamicTableSource(
getDorisOptions(helper.getOptions()),
@@ -177,12 +182,13 @@ public final class DorisDynamicTableFactory implements DynamicTableSourceFactory
private DorisOptions getDorisOptions(ReadableConfig readableConfig) {
final String fenodes = readableConfig.get(FENODES);
final String benodes = readableConfig.get(BENODES);
- final DorisOptions.Builder builder = DorisOptions.builder()
- .setFenodes(fenodes)
- .setBenodes(benodes)
- .setAutoRedirect(readableConfig.get(AUTO_REDIRECT))
- .setJdbcUrl(readableConfig.get(JDBC_URL))
- .setTableIdentifier(readableConfig.get(TABLE_IDENTIFIER));
+ final DorisOptions.Builder builder =
+ DorisOptions.builder()
+ .setFenodes(fenodes)
+ .setBenodes(benodes)
+ .setAutoRedirect(readableConfig.get(AUTO_REDIRECT))
+ .setJdbcUrl(readableConfig.get(JDBC_URL))
+ .setTableIdentifier(readableConfig.get(TABLE_IDENTIFIER));
readableConfig.getOptional(USERNAME).ifPresent(builder::setUsername);
readableConfig.getOptional(PASSWORD).ifPresent(builder::setPassword);
@@ -206,7 +212,8 @@ public final class DorisDynamicTableFactory implements DynamicTableSourceFactory
return builder.build();
}
- private DorisExecutionOptions getDorisExecutionOptions(ReadableConfig readableConfig, Properties streamLoadProp) {
+ private DorisExecutionOptions getDorisExecutionOptions(
+ ReadableConfig readableConfig, Properties streamLoadProp) {
final DorisExecutionOptions.Builder builder = DorisExecutionOptions.builder();
builder.setCheckInterval(readableConfig.get(SINK_CHECK_INTERVAL));
builder.setMaxRetries(readableConfig.get(SINK_MAX_RETRIES));
@@ -219,8 +226,8 @@ public final class DorisDynamicTableFactory implements DynamicTableSourceFactory
if (!readableConfig.get(SINK_ENABLE_2PC)) {
builder.disable2PC();
- } else if (readableConfig.getOptional(SINK_ENABLE_2PC).isPresent()){
- //force open 2pc
+ } else if (readableConfig.getOptional(SINK_ENABLE_2PC).isPresent()) {
+ // force open 2pc
builder.enable2PC();
}
@@ -246,7 +253,7 @@ public final class DorisDynamicTableFactory implements DynamicTableSourceFactory
return streamLoadProp;
}
- private DorisLookupOptions getDorisLookupOptions(ReadableConfig readableConfig){
+ private DorisLookupOptions getDorisLookupOptions(ReadableConfig readableConfig) {
final DorisLookupOptions.Builder builder = DorisLookupOptions.builder();
builder.setCacheExpireMs(readableConfig.get(LOOKUP_CACHE_TTL).toMillis());
builder.setCacheMaxSize(readableConfig.get(LOOKUP_CACHE_MAX_ROWS));
@@ -261,8 +268,7 @@ public final class DorisDynamicTableFactory implements DynamicTableSourceFactory
@Override
public DynamicTableSink createDynamicTableSink(Context context) {
final FactoryUtil.TableFactoryHelper helper =
- FactoryUtil.createTableFactoryHelper(
- this, context);
+ FactoryUtil.createTableFactoryHelper(this, context);
// validate all options
helper.validateExcept(STREAM_LOAD_PROP_PREFIX);
@@ -278,7 +284,6 @@ public final class DorisDynamicTableFactory implements DynamicTableSourceFactory
getDorisReadOptions(helper.getOptions()),
getDorisExecutionOptions(helper.getOptions(), streamLoadProp),
physicalSchema,
- parallelism
- );
+ parallelism);
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
index a961c3c..83c10a9 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
@@ -17,6 +17,13 @@
package org.apache.doris.flink.table;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.sink.SinkV2Provider;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.util.Preconditions;
+
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
@@ -24,12 +31,6 @@ import org.apache.doris.flink.rest.RestService;
import org.apache.doris.flink.sink.DorisSink;
import org.apache.doris.flink.sink.batch.DorisBatchSink;
import org.apache.doris.flink.sink.writer.serializer.RowDataSerializer;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.connector.ChangelogMode;
-import org.apache.flink.table.connector.sink.DynamicTableSink;
-import org.apache.flink.table.connector.sink.SinkV2Provider;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,9 +45,7 @@ import static org.apache.doris.flink.sink.writer.LoadConstants.FIELD_DELIMITER_D
import static org.apache.doris.flink.sink.writer.LoadConstants.FIELD_DELIMITER_KEY;
import static org.apache.doris.flink.sink.writer.LoadConstants.FORMAT_KEY;
-/**
- * DorisDynamicTableSink
- **/
+/** DorisDynamicTableSink. */
public class DorisDynamicTableSink implements DynamicTableSink {
private static final Logger LOG = LoggerFactory.getLogger(DorisDynamicTableSink.class);
private final DorisOptions options;
@@ -55,11 +54,12 @@ public class DorisDynamicTableSink implements DynamicTableSink {
private final TableSchema tableSchema;
private final Integer sinkParallelism;
- public DorisDynamicTableSink(DorisOptions options,
- DorisReadOptions readOptions,
- DorisExecutionOptions executionOptions,
- TableSchema tableSchema,
- Integer sinkParallelism) {
+ public DorisDynamicTableSink(
+ DorisOptions options,
+ DorisReadOptions readOptions,
+ DorisExecutionOptions executionOptions,
+ TableSchema tableSchema,
+ Integer sinkParallelism) {
this.options = options;
this.readOptions = readOptions;
this.executionOptions = executionOptions;
@@ -69,9 +69,9 @@ public class DorisDynamicTableSink implements DynamicTableSink {
@Override
public ChangelogMode getChangelogMode(ChangelogMode changelogMode) {
- if(executionOptions.getIgnoreUpdateBefore()){
+ if (executionOptions.getIgnoreUpdateBefore()) {
return ChangelogMode.upsert();
- }else{
+ } else {
return ChangelogMode.all();
}
}
@@ -79,11 +79,21 @@ public class DorisDynamicTableSink implements DynamicTableSink {
@Override
public SinkRuntimeProvider getSinkRuntimeProvider(Context context) {
Properties loadProperties = executionOptions.getStreamLoadProp();
- boolean deletable = executionOptions.getDeletable() && RestService.isUniqueKeyType(options, readOptions, LOG);
+ boolean deletable =
+ executionOptions.getDeletable()
+ && RestService.isUniqueKeyType(options, readOptions, LOG);
if (!loadProperties.containsKey(COLUMNS_KEY)) {
String[] fieldNames = tableSchema.getFieldNames();
Preconditions.checkState(fieldNames != null && fieldNames.length > 0);
- String columns = String.join(",", Arrays.stream(fieldNames).map(item -> String.format("`%s`", item.trim().replace("`", ""))).collect(Collectors.toList()));
+ String columns =
+ String.join(
+ ",",
+ Arrays.stream(fieldNames)
+ .map(
+ item ->
+ String.format(
+ "`%s`", item.trim().replace("`", "")))
+ .collect(Collectors.toList()));
if (deletable) {
columns = String.format("%s,%s", columns, DORIS_DELETE_SIGN);
}
@@ -91,22 +101,26 @@ public class DorisDynamicTableSink implements DynamicTableSink {
}
RowDataSerializer.Builder serializerBuilder = RowDataSerializer.builder();
- serializerBuilder.setFieldNames(tableSchema.getFieldNames())
+ serializerBuilder
+ .setFieldNames(tableSchema.getFieldNames())
.setFieldType(tableSchema.getFieldDataTypes())
.setType(loadProperties.getProperty(FORMAT_KEY, CSV))
.enableDelete(deletable)
- .setFieldDelimiter(loadProperties.getProperty(FIELD_DELIMITER_KEY, FIELD_DELIMITER_DEFAULT));
+ .setFieldDelimiter(
+ loadProperties.getProperty(FIELD_DELIMITER_KEY, FIELD_DELIMITER_DEFAULT));
- if(!executionOptions.enableBatchMode()){
+ if (!executionOptions.enableBatchMode()) {
DorisSink.Builder<RowData> dorisSinkBuilder = DorisSink.builder();
- dorisSinkBuilder.setDorisOptions(options)
+ dorisSinkBuilder
+ .setDorisOptions(options)
.setDorisReadOptions(readOptions)
.setDorisExecutionOptions(executionOptions)
.setSerializer(serializerBuilder.build());
return SinkV2Provider.of(dorisSinkBuilder.build(), sinkParallelism);
- }else{
+ } else {
DorisBatchSink.Builder<RowData> dorisBatchSinkBuilder = DorisBatchSink.builder();
- dorisBatchSinkBuilder.setDorisOptions(options)
+ dorisBatchSinkBuilder
+ .setDorisOptions(options)
.setDorisReadOptions(readOptions)
.setDorisExecutionOptions(executionOptions)
.setSerializer(serializerBuilder.build());
@@ -116,7 +130,8 @@ public class DorisDynamicTableSink implements DynamicTableSink {
@Override
public DynamicTableSink copy() {
- return new DorisDynamicTableSink(options, readOptions, executionOptions, tableSchema, sinkParallelism);
+ return new DorisDynamicTableSink(
+ options, readOptions, executionOptions, tableSchema, sinkParallelism);
}
@Override
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
index 35a4489..9057f1f 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
@@ -17,14 +17,6 @@
package org.apache.doris.flink.table;
-import org.apache.doris.flink.cfg.DorisLookupOptions;
-import org.apache.doris.flink.cfg.DorisOptions;
-import org.apache.doris.flink.cfg.DorisReadOptions;
-import org.apache.doris.flink.deserialization.RowDataDeserializationSchema;
-import org.apache.doris.flink.exception.DorisException;
-import org.apache.doris.flink.rest.PartitionDefinition;
-import org.apache.doris.flink.rest.RestService;
-import org.apache.doris.flink.source.DorisSource;
import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.table.api.TableSchema;
@@ -41,6 +33,15 @@ import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushD
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.logical.RowType;
+
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.deserialization.RowDataDeserializationSchema;
+import org.apache.doris.flink.exception.DorisException;
+import org.apache.doris.flink.rest.PartitionDefinition;
+import org.apache.doris.flink.rest.RestService;
+import org.apache.doris.flink.source.DorisSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,32 +52,33 @@ import java.util.stream.Collectors;
/**
* The {@link DorisDynamicTableSource} is used during planning.
*
- * <p>In our example, we don't implement any of the available ability interfaces such as {@link SupportsFilterPushDown}
- * or {@link SupportsProjectionPushDown}. Therefore, the main logic can be found in {@link #getScanRuntimeProvider(ScanContext)}
- * where we instantiate the required {@link SourceFunction} and its {@link DeserializationSchema} for
- * runtime. Both instances are parameterized to return internal data structures (i.e. {@link RowData}).
+ * <p>In our example, we don't implement any of the available ability interfaces such as {@link
+ * SupportsFilterPushDown} or {@link SupportsProjectionPushDown}. Therefore, the main logic can be
+ * found in {@link #getScanRuntimeProvider(ScanContext)} where we instantiate the required {@link
+ * SourceFunction} and its {@link DeserializationSchema} for runtime. Both instances are
+ * parameterized to return internal data structures (i.e. {@link RowData}).
*/
public final class DorisDynamicTableSource implements ScanTableSource, LookupTableSource {
private static final Logger LOG = LoggerFactory.getLogger(DorisDynamicTableSource.class);
private final DorisOptions options;
private final DorisReadOptions readOptions;
- private DorisLookupOptions lookupOptions;
+ private DorisLookupOptions lookupOptions;
private TableSchema physicalSchema;
- public DorisDynamicTableSource(DorisOptions options,
- DorisReadOptions readOptions,
- DorisLookupOptions lookupOptions,
- TableSchema physicalSchema) {
+ public DorisDynamicTableSource(
+ DorisOptions options,
+ DorisReadOptions readOptions,
+ DorisLookupOptions lookupOptions,
+ TableSchema physicalSchema) {
this.options = options;
this.lookupOptions = lookupOptions;
this.readOptions = readOptions;
this.physicalSchema = physicalSchema;
}
- public DorisDynamicTableSource(DorisOptions options,
- DorisReadOptions readOptions,
- TableSchema physicalSchema) {
+ public DorisDynamicTableSource(
+ DorisOptions options, DorisReadOptions readOptions, TableSchema physicalSchema) {
this.options = options;
this.readOptions = readOptions;
this.physicalSchema = physicalSchema;
@@ -91,9 +93,10 @@ public final class DorisDynamicTableSource implements ScanTableSource, LookupTab
@Override
public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderContext) {
- readOptions.setReadFields(Arrays.stream(physicalSchema.getFieldNames())
- .map(item->String.format("`%s`", item.trim().replace("`", "")))
- .collect(Collectors.joining(", ")));
+ readOptions.setReadFields(
+ Arrays.stream(physicalSchema.getFieldNames())
+ .map(item -> String.format("`%s`", item.trim().replace("`", "")))
+ .collect(Collectors.joining(", ")));
if (readOptions.getUseOldApi()) {
List<PartitionDefinition> dorisPartitions;
@@ -102,23 +105,30 @@ public final class DorisDynamicTableSource implements ScanTableSource, LookupTab
} catch (DorisException e) {
throw new RuntimeException("Failed fetch doris partitions");
}
- DorisRowDataInputFormat.Builder builder = DorisRowDataInputFormat.builder()
- .setFenodes(options.getFenodes())
- .setBenodes(options.getBenodes())
- .setUsername(options.getUsername())
- .setPassword(options.getPassword())
- .setTableIdentifier(options.getTableIdentifier())
- .setPartitions(dorisPartitions)
- .setReadOptions(readOptions)
- .setRowType((RowType) physicalSchema.toRowDataType().getLogicalType());
+ DorisRowDataInputFormat.Builder builder =
+ DorisRowDataInputFormat.builder()
+ .setFenodes(options.getFenodes())
+ .setBenodes(options.getBenodes())
+ .setUsername(options.getUsername())
+ .setPassword(options.getPassword())
+ .setTableIdentifier(options.getTableIdentifier())
+ .setPartitions(dorisPartitions)
+ .setReadOptions(readOptions)
+ .setRowType((RowType) physicalSchema.toRowDataType().getLogicalType());
return InputFormatProvider.of(builder.build());
} else {
- //Read data using the interface of the FLIP-27 specification
- DorisSource<RowData> build = DorisSource.<RowData>builder()
- .setDorisReadOptions(readOptions)
- .setDorisOptions(options)
- .setDeserializer(new RowDataDeserializationSchema((RowType) physicalSchema.toRowDataType().getLogicalType()))
- .build();
+ // Read data using the interface of the FLIP-27 specification
+ DorisSource<RowData> build =
+ DorisSource.<RowData>builder()
+ .setDorisReadOptions(readOptions)
+ .setDorisOptions(options)
+ .setDeserializer(
+ new RowDataDeserializationSchema(
+ (RowType)
+ physicalSchema
+ .toRowDataType()
+ .getLogicalType()))
+ .build();
return SourceProvider.of(build);
}
}
@@ -139,7 +149,8 @@ public final class DorisDynamicTableSource implements ScanTableSource, LookupTab
options,
lookupOptions,
DataType.getFieldNames(physicalRowDataType).toArray(new String[0]),
- DataType.getFieldDataTypes(physicalRowDataType).toArray(new DataType[0]),
+ DataType.getFieldDataTypes(physicalRowDataType)
+ .toArray(new DataType[0]),
keyNames,
keyIndexs));
} else {
@@ -148,7 +159,8 @@ public final class DorisDynamicTableSource implements ScanTableSource, LookupTab
options,
lookupOptions,
DataType.getFieldNames(physicalRowDataType).toArray(new String[0]),
- DataType.getFieldDataTypes(physicalRowDataType).toArray(new DataType[0]),
+ DataType.getFieldDataTypes(physicalRowDataType)
+ .toArray(new DataType[0]),
keyNames,
keyIndexs));
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
index da168ba..a20a9e1 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
@@ -17,20 +17,21 @@
package org.apache.doris.flink.table;
-import org.apache.doris.flink.cfg.DorisLookupOptions;
-import org.apache.doris.flink.cfg.DorisOptions;
-import org.apache.doris.flink.lookup.DorisJdbcLookupReader;
-import org.apache.doris.flink.lookup.DorisLookupReader;
-import org.apache.doris.flink.lookup.LookupSchema;
import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.shaded.guava30.com.google.common.cache.Cache;
-import org.apache.flink.shaded.guava30.com.google.common.cache.CacheBuilder;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.functions.AsyncTableFunction;
import org.apache.flink.table.functions.FunctionContext;
import org.apache.flink.table.types.DataType;
import org.apache.flink.util.Preconditions;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.lookup.DorisJdbcLookupReader;
+import org.apache.doris.flink.lookup.DorisLookupReader;
+import org.apache.doris.flink.lookup.LookupSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +43,8 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
public class DorisRowDataAsyncLookupFunction extends AsyncTableFunction<RowData> {
- private static final Logger LOG = LoggerFactory.getLogger(DorisRowDataAsyncLookupFunction.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DorisRowDataAsyncLookupFunction.class);
private final DorisOptions options;
private final DorisLookupOptions lookupOptions;
private final long cacheMaxSize;
@@ -51,39 +53,49 @@ public class DorisRowDataAsyncLookupFunction extends AsyncTableFunction<RowData>
private DorisLookupReader lookupReader;
private LookupSchema lookupSchema;
- public DorisRowDataAsyncLookupFunction(DorisOptions options,
- DorisLookupOptions lookupOptions,
- String[] selectFields,
- DataType[] fieldTypes,
- String[] conditionFields,
- int[] keyIndex) {
- Preconditions.checkNotNull(options.getJdbcUrl(), "jdbc-url is required in jdbc mode lookup");
+ public DorisRowDataAsyncLookupFunction(
+ DorisOptions options,
+ DorisLookupOptions lookupOptions,
+ String[] selectFields,
+ DataType[] fieldTypes,
+ String[] conditionFields,
+ int[] keyIndex) {
+ Preconditions.checkNotNull(
+ options.getJdbcUrl(), "jdbc-url is required in jdbc mode lookup");
this.options = options;
this.cacheMaxSize = lookupOptions.getCacheMaxSize();
this.cacheExpireMs = lookupOptions.getCacheExpireMs();
this.lookupOptions = lookupOptions;
- this.lookupSchema = new LookupSchema(options.getTableIdentifier(), selectFields, fieldTypes, conditionFields, keyIndex);
+ this.lookupSchema =
+ new LookupSchema(
+ options.getTableIdentifier(),
+ selectFields,
+ fieldTypes,
+ conditionFields,
+ keyIndex);
}
@Override
public void open(FunctionContext context) throws Exception {
super.open(context);
- LOG.info("lookup options: threadSize {}, batchSize {}, queueSize {}",
- lookupOptions.getJdbcReadThreadSize(), lookupOptions.getJdbcReadBatchSize(), lookupOptions.getJdbcReadBatchQueueSize());
- this.cache = cacheMaxSize == -1 || cacheExpireMs == -1
- ? null
- : CacheBuilder.newBuilder()
- .expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
- .maximumSize(cacheMaxSize)
- .build();
+ LOG.info(
+ "lookup options: threadSize {}, batchSize {}, queueSize {}",
+ lookupOptions.getJdbcReadThreadSize(),
+ lookupOptions.getJdbcReadBatchSize(),
+ lookupOptions.getJdbcReadBatchQueueSize());
+ this.cache =
+ cacheMaxSize == -1 || cacheExpireMs == -1
+ ? null
+ : CacheBuilder.newBuilder()
+ .expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
+ .maximumSize(cacheMaxSize)
+ .build();
this.lookupReader = new DorisJdbcLookupReader(options, lookupOptions, lookupSchema);
}
- /**
- * This is a lookup method which is called by Flink framework in runtime.
- *
- */
- public void eval(CompletableFuture<Collection<RowData>> future, Object... keys) throws IOException {
+ /** This is a lookup method which is called by Flink framework in runtime. */
+ public void eval(CompletableFuture<Collection<RowData>> future, Object... keys)
+ throws IOException {
RowData keyRow = GenericRowData.of(keys);
if (cache != null) {
List<RowData> cachedRows = cache.getIfPresent(keyRow);
@@ -100,12 +112,12 @@ public class DorisRowDataAsyncLookupFunction extends AsyncTableFunction<RowData>
future.completeExceptionally(throwable);
} else {
if (resultRows == null || resultRows.isEmpty()) {
- if(cache != null){
+ if (cache != null) {
cache.put(keyRow, Collections.emptyList());
}
future.complete(Collections.emptyList());
} else {
- if(cache != null){
+ if (cache != null) {
cache.put(keyRow, resultRows);
}
future.complete(resultRows);
@@ -121,7 +133,7 @@ public class DorisRowDataAsyncLookupFunction extends AsyncTableFunction<RowData>
@Override
public void close() throws Exception {
super.close();
- if(lookupReader != null){
+ if (lookupReader != null) {
lookupReader.close();
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
index 47bcf69..56b0095 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
@@ -14,13 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.table;
-import org.apache.doris.flink.cfg.DorisOptions;
-import org.apache.doris.flink.cfg.DorisReadOptions;
-import org.apache.doris.flink.deserialization.converter.DorisRowConverter;
-import org.apache.doris.flink.rest.PartitionDefinition;
-import org.apache.doris.flink.source.reader.DorisValueReader;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.io.DefaultInputSplitAssigner;
import org.apache.flink.api.common.io.InputFormat;
@@ -32,6 +28,12 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.InputSplitAssigner;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.logical.RowType;
+
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.deserialization.converter.DorisRowConverter;
+import org.apache.doris.flink.rest.PartitionDefinition;
+import org.apache.doris.flink.source.reader.DorisValueReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,11 +42,10 @@ import java.sql.PreparedStatement;
import java.util.ArrayList;
import java.util.List;
-/**
- * InputFormat for {@link DorisDynamicTableSource}.
- */
+/** InputFormat for {@link DorisDynamicTableSource}. */
@Internal
-public class DorisRowDataInputFormat extends RichInputFormat<RowData, DorisTableInputSplit> implements ResultTypeQueryable<RowData> {
+public class DorisRowDataInputFormat extends RichInputFormat<RowData, DorisTableInputSplit>
+ implements ResultTypeQueryable<RowData> {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(DorisRowDataInputFormat.class);
@@ -59,10 +60,11 @@ public class DorisRowDataInputFormat extends RichInputFormat<RowData, DorisTable
private final DorisRowConverter rowConverter;
- public DorisRowDataInputFormat(DorisOptions options,
- List<PartitionDefinition> dorisPartitions,
- DorisReadOptions readOptions,
- RowType rowType) {
+ public DorisRowDataInputFormat(
+ DorisOptions options,
+ List<PartitionDefinition> dorisPartitions,
+ DorisReadOptions readOptions,
+ RowType rowType) {
this.options = options;
this.dorisPartitions = dorisPartitions;
this.readOptions = readOptions;
@@ -71,31 +73,26 @@ public class DorisRowDataInputFormat extends RichInputFormat<RowData, DorisTable
@Override
public void configure(Configuration parameters) {
- //do nothing here
+ // do nothing here
}
@Override
public void openInputFormat() {
- //called once per inputFormat (on open)
+ // called once per inputFormat (on open)
}
@Override
public void closeInputFormat() {
- //called once per inputFormat (on close)
+ // called once per inputFormat (on close)
}
/**
- * Connects to the source database and executes the query in a <b>parallel
- * fashion</b> if
- * this {@link InputFormat} is built using a parameterized query (i.e. using
- * a {@link PreparedStatement})
- * and a proper {@link }, in a <b>non-parallel
- * fashion</b> otherwise.
+ * Connects to the source database and executes the query in a <b>parallel fashion</b> if this
+ * {@link InputFormat} is built using a parameterized query (i.e. using a {@link
+ * PreparedStatement}) and a proper, in a <b>non-parallel fashion</b> otherwise
*
- * @param inputSplit which is ignored if this InputFormat is executed as a
- * non-parallel source,
- * a "hook" to the query parameters otherwise (using its
- * <i>splitNumber</i>)
+ * @param inputSplit which is ignored if this InputFormat is executed as a non-parallel source,
+ * a "hook" to the query parameters otherwise (using its <i>splitNumber</i>)
* @throws IOException if there's an error during the execution of the query
*/
@Override
@@ -110,9 +107,7 @@ public class DorisRowDataInputFormat extends RichInputFormat<RowData, DorisTable
* @throws IOException Indicates that a resource could not be closed.
*/
@Override
- public void close() throws IOException {
-
- }
+ public void close() throws IOException {}
@Override
public TypeInformation<RowData> getProducedType() {
@@ -144,7 +139,7 @@ public class DorisRowDataInputFormat extends RichInputFormat<RowData, DorisTable
}
List next = valueReader.next();
RowData genericRowData = rowConverter.convertInternal(next);
- //update hasNext after we've read the record
+ // update hasNext after we've read the record
hasNext = valueReader.hasNext();
return genericRowData;
}
@@ -179,16 +174,13 @@ public class DorisRowDataInputFormat extends RichInputFormat<RowData, DorisTable
return new Builder();
}
- /**
- * Builder for {@link DorisRowDataInputFormat}.
- */
+ /** Builder for {@link DorisRowDataInputFormat}. */
public static class Builder {
private DorisOptions.Builder optionsBuilder;
private List<PartitionDefinition> partitions;
private DorisReadOptions readOptions;
private RowType rowType;
-
public Builder() {
this.optionsBuilder = DorisOptions.builder();
}
@@ -235,8 +227,7 @@ public class DorisRowDataInputFormat extends RichInputFormat<RowData, DorisTable
public DorisRowDataInputFormat build() {
return new DorisRowDataInputFormat(
- optionsBuilder.build(), partitions, readOptions, rowType
- );
+ optionsBuilder.build(), partitions, readOptions, rowType);
}
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
index bb4d671..4376a53 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
@@ -17,20 +17,21 @@
package org.apache.doris.flink.table;
-import org.apache.doris.flink.cfg.DorisLookupOptions;
-import org.apache.doris.flink.cfg.DorisOptions;
-import org.apache.doris.flink.lookup.DorisJdbcLookupReader;
-import org.apache.doris.flink.lookup.DorisLookupReader;
-import org.apache.doris.flink.lookup.LookupSchema;
import org.apache.flink.annotation.VisibleForTesting;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.functions.FunctionContext;
import org.apache.flink.table.functions.TableFunction;
import org.apache.flink.table.types.DataType;
import org.apache.flink.util.Preconditions;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.lookup.DorisJdbcLookupReader;
+import org.apache.doris.flink.lookup.DorisLookupReader;
+import org.apache.doris.flink.lookup.LookupSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,9 +40,7 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
-/**
- * use jdbc to query
- */
+/** use jdbc to query. */
public class DorisRowDataJdbcLookupFunction extends TableFunction<RowData> {
private static final Logger LOG = LoggerFactory.getLogger(DorisRowDataJdbcLookupFunction.class);
private final DorisOptions options;
@@ -52,29 +51,38 @@ public class DorisRowDataJdbcLookupFunction extends TableFunction<RowData> {
private DorisLookupReader lookupReader;
private LookupSchema lookupSchema;
- public DorisRowDataJdbcLookupFunction(DorisOptions options,
- DorisLookupOptions lookupOptions,
- String[] selectFields,
- DataType[] fieldTypes,
- String[] conditionFields,
- int[] keyIndex) {
- Preconditions.checkNotNull(options.getJdbcUrl(), "jdbc-url is required in jdbc mode lookup");
+ public DorisRowDataJdbcLookupFunction(
+ DorisOptions options,
+ DorisLookupOptions lookupOptions,
+ String[] selectFields,
+ DataType[] fieldTypes,
+ String[] conditionFields,
+ int[] keyIndex) {
+ Preconditions.checkNotNull(
+ options.getJdbcUrl(), "jdbc-url is required in jdbc mode lookup");
this.options = options;
this.cacheMaxSize = lookupOptions.getCacheMaxSize();
this.cacheExpireMs = lookupOptions.getCacheExpireMs();
this.lookupOptions = lookupOptions;
- this.lookupSchema = new LookupSchema(options.getTableIdentifier(), selectFields, fieldTypes, conditionFields, keyIndex);
+ this.lookupSchema =
+ new LookupSchema(
+ options.getTableIdentifier(),
+ selectFields,
+ fieldTypes,
+ conditionFields,
+ keyIndex);
}
@Override
public void open(FunctionContext context) throws Exception {
super.open(context);
- this.cache = cacheMaxSize == -1 || cacheExpireMs == -1
- ? null
- : CacheBuilder.newBuilder()
- .expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
- .maximumSize(cacheMaxSize)
- .build();
+ this.cache =
+ cacheMaxSize == -1 || cacheExpireMs == -1
+ ? null
+ : CacheBuilder.newBuilder()
+ .expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
+ .maximumSize(cacheMaxSize)
+ .build();
this.lookupReader = new DorisJdbcLookupReader(options, lookupOptions, lookupSchema);
}
@@ -99,10 +107,10 @@ public class DorisRowDataJdbcLookupFunction extends TableFunction<RowData> {
private void queryRecord(RowData keyRow) throws IOException {
List<RowData> rowData = lookupReader.get(keyRow);
- if(rowData == null){
+ if (rowData == null) {
rowData = Collections.emptyList();
}
- if(cache != null){
+ if (cache != null) {
cache.put(keyRow, rowData);
}
rowData.forEach(this::collect);
@@ -111,7 +119,7 @@ public class DorisRowDataJdbcLookupFunction extends TableFunction<RowData> {
@Override
public void close() throws Exception {
super.close();
- if(lookupReader != null){
+ if (lookupReader != null) {
lookupReader.close();
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataLookupFunction.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataLookupFunction.java
index d6e0edd..a86b53f 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataLookupFunction.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataLookupFunction.java
@@ -17,6 +17,15 @@
package org.apache.doris.flink.table;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.functions.TableFunction;
+import org.apache.flink.table.types.DataType;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
import org.apache.doris.flink.cfg.DorisLookupOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
@@ -25,14 +34,6 @@ import org.apache.doris.flink.exception.DorisException;
import org.apache.doris.flink.rest.PartitionDefinition;
import org.apache.doris.flink.rest.RestService;
import org.apache.doris.flink.source.reader.DorisValueReader;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.shaded.guava30.com.google.common.cache.Cache;
-import org.apache.flink.shaded.guava30.com.google.common.cache.CacheBuilder;
-import org.apache.flink.table.data.GenericRowData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.functions.FunctionContext;
-import org.apache.flink.table.functions.TableFunction;
-import org.apache.flink.table.types.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,12 +57,13 @@ public class DorisRowDataLookupFunction extends TableFunction<RowData> {
private final DorisRowConverter rowConverter;
private transient Cache<RowData, List<RowData>> cache;
- public DorisRowDataLookupFunction(DorisOptions options,
- DorisReadOptions readOptions,
- DorisLookupOptions lookupOptions,
- String[] selectFields,
- DataType[] fieldTypes,
- String[] conditionFields) {
+ public DorisRowDataLookupFunction(
+ DorisOptions options,
+ DorisReadOptions readOptions,
+ DorisLookupOptions lookupOptions,
+ String[] selectFields,
+ DataType[] fieldTypes,
+ String[] conditionFields) {
this.options = options;
this.readOptions = readOptions;
this.selectFields = selectFields;
@@ -75,12 +77,13 @@ public class DorisRowDataLookupFunction extends TableFunction<RowData> {
@Override
public void open(FunctionContext context) throws Exception {
super.open(context);
- this.cache = cacheMaxSize == -1 || cacheExpireMs == -1
- ? null
- : CacheBuilder.newBuilder()
- .expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
- .maximumSize(cacheMaxSize)
- .build();
+ this.cache =
+ cacheMaxSize == -1 || cacheExpireMs == -1
+ ? null
+ : CacheBuilder.newBuilder()
+ .expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
+ .maximumSize(cacheMaxSize)
+ .build();
}
/**
@@ -105,7 +108,8 @@ public class DorisRowDataLookupFunction extends TableFunction<RowData> {
try {
ArrayList<RowData> rows = new ArrayList<>();
for (PartitionDefinition part : partitions) {
- try (DorisValueReader valueReader = new DorisValueReader(part, options, readOptions)) {
+ try (DorisValueReader valueReader =
+ new DorisValueReader(part, options, readOptions)) {
while (valueReader.hasNext()) {
List<?> record = valueReader.next();
GenericRowData rowData = rowConverter.convertInternal(record);
@@ -114,7 +118,7 @@ public class DorisRowDataLookupFunction extends TableFunction<RowData> {
}
}
}
- if(cache != null){
+ if (cache != null) {
rows.trimToSize();
cache.put(keyRow, rows);
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
index 5e81cc1..9620c57 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
@@ -14,19 +14,17 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.table;
-import org.apache.doris.flink.rest.PartitionDefinition;
import org.apache.flink.core.io.InputSplit;
-/**
- * DorisTableInputSplit
- **/
+import org.apache.doris.flink.rest.PartitionDefinition;
+
+/** DorisTableInputSplit. */
public class DorisTableInputSplit implements InputSplit, java.io.Serializable {
- /**
- * The number of the split.
- */
+ /** The number of the split. */
private final int splitNumber;
protected final PartitionDefinition partition;
@@ -37,10 +35,8 @@ public class DorisTableInputSplit implements InputSplit, java.io.Serializable {
this.partition = partition;
}
-
@Override
public int getSplitNumber() {
return splitNumber;
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java
index fe35ee3..18adde2 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java
@@ -14,26 +14,25 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc;
+import org.apache.flink.api.java.utils.MultipleParameterTool;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.StringUtils;
import org.apache.doris.flink.tools.cdc.mysql.MysqlDatabaseSync;
import org.apache.doris.flink.tools.cdc.oracle.OracleDatabaseSync;
import org.apache.doris.flink.tools.cdc.postgres.PostgresDatabaseSync;
import org.apache.doris.flink.tools.cdc.sqlserver.SqlServerDatabaseSync;
-import org.apache.flink.api.java.utils.MultipleParameterTool;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.StringUtils;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-/**
- * cdc sync tools
- */
+/** cdc sync tools. */
public class CdcTools {
private static final String MYSQL_SYNC_DATABASE = "mysql-sync-database";
private static final String ORACLE_SYNC_DATABASE = "oracle-sync-database";
@@ -96,7 +95,12 @@ public class CdcTools {
syncDatabase(params, databaseSync, postgresConfig, "SqlServer");
}
- private static void syncDatabase(MultipleParameterTool params, DatabaseSync databaseSync, Configuration config, String type) throws Exception {
+ private static void syncDatabase(
+ MultipleParameterTool params,
+ DatabaseSync databaseSync,
+ Configuration config,
+ String type)
+ throws Exception {
String jobName = params.get("job-name");
String database = params.get("database");
String tablePrefix = params.get("table-prefix");
@@ -115,7 +119,8 @@ public class CdcTools {
Configuration sinkConfig = Configuration.fromMap(sinkMap);
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- databaseSync.setEnv(env)
+ databaseSync
+ .setEnv(env)
.setDatabase(database)
.setConfig(config)
.setTablePrefix(tablePrefix)
@@ -132,8 +137,11 @@ public class CdcTools {
.setSingleSink(singleSink)
.create();
databaseSync.build();
- if(StringUtils.isNullOrWhitespaceOnly(jobName)){
- jobName = String.format("%s-Doris Sync Database: %s", type, config.getString("database-name","db"));
+ if (StringUtils.isNullOrWhitespaceOnly(jobName)) {
+ jobName =
+ String.format(
+ "%s-Doris Sync Database: %s",
+ type, config.getString("database-name", "db"));
}
env.execute(jobName);
}
@@ -149,15 +157,14 @@ public class CdcTools {
if (kv.length == 2) {
map.put(kv[0], kv[1]);
continue;
- }else if(kv.length == 1 && EMPTY_KEYS.contains(kv[0])){
+ } else if (kv.length == 1 && EMPTY_KEYS.contains(kv[0])) {
map.put(kv[0], "");
continue;
}
- System.err.println(
- "Invalid " + key + " " + param + ".\n");
+ System.err.println("Invalid " + key + " " + param + ".\n");
return null;
}
return map;
}
-}
\ No newline at end of file
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java
index fe5357e..77bd631 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java
@@ -14,8 +14,18 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.OutputTag;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+
import org.apache.doris.flink.catalog.doris.DorisSystem;
import org.apache.doris.flink.catalog.doris.TableSchema;
import org.apache.doris.flink.cfg.DorisConnectionOptions;
@@ -25,14 +35,6 @@ import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.doris.flink.sink.DorisSink;
import org.apache.doris.flink.sink.writer.serializer.JsonDebeziumSchemaSerializer;
import org.apache.doris.flink.table.DorisConfigOptions;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
-import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.OutputTag;
-import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -84,9 +86,7 @@ public abstract class DatabaseSync {
public abstract DataStreamSource<String> buildCdcSource(StreamExecutionEnvironment env);
- /**
- * Get the prefix of a specific tableList, for example, mysql is database, oracle is schema
- */
+ /** Get the prefix of a specific tableList, for example, mysql is database, oracle is schema. */
public abstract String getTableListPrefix();
public DatabaseSync() throws SQLException {
@@ -96,10 +96,10 @@ public abstract class DatabaseSync {
public void create() {
this.includingPattern = includingTables == null ? null : Pattern.compile(includingTables);
this.excludingPattern = excludingTables == null ? null : Pattern.compile(excludingTables);
- this.multiToOneRulesPattern = multiToOneRulesParser(multiToOneOrigin,multiToOneTarget);
- this.converter = new TableNameConverter(tablePrefix, tableSuffix,multiToOneRulesPattern);
- //default enable light schema change
- if(!this.tableConfig.containsKey(LIGHT_SCHEMA_CHANGE)){
+ this.multiToOneRulesPattern = multiToOneRulesParser(multiToOneOrigin, multiToOneTarget);
+ this.converter = new TableNameConverter(tablePrefix, tableSuffix, multiToOneRulesPattern);
+ // default enable light schema change
+ if (!this.tableConfig.containsKey(LIGHT_SCHEMA_CHANGE)) {
this.tableConfig.put(LIGHT_SCHEMA_CHANGE, "true");
}
}
@@ -121,34 +121,43 @@ public abstract class DatabaseSync {
syncTables.add(schema.getTableName());
String dorisTable = converter.convert(schema.getTableName());
- //Calculate the mapping relationship between upstream and downstream tables
- tableMapping.put(schema.getTableIdentifier(), String.format("%s.%s", database, dorisTable));
+ // Calculate the mapping relationship between upstream and downstream tables
+ tableMapping.put(
+ schema.getTableIdentifier(), String.format("%s.%s", database, dorisTable));
if (!dorisSystem.tableExists(database, dorisTable)) {
TableSchema dorisSchema = schema.convertTableSchema(tableConfig);
- //set doris target database
+ // set doris target database
dorisSchema.setDatabase(database);
dorisSchema.setTable(dorisTable);
dorisSystem.createTable(dorisSchema);
}
- if(!dorisTables.contains(dorisTable)){
+ if (!dorisTables.contains(dorisTable)) {
dorisTables.add(dorisTable);
}
}
- if(createTableOnly){
+ if (createTableOnly) {
System.out.println("Create table finished.");
System.exit(0);
}
config.setString(TABLE_NAME_OPTIONS, getSyncTableList(syncTables));
DataStreamSource<String> streamSource = buildCdcSource(env);
- if(singleSink){
+ if (singleSink) {
streamSource.sinkTo(buildDorisSink());
- }else{
- SingleOutputStreamOperator<Void> parsedStream = streamSource.process(new ParsingProcessFunction(converter));
+ } else {
+ SingleOutputStreamOperator<Void> parsedStream =
+ streamSource.process(new ParsingProcessFunction(converter));
for (String table : dorisTables) {
- OutputTag<String> recordOutputTag = ParsingProcessFunction.createRecordOutputTag(table);
+ OutputTag<String> recordOutputTag =
+ ParsingProcessFunction.createRecordOutputTag(table);
DataStream<String> sideOutput = parsedStream.getSideOutput(recordOutputTag);
- int sinkParallel = sinkConfig.getInteger(DorisConfigOptions.SINK_PARALLELISM, sideOutput.getParallelism());
- sideOutput.sinkTo(buildDorisSink(table)).setParallelism(sinkParallel).name(table).uid(table);
+ int sinkParallel =
+ sinkConfig.getInteger(
+ DorisConfigOptions.SINK_PARALLELISM, sideOutput.getParallelism());
+ sideOutput
+ .sinkTo(buildDorisSink(table))
+ .setParallelism(sinkParallel)
+ .name(table)
+ .uid(table);
}
}
}
@@ -162,25 +171,22 @@ public abstract class DatabaseSync {
Preconditions.checkNotNull(fenodes, "fenodes is empty in sink-conf");
Preconditions.checkNotNull(user, "username is empty in sink-conf");
Preconditions.checkNotNull(jdbcUrl, "jdbcurl is empty in sink-conf");
- DorisConnectionOptions.DorisConnectionOptionsBuilder builder = new DorisConnectionOptions.DorisConnectionOptionsBuilder()
- .withFenodes(fenodes)
- .withBenodes(benodes)
- .withUsername(user)
- .withPassword(passwd)
- .withJdbcUrl(jdbcUrl);
+ DorisConnectionOptions.DorisConnectionOptionsBuilder builder =
+ new DorisConnectionOptions.DorisConnectionOptionsBuilder()
+ .withFenodes(fenodes)
+ .withBenodes(benodes)
+ .withUsername(user)
+ .withPassword(passwd)
+ .withJdbcUrl(jdbcUrl);
return builder.build();
}
- /**
- * create doris sink for multi table
- */
- public DorisSink<String> buildDorisSink(){
+ /** create doris sink for multi table. */
+ public DorisSink<String> buildDorisSink() {
return buildDorisSink(null);
}
- /**
- * create doris sink
- */
+ /** create doris sink. */
public DorisSink<String> buildDorisSink(String table) {
String fenodes = sinkConfig.getString(DorisConfigOptions.FENODES);
String benodes = sinkConfig.getString(DorisConfigOptions.BENODES);
@@ -189,69 +195,92 @@ public abstract class DatabaseSync {
DorisSink.Builder<String> builder = DorisSink.builder();
DorisOptions.Builder dorisBuilder = DorisOptions.builder();
- dorisBuilder.setFenodes(fenodes)
- .setBenodes(benodes)
- .setUsername(user)
- .setPassword(passwd);
- sinkConfig.getOptional(DorisConfigOptions.AUTO_REDIRECT).ifPresent(dorisBuilder::setAutoRedirect);
-
- //single sink not need table identifier
- if(!singleSink && !StringUtils.isNullOrWhitespaceOnly(table)){
+ dorisBuilder.setFenodes(fenodes).setBenodes(benodes).setUsername(user).setPassword(passwd);
+ sinkConfig
+ .getOptional(DorisConfigOptions.AUTO_REDIRECT)
+ .ifPresent(dorisBuilder::setAutoRedirect);
+
+ // single sink not need table identifier
+ if (!singleSink && !StringUtils.isNullOrWhitespaceOnly(table)) {
dorisBuilder.setTableIdentifier(database + "." + table);
}
Properties pro = new Properties();
- //default json data format
+ // default json data format
pro.setProperty("format", "json");
pro.setProperty("read_json_by_line", "true");
- //customer stream load properties
+ // customer stream load properties
Properties streamLoadProp = DorisConfigOptions.getStreamLoadProp(sinkConfig.toMap());
pro.putAll(streamLoadProp);
- DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder()
- .setStreamLoadProp(pro);
-
- sinkConfig.getOptional(DorisConfigOptions.SINK_LABEL_PREFIX).ifPresent(executionBuilder::setLabelPrefix);
- sinkConfig.getOptional(DorisConfigOptions.SINK_ENABLE_DELETE).ifPresent(executionBuilder::setDeletable);
- sinkConfig.getOptional(DorisConfigOptions.SINK_BUFFER_COUNT).ifPresent(executionBuilder::setBufferCount);
- sinkConfig.getOptional(DorisConfigOptions.SINK_BUFFER_SIZE).ifPresent(executionBuilder::setBufferSize);
- sinkConfig.getOptional(DorisConfigOptions.SINK_CHECK_INTERVAL).ifPresent(executionBuilder::setCheckInterval);
- sinkConfig.getOptional(DorisConfigOptions.SINK_MAX_RETRIES).ifPresent(executionBuilder::setMaxRetries);
- sinkConfig.getOptional(DorisConfigOptions.SINK_IGNORE_UPDATE_BEFORE).ifPresent(executionBuilder::setIgnoreUpdateBefore);
-
-
- if(!sinkConfig.getBoolean(DorisConfigOptions.SINK_ENABLE_2PC)){
+ DorisExecutionOptions.Builder executionBuilder =
+ DorisExecutionOptions.builder().setStreamLoadProp(pro);
+
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_LABEL_PREFIX)
+ .ifPresent(executionBuilder::setLabelPrefix);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_ENABLE_DELETE)
+ .ifPresent(executionBuilder::setDeletable);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_BUFFER_COUNT)
+ .ifPresent(executionBuilder::setBufferCount);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_BUFFER_SIZE)
+ .ifPresent(executionBuilder::setBufferSize);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_CHECK_INTERVAL)
+ .ifPresent(executionBuilder::setCheckInterval);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_MAX_RETRIES)
+ .ifPresent(executionBuilder::setMaxRetries);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_IGNORE_UPDATE_BEFORE)
+ .ifPresent(executionBuilder::setIgnoreUpdateBefore);
+
+ if (!sinkConfig.getBoolean(DorisConfigOptions.SINK_ENABLE_2PC)) {
executionBuilder.disable2PC();
- } else if(sinkConfig.getOptional(DorisConfigOptions.SINK_ENABLE_2PC).isPresent()){
- //force open 2pc
+ } else if (sinkConfig.getOptional(DorisConfigOptions.SINK_ENABLE_2PC).isPresent()) {
+ // force open 2pc
executionBuilder.enable2PC();
}
- sinkConfig.getOptional(DorisConfigOptions.SINK_ENABLE_BATCH_MODE).ifPresent(executionBuilder::setBatchMode);
- sinkConfig.getOptional(DorisConfigOptions.SINK_FLUSH_QUEUE_SIZE).ifPresent(executionBuilder::setFlushQueueSize);
- sinkConfig.getOptional(DorisConfigOptions.SINK_BUFFER_FLUSH_MAX_ROWS).ifPresent(executionBuilder::setBufferFlushMaxRows);
- sinkConfig.getOptional(DorisConfigOptions.SINK_BUFFER_FLUSH_MAX_BYTES).ifPresent(executionBuilder::setBufferFlushMaxBytes);
- sinkConfig.getOptional(DorisConfigOptions.SINK_BUFFER_FLUSH_INTERVAL).ifPresent(v-> executionBuilder.setBufferFlushIntervalMs(v.toMillis()));
-
- sinkConfig.getOptional(DorisConfigOptions.SINK_USE_CACHE).ifPresent(executionBuilder::setUseCache);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_ENABLE_BATCH_MODE)
+ .ifPresent(executionBuilder::setBatchMode);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_FLUSH_QUEUE_SIZE)
+ .ifPresent(executionBuilder::setFlushQueueSize);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_BUFFER_FLUSH_MAX_ROWS)
+ .ifPresent(executionBuilder::setBufferFlushMaxRows);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_BUFFER_FLUSH_MAX_BYTES)
+ .ifPresent(executionBuilder::setBufferFlushMaxBytes);
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_BUFFER_FLUSH_INTERVAL)
+ .ifPresent(v -> executionBuilder.setBufferFlushIntervalMs(v.toMillis()));
+
+ sinkConfig
+ .getOptional(DorisConfigOptions.SINK_USE_CACHE)
+ .ifPresent(executionBuilder::setUseCache);
DorisExecutionOptions executionOptions = executionBuilder.build();
builder.setDorisReadOptions(DorisReadOptions.builder().build())
.setDorisExecutionOptions(executionOptions)
- .setSerializer(JsonDebeziumSchemaSerializer.builder()
- .setDorisOptions(dorisBuilder.build())
- .setNewSchemaChange(newSchemaChange)
- .setExecutionOptions(executionOptions)
- .setTableMapping(tableMapping)
- .setTableProperties(tableConfig)
- .setTargetDatabase(database)
- .build())
+ .setSerializer(
+ JsonDebeziumSchemaSerializer.builder()
+ .setDorisOptions(dorisBuilder.build())
+ .setNewSchemaChange(newSchemaChange)
+ .setExecutionOptions(executionOptions)
+ .setTableMapping(tableMapping)
+ .setTableProperties(tableConfig)
+ .setTargetDatabase(database)
+ .build())
.setDorisOptions(dorisBuilder.build());
return builder.build();
}
- /**
- * Filter table that need to be synchronized
- */
+ /** Filter table that need to be synchronized. */
protected boolean isSyncNeeded(String tableName) {
boolean sync = true;
if (includingPattern != null) {
@@ -264,40 +293,43 @@ public abstract class DatabaseSync {
return sync;
}
- protected String getSyncTableList(List<String> syncTables){
- if(!singleSink){
+ protected String getSyncTableList(List<String> syncTables) {
+ if (!singleSink) {
return syncTables.stream()
- .map(v-> getTableListPrefix() + "\\." + v)
+ .map(v -> getTableListPrefix() + "\\." + v)
.collect(Collectors.joining("|"));
- }else{
+ } else {
// includingTablePattern and ^excludingPattern
- String includingPattern = String.format("(%s)\\.(%s)", getTableListPrefix(), includingTables);
+ String includingPattern =
+ String.format("(%s)\\.(%s)", getTableListPrefix(), includingTables);
if (StringUtils.isNullOrWhitespaceOnly(excludingTables)) {
return includingPattern;
- }else{
- String excludingPattern = String.format("?!(%s\\.(%s))$", getTableListPrefix(), excludingTables);
+ } else {
+ String excludingPattern =
+ String.format("?!(%s\\.(%s))$", getTableListPrefix(), excludingTables);
return String.format("(%s)(%s)", includingPattern, excludingPattern);
}
}
}
- /**
- * Filter table that many tables merge to one
- */
- protected HashMap<Pattern,String> multiToOneRulesParser(String multiToOneOrigin,String multiToOneTarget){
- if(StringUtils.isNullOrWhitespaceOnly(multiToOneOrigin) || StringUtils.isNullOrWhitespaceOnly(multiToOneTarget)){
+ /** Filter table that many tables merge to one. */
+ protected HashMap<Pattern, String> multiToOneRulesParser(
+ String multiToOneOrigin, String multiToOneTarget) {
+ if (StringUtils.isNullOrWhitespaceOnly(multiToOneOrigin)
+ || StringUtils.isNullOrWhitespaceOnly(multiToOneTarget)) {
return null;
}
- HashMap<Pattern,String> multiToOneRulesPattern= new HashMap<>();
+ HashMap<Pattern, String> multiToOneRulesPattern = new HashMap<>();
String[] origins = multiToOneOrigin.split("\\|");
String[] targets = multiToOneTarget.split("\\|");
- if(origins.length!=targets.length){
- System.out.println("param error : multi to one params length are not equal,please check your params.");
+ if (origins.length != targets.length) {
+ System.out.println(
+ "param error : multi to one params length are not equal,please check your params.");
System.exit(1);
}
try {
for (int i = 0; i < origins.length; i++) {
- multiToOneRulesPattern.put(Pattern.compile(origins[i]),targets[i]);
+ multiToOneRulesPattern.put(Pattern.compile(origins[i]), targets[i]);
}
} catch (Exception e) {
System.out.println("param error : Your regular expression is incorrect,please check.");
@@ -306,7 +338,6 @@ public abstract class DatabaseSync {
return multiToOneRulesPattern;
}
-
public DatabaseSync setEnv(StreamExecutionEnvironment env) {
this.env = env;
return this;
@@ -386,10 +417,10 @@ public abstract class DatabaseSync {
private static final long serialVersionUID = 1L;
private final String prefix;
private final String suffix;
- private Map<Pattern,String> multiToOneRulesPattern;
+ private Map<Pattern, String> multiToOneRulesPattern;
- TableNameConverter(){
- this("","");
+ TableNameConverter() {
+ this("", "");
}
TableNameConverter(String prefix, String suffix) {
@@ -397,35 +428,35 @@ public abstract class DatabaseSync {
this.suffix = suffix == null ? "" : suffix;
}
- TableNameConverter(String prefix, String suffix,Map<Pattern, String> multiToOneRulesPattern) {
+ TableNameConverter(
+ String prefix, String suffix, Map<Pattern, String> multiToOneRulesPattern) {
this.prefix = prefix == null ? "" : prefix;
this.suffix = suffix == null ? "" : suffix;
this.multiToOneRulesPattern = multiToOneRulesPattern;
}
public String convert(String tableName) {
- if(multiToOneRulesPattern==null){
+ if (multiToOneRulesPattern == null) {
return prefix + tableName + suffix;
}
- String target=null;
+ String target = null;
- for (Map.Entry<Pattern, String> patternStringEntry : multiToOneRulesPattern.entrySet()) {
- if(patternStringEntry.getKey().matcher(tableName).matches()){
- target=patternStringEntry.getValue();
+ for (Map.Entry<Pattern, String> patternStringEntry :
+ multiToOneRulesPattern.entrySet()) {
+ if (patternStringEntry.getKey().matcher(tableName).matches()) {
+ target = patternStringEntry.getValue();
}
}
/**
- * If multiToOneRulesPattern is not null and target is not assigned,
- * then the synchronization task contains both multi to one and one to one ,
- * prefixes and suffixes are added to common one-to-one mapping tables
- * */
- if(target==null){
+ * If multiToOneRulesPattern is not null and target is not assigned, then the
+ * synchronization task contains both multi to one and one to one , prefixes and
+ * suffixes are added to common one-to-one mapping tables
+ */
+ if (target == null) {
return prefix + tableName + suffix;
}
return target;
}
}
-
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java
index a1c17f6..95b687c 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java
@@ -14,15 +14,17 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
import java.util.HashMap;
import java.util.Map;
@@ -41,7 +43,9 @@ public class ParsingProcessFunction extends ProcessFunction<String, Void> {
}
@Override
- public void processElement(String record, ProcessFunction<String, Void>.Context context, Collector<Void> collector) throws Exception {
+ public void processElement(
+ String record, ProcessFunction<String, Void>.Context context, Collector<Void> collector)
+ throws Exception {
JsonNode recordRoot = objectMapper.readValue(record, JsonNode.class);
String tableName = extractJsonNode(recordRoot.get("source"), "table");
String dorisName = converter.convert(tableName);
@@ -58,7 +62,6 @@ public class ParsingProcessFunction extends ProcessFunction<String, Void> {
}
public static OutputTag<String> createRecordOutputTag(String tableName) {
- return new OutputTag<String>("record-" + tableName) {
- };
+ return new OutputTag<String>("record-" + tableName) {};
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java
index a404dea..136a925 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java
@@ -18,7 +18,6 @@
package org.apache.doris.flink.tools.cdc;
public enum SourceConnector {
-
MYSQL("mysql"),
ORACLE("oracle"),
POSTGRES("postgres"),
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java
index 51d8013..426d9ed 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java
@@ -14,12 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc;
+import org.apache.flink.util.StringUtils;
+
import org.apache.doris.flink.catalog.doris.DataModel;
import org.apache.doris.flink.catalog.doris.FieldSchema;
import org.apache.doris.flink.catalog.doris.TableSchema;
-import org.apache.flink.util.StringUtils;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
@@ -40,7 +42,11 @@ public abstract class SourceSchema {
public DataModel model = DataModel.UNIQUE;
public SourceSchema(
- DatabaseMetaData metaData, String databaseName, String schemaName, String tableName, String tableComment)
+ DatabaseMetaData metaData,
+ String databaseName,
+ String schemaName,
+ String tableName,
+ String tableComment)
throws Exception {
this.databaseName = databaseName;
this.schemaName = schemaName;
@@ -78,20 +84,20 @@ public abstract class SourceSchema {
public abstract String convertToDorisType(String fieldType, Integer precision, Integer scale);
- public String getTableIdentifier(){
+ public String getTableIdentifier() {
return getString(databaseName, schemaName, tableName);
}
public static String getString(String databaseName, String schemaName, String tableName) {
StringJoiner identifier = new StringJoiner(".");
- if(!StringUtils.isNullOrWhitespaceOnly(databaseName)){
+ if (!StringUtils.isNullOrWhitespaceOnly(databaseName)) {
identifier.add(databaseName);
}
- if(!StringUtils.isNullOrWhitespaceOnly(schemaName)){
+ if (!StringUtils.isNullOrWhitespaceOnly(schemaName)) {
identifier.add(schemaName);
}
- if(!StringUtils.isNullOrWhitespaceOnly(tableName)){
+ if (!StringUtils.isNullOrWhitespaceOnly(tableName)) {
identifier.add(tableName);
}
@@ -109,15 +115,15 @@ public abstract class SourceSchema {
return tableSchema;
}
- private List<String> buildKeys(){
+ private List<String> buildKeys() {
return buildDistributeKeys();
}
- private List<String> buildDistributeKeys(){
- if(!this.primaryKeys.isEmpty()){
+ private List<String> buildDistributeKeys() {
+ if (!this.primaryKeys.isEmpty()) {
return primaryKeys;
}
- if(!this.fields.isEmpty()){
+ if (!this.fields.isEmpty()) {
Map.Entry<String, FieldSchema> firstField = this.fields.entrySet().iterator().next();
return Collections.singletonList(firstField.getKey());
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java
index e779be0..2080d02 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java
@@ -43,11 +43,12 @@ public class DateToStringConverter implements CustomConverter<SchemaBuilder, Rel
private DateTimeFormatter timestampFormatter = DateTimeFormatter.ISO_DATE_TIME;
private ZoneId timestampZoneId = ZoneId.systemDefault();
- public static Properties DEFAULT_PROPS = new Properties();
+ public static final Properties DEFAULT_PROPS = new Properties();
static {
DEFAULT_PROPS.setProperty("converters", "date");
- DEFAULT_PROPS.setProperty("date.type", "org.apache.doris.flink.tools.cdc.mysql.DateToStringConverter");
+ DEFAULT_PROPS.setProperty(
+ "date.type", "org.apache.doris.flink.tools.cdc.mysql.DateToStringConverter");
DEFAULT_PROPS.setProperty("date.format.date", "yyyy-MM-dd");
DEFAULT_PROPS.setProperty("date.format.datetime", "yyyy-MM-dd HH:mm:ss.SSSSSS");
DEFAULT_PROPS.setProperty("date.format.timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS");
@@ -58,8 +59,12 @@ public class DateToStringConverter implements CustomConverter<SchemaBuilder, Rel
public void configure(Properties props) {
readProps(props, "format.date", p -> dateFormatter = DateTimeFormatter.ofPattern(p));
readProps(props, "format.time", p -> timeFormatter = DateTimeFormatter.ofPattern(p));
- readProps(props, "format.datetime", p -> datetimeFormatter = DateTimeFormatter.ofPattern(p));
- readProps(props, "format.timestamp", p -> timestampFormatter = DateTimeFormatter.ofPattern(p));
+ readProps(
+ props, "format.datetime", p -> datetimeFormatter = DateTimeFormatter.ofPattern(p));
+ readProps(
+ props,
+ "format.timestamp",
+ p -> timestampFormatter = DateTimeFormatter.ofPattern(p));
readProps(props, "format.timestamp.zone", z -> timestampZoneId = ZoneId.of(z));
}
@@ -77,7 +82,8 @@ public class DateToStringConverter implements CustomConverter<SchemaBuilder, Rel
}
@Override
- public void converterFor(RelationalColumn column, ConverterRegistration<SchemaBuilder> registration) {
+ public void converterFor(
+ RelationalColumn column, ConverterRegistration<SchemaBuilder> registration) {
String sqlType = column.typeName().toUpperCase();
SchemaBuilder schemaBuilder = null;
Converter converter = null;
@@ -134,14 +140,16 @@ public class DateToStringConverter implements CustomConverter<SchemaBuilder, Rel
private String convertTimestamp(Object input) {
if (input instanceof ZonedDateTime) {
- // mysql timestamp will be converted to UTC storage,and the zonedDatetime here is UTC time
+ // mysql timestamp will be converted to UTC storage,
+ // and the zonedDatetime here is UTC time
ZonedDateTime zonedDateTime = (ZonedDateTime) input;
- LocalDateTime localDateTime = zonedDateTime.withZoneSameInstant(timestampZoneId).toLocalDateTime();
+ LocalDateTime localDateTime =
+ zonedDateTime.withZoneSameInstant(timestampZoneId).toLocalDateTime();
return timestampFormatter.format(localDateTime);
} else if (input instanceof Timestamp) {
- return timestampFormatter.format(((Timestamp) input).toInstant().atZone(timestampZoneId).toLocalDateTime());
+ return timestampFormatter.format(
+ ((Timestamp) input).toInstant().atZone(timestampZoneId).toLocalDateTime());
}
return null;
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java
index a86b2ab..f00acfc 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java
@@ -14,8 +14,16 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.mysql;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+
import com.ververica.cdc.connectors.mysql.source.MySqlSource;
import com.ververica.cdc.connectors.mysql.source.MySqlSourceBuilder;
import com.ververica.cdc.connectors.mysql.source.config.MySqlSourceOptions;
@@ -30,12 +38,6 @@ import org.apache.doris.flink.catalog.doris.DataModel;
import org.apache.doris.flink.deserialization.DorisJsonDebeziumDeserializationSchema;
import org.apache.doris.flink.tools.cdc.DatabaseSync;
import org.apache.doris.flink.tools.cdc.SourceSchema;
-import org.apache.flink.api.common.eventtime.WatermarkStrategy;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.table.catalog.ObjectPath;
-import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,11 +68,13 @@ public class MysqlDatabaseSync extends DatabaseSync {
try {
Class.forName("com.mysql.cj.jdbc.Driver");
} catch (ClassNotFoundException ex) {
- LOG.warn("can not found class com.mysql.cj.jdbc.Driver, use class com.mysql.jdbc.Driver");
+ LOG.warn(
+ "can not found class com.mysql.cj.jdbc.Driver, use class com.mysql.jdbc.Driver");
try {
Class.forName("com.mysql.jdbc.Driver");
} catch (Exception e) {
- throw new SQLException("No suitable driver found, can not found class com.mysql.cj.jdbc.Driver and com.mysql.jdbc.Driver");
+ throw new SQLException(
+ "No suitable driver found, can not found class com.mysql.cj.jdbc.Driver and com.mysql.jdbc.Driver");
}
}
}
@@ -79,10 +83,18 @@ public class MysqlDatabaseSync extends DatabaseSync {
public Connection getConnection() throws SQLException {
Properties jdbcProperties = getJdbcProperties();
StringBuilder jdbcUrlSb = new StringBuilder(JDBC_URL);
- jdbcProperties.forEach((key, value) -> jdbcUrlSb.append("&").append(key).append("=").append(value));
- String jdbcUrl = String.format(jdbcUrlSb.toString(), config.get(MySqlSourceOptions.HOSTNAME), config.get(MySqlSourceOptions.PORT));
+ jdbcProperties.forEach(
+ (key, value) -> jdbcUrlSb.append("&").append(key).append("=").append(value));
+ String jdbcUrl =
+ String.format(
+ jdbcUrlSb.toString(),
+ config.get(MySqlSourceOptions.HOSTNAME),
+ config.get(MySqlSourceOptions.PORT));
- return DriverManager.getConnection(jdbcUrl,config.get(MySqlSourceOptions.USERNAME),config.get(MySqlSourceOptions.PASSWORD));
+ return DriverManager.getConnection(
+ jdbcUrl,
+ config.get(MySqlSourceOptions.USERNAME),
+ config.get(MySqlSourceOptions.PASSWORD));
}
@Override
@@ -92,7 +104,7 @@ public class MysqlDatabaseSync extends DatabaseSync {
try (Connection conn = getConnection()) {
DatabaseMetaData metaData = conn.getMetaData();
try (ResultSet tables =
- metaData.getTables(databaseName, null, "%", new String[]{"TABLE"})) {
+ metaData.getTables(databaseName, null, "%", new String[] {"TABLE"})) {
while (tables.next()) {
String tableName = tables.getString("TABLE_NAME");
String tableComment = tables.getString("REMARKS");
@@ -101,7 +113,10 @@ public class MysqlDatabaseSync extends DatabaseSync {
}
SourceSchema sourceSchema =
new MysqlSchema(metaData, databaseName, tableName, tableComment);
- sourceSchema.setModel(!sourceSchema.primaryKeys.isEmpty() ? DataModel.UNIQUE : DataModel.DUPLICATE);
+ sourceSchema.setModel(
+ !sourceSchema.primaryKeys.isEmpty()
+ ? DataModel.UNIQUE
+ : DataModel.DUPLICATE);
schemaList.add(sourceSchema);
}
}
@@ -125,32 +140,23 @@ public class MysqlDatabaseSync extends DatabaseSync {
.tableList(tableName);
config.getOptional(MySqlSourceOptions.SERVER_ID).ifPresent(sourceBuilder::serverId);
- config
- .getOptional(MySqlSourceOptions.SERVER_TIME_ZONE)
+ config.getOptional(MySqlSourceOptions.SERVER_TIME_ZONE)
.ifPresent(sourceBuilder::serverTimeZone);
- config
- .getOptional(MySqlSourceOptions.SCAN_SNAPSHOT_FETCH_SIZE)
+ config.getOptional(MySqlSourceOptions.SCAN_SNAPSHOT_FETCH_SIZE)
.ifPresent(sourceBuilder::fetchSize);
- config
- .getOptional(MySqlSourceOptions.CONNECT_TIMEOUT)
+ config.getOptional(MySqlSourceOptions.CONNECT_TIMEOUT)
.ifPresent(sourceBuilder::connectTimeout);
- config
- .getOptional(MySqlSourceOptions.CONNECT_MAX_RETRIES)
+ config.getOptional(MySqlSourceOptions.CONNECT_MAX_RETRIES)
.ifPresent(sourceBuilder::connectMaxRetries);
- config
- .getOptional(MySqlSourceOptions.CONNECTION_POOL_SIZE)
+ config.getOptional(MySqlSourceOptions.CONNECTION_POOL_SIZE)
.ifPresent(sourceBuilder::connectionPoolSize);
- config
- .getOptional(MySqlSourceOptions.HEARTBEAT_INTERVAL)
+ config.getOptional(MySqlSourceOptions.HEARTBEAT_INTERVAL)
.ifPresent(sourceBuilder::heartbeatInterval);
- config
- .getOptional(MySqlSourceOptions.SCAN_NEWLY_ADDED_TABLE_ENABLED)
+ config.getOptional(MySqlSourceOptions.SCAN_NEWLY_ADDED_TABLE_ENABLED)
.ifPresent(sourceBuilder::scanNewlyAddedTableEnabled);
- config
- .getOptional(MySqlSourceOptions.SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE)
+ config.getOptional(MySqlSourceOptions.SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE)
.ifPresent(sourceBuilder::splitSize);
- config
- .getOptional(MySqlSourceOptions.SCAN_INCREMENTAL_CLOSE_IDLE_READER_ENABLED)
+ config.getOptional(MySqlSourceOptions.SCAN_INCREMENTAL_CLOSE_IDLE_READER_ENABLED)
.ifPresent(sourceBuilder::closeIdleReaders);
setChunkColumns(sourceBuilder);
@@ -168,14 +174,11 @@ public class MysqlDatabaseSync extends DatabaseSync {
if (file != null && pos != null) {
offsetBuilder.setBinlogFilePosition(file, pos);
}
- config
- .getOptional(MySqlSourceOptions.SCAN_STARTUP_SPECIFIC_OFFSET_GTID_SET)
+ config.getOptional(MySqlSourceOptions.SCAN_STARTUP_SPECIFIC_OFFSET_GTID_SET)
.ifPresent(offsetBuilder::setGtidSet);
- config
- .getOptional(MySqlSourceOptions.SCAN_STARTUP_SPECIFIC_OFFSET_SKIP_EVENTS)
+ config.getOptional(MySqlSourceOptions.SCAN_STARTUP_SPECIFIC_OFFSET_SKIP_EVENTS)
.ifPresent(offsetBuilder::setSkipEvents);
- config
- .getOptional(MySqlSourceOptions.SCAN_STARTUP_SPECIFIC_OFFSET_SKIP_ROWS)
+ config.getOptional(MySqlSourceOptions.SCAN_STARTUP_SPECIFIC_OFFSET_SKIP_ROWS)
.ifPresent(offsetBuilder::setSkipRows);
sourceBuilder.startupOptions(StartupOptions.specificOffset(offsetBuilder.build()));
} else if ("timestamp".equalsIgnoreCase(startupMode)) {
@@ -186,7 +189,7 @@ public class MysqlDatabaseSync extends DatabaseSync {
Properties jdbcProperties = new Properties();
Properties debeziumProperties = new Properties();
- //date to string
+ // date to string
debeziumProperties.putAll(DateToStringConverter.DEFAULT_PROPS);
for (Map.Entry<String, String> entry : config.toMap().entrySet()) {
@@ -209,10 +212,10 @@ public class MysqlDatabaseSync extends DatabaseSync {
customConverterConfigs.put(JsonConverterConfig.DECIMAL_FORMAT_CONFIG, "numeric");
schema = new JsonDebeziumDeserializationSchema(false, customConverterConfigs);
}
- MySqlSource<String> mySqlSource = sourceBuilder.deserializer(schema).includeSchemaChanges(true).build();
+ MySqlSource<String> mySqlSource =
+ sourceBuilder.deserializer(schema).includeSchemaChanges(true).build();
- return env.fromSource(
- mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL Source");
+ return env.fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL Source");
}
@Override
@@ -222,25 +225,27 @@ public class MysqlDatabaseSync extends DatabaseSync {
}
/**
- * set chunkkeyColumn,eg: db.table1:column1,db.table2:column2
+ * set chunkkeyColumn,eg: db.table1:column1,db.table2:column2.
+ *
* @param sourceBuilder
*/
private void setChunkColumns(MySqlSourceBuilder<String> sourceBuilder) {
Map<ObjectPath, String> chunkColumnMap = getChunkColumnMap();
- for(Map.Entry<ObjectPath, String> entry : chunkColumnMap.entrySet()){
+ for (Map.Entry<ObjectPath, String> entry : chunkColumnMap.entrySet()) {
sourceBuilder.chunkKeyColumn(entry.getKey(), entry.getValue());
}
}
- private Map<ObjectPath, String> getChunkColumnMap(){
+ private Map<ObjectPath, String> getChunkColumnMap() {
Map<ObjectPath, String> chunkMap = new HashMap<>();
- String chunkColumn = config.getString(MySqlSourceOptions.SCAN_INCREMENTAL_SNAPSHOT_CHUNK_KEY_COLUMN);
- if(!StringUtils.isNullOrWhitespaceOnly(chunkColumn)){
+ String chunkColumn =
+ config.getString(MySqlSourceOptions.SCAN_INCREMENTAL_SNAPSHOT_CHUNK_KEY_COLUMN);
+ if (!StringUtils.isNullOrWhitespaceOnly(chunkColumn)) {
final Pattern chunkPattern = Pattern.compile("(\\S+)\\.(\\S+):(\\S+)");
String[] tblColumns = chunkColumn.split(",");
- for(String tblCol : tblColumns){
+ for (String tblCol : tblColumns) {
Matcher matcher = chunkPattern.matcher(tblCol);
- if(matcher.find()){
+ if (matcher.find()) {
String db = matcher.group(1);
String table = matcher.group(2);
String col = matcher.group(3);
@@ -251,7 +256,7 @@ public class MysqlDatabaseSync extends DatabaseSync {
return chunkMap;
}
- private Properties getJdbcProperties(){
+ private Properties getJdbcProperties() {
Properties jdbcProps = new Properties();
for (Map.Entry<String, String> entry : config.toMap().entrySet()) {
String key = entry.getKey();
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java
index ff47fd4..bf0eee7 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.mysql;
import org.apache.doris.flink.tools.cdc.SourceSchema;
@@ -22,11 +23,13 @@ import java.sql.DatabaseMetaData;
public class MysqlSchema extends SourceSchema {
- public MysqlSchema(DatabaseMetaData metaData, String databaseName, String tableName, String tableComment) throws Exception {
+ public MysqlSchema(
+ DatabaseMetaData metaData, String databaseName, String tableName, String tableComment)
+ throws Exception {
super(metaData, databaseName, null, tableName, tableComment);
}
- public String convertToDorisType(String fieldType, Integer precision, Integer scale){
+ public String convertToDorisType(String fieldType, Integer precision, Integer scale) {
return MysqlType.toDorisType(fieldType, precision, scale);
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
index 143ea52..e1e8854 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
@@ -14,11 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.mysql;
-import org.apache.doris.flink.catalog.doris.DorisType;
import org.apache.flink.util.Preconditions;
+import org.apache.doris.flink.catalog.doris.DorisType;
+
public class MysqlType {
private static final String BIT = "BIT";
private static final String BOOLEAN = "BOOLEAN";
@@ -132,17 +134,24 @@ public class MysqlType {
case DECIMAL_UNSIGNED:
case DECIMAL_UNSIGNED_ZEROFILL:
return length != null && length <= 38
- ? String.format("%s(%s,%s)", DorisType.DECIMAL_V3, length, scale != null && scale >= 0 ? scale : 0)
+ ? String.format(
+ "%s(%s,%s)",
+ DorisType.DECIMAL_V3,
+ length,
+ scale != null && scale >= 0 ? scale : 0)
: DorisType.STRING;
case DATE:
return DorisType.DATE_V2;
case DATETIME:
case TIMESTAMP:
- return String.format("%s(%s)", DorisType.DATETIME_V2, Math.min(length == null ? 0 : length, 6));
+ return String.format(
+ "%s(%s)", DorisType.DATETIME_V2, Math.min(length == null ? 0 : length, 6));
case CHAR:
case VARCHAR:
Preconditions.checkNotNull(length);
- return length * 3 > 65533 ? DorisType.STRING : String.format("%s(%s)", DorisType.VARCHAR, length * 3);
+ return length * 3 > 65533
+ ? DorisType.STRING
+ : String.format("%s(%s)", DorisType.VARCHAR, length * 3);
case TINYTEXT:
case TEXT:
case MEDIUMTEXT:
@@ -161,6 +170,5 @@ public class MysqlType {
default:
throw new UnsupportedOperationException("Unsupported MySQL Type: " + type);
}
-
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java
index 92c1f95..0dd4262 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java
@@ -14,8 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.oracle;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
import com.ververica.cdc.connectors.base.options.StartupOptions;
import com.ververica.cdc.connectors.base.source.jdbc.JdbcIncrementalSource;
@@ -28,11 +34,6 @@ import com.ververica.cdc.debezium.table.DebeziumOptions;
import org.apache.doris.flink.catalog.doris.DataModel;
import org.apache.doris.flink.tools.cdc.DatabaseSync;
import org.apache.doris.flink.tools.cdc.SourceSchema;
-import org.apache.flink.api.common.eventtime.WatermarkStrategy;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,11 +71,13 @@ public class OracleDatabaseSync extends DatabaseSync {
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
} catch (ClassNotFoundException ex) {
- LOG.warn("can not found class oracle.jdbc.driver.OracleDriver, use class oracle.jdbc.OracleDriver");
+ LOG.warn(
+ "can not found class oracle.jdbc.driver.OracleDriver, use class oracle.jdbc.OracleDriver");
try {
Class.forName("oracle.jdbc.OracleDriver");
} catch (Exception e) {
- throw new SQLException("No suitable driver found, can not found class oracle.jdbc.driver.OracleDriver and oracle.jdbc.OracleDriver");
+ throw new SQLException(
+ "No suitable driver found, can not found class oracle.jdbc.driver.OracleDriver and oracle.jdbc.OracleDriver");
}
}
}
@@ -82,10 +85,15 @@ public class OracleDatabaseSync extends DatabaseSync {
@Override
public Connection getConnection() throws SQLException {
String jdbcUrl;
- if(!StringUtils.isNullOrWhitespaceOnly(config.get(OracleSourceOptions.URL))){
+ if (!StringUtils.isNullOrWhitespaceOnly(config.get(OracleSourceOptions.URL))) {
jdbcUrl = config.get(OracleSourceOptions.URL);
- }else{
- jdbcUrl = String.format(JDBC_URL, config.get(OracleSourceOptions.HOSTNAME), config.get(OracleSourceOptions.PORT),config.get(OracleSourceOptions.DATABASE_NAME));
+ } else {
+ jdbcUrl =
+ String.format(
+ JDBC_URL,
+ config.get(OracleSourceOptions.HOSTNAME),
+ config.get(OracleSourceOptions.PORT),
+ config.get(OracleSourceOptions.DATABASE_NAME));
}
Properties pro = new Properties();
pro.setProperty("user", config.get(OracleSourceOptions.USERNAME));
@@ -103,7 +111,7 @@ public class OracleDatabaseSync extends DatabaseSync {
try (Connection conn = getConnection()) {
DatabaseMetaData metaData = conn.getMetaData();
try (ResultSet tables =
- metaData.getTables(databaseName, schemaName, "%", new String[]{"TABLE"})) {
+ metaData.getTables(databaseName, schemaName, "%", new String[] {"TABLE"})) {
while (tables.next()) {
String tableName = tables.getString("TABLE_NAME");
String tableComment = tables.getString("REMARKS");
@@ -111,8 +119,12 @@ public class OracleDatabaseSync extends DatabaseSync {
continue;
}
SourceSchema sourceSchema =
- new OracleSchema(metaData, databaseName, schemaName, tableName, tableComment);
- sourceSchema.setModel(!sourceSchema.primaryKeys.isEmpty() ? DataModel.UNIQUE : DataModel.DUPLICATE);
+ new OracleSchema(
+ metaData, databaseName, schemaName, tableName, tableComment);
+ sourceSchema.setModel(
+ !sourceSchema.primaryKeys.isEmpty()
+ ? DataModel.UNIQUE
+ : DataModel.DUPLICATE);
schemaList.add(sourceSchema);
}
}
@@ -128,11 +140,13 @@ public class OracleDatabaseSync extends DatabaseSync {
Preconditions.checkNotNull(databaseName, "database-name in oracle is required");
Preconditions.checkNotNull(schemaName, "schema-name in oracle is required");
String tableName = config.get(OracleSourceOptions.TABLE_NAME);
- //When debezium incrementally reads, it will be judged based on regexp_like.
- //When the regular length exceeds 512, an error will be reported, like ORA-12733: regular expression too long
- if(tableName.length() > 384){
- //max database name length 128
- tableName = StringUtils.isNullOrWhitespaceOnly(includingTables) ? ".*" : includingTables;
+ // When debezium incrementally reads, it will be judged based on regexp_like.
+ // When the regular length exceeds 512, an error will be reported,
+ // like ORA-12733: regular expression too long
+ if (tableName.length() > 384) {
+ // max database name length 128
+ tableName =
+ StringUtils.isNullOrWhitespaceOnly(includingTables) ? ".*" : includingTables;
}
String url = config.get(OracleSourceOptions.URL);
@@ -149,9 +163,9 @@ public class OracleDatabaseSync extends DatabaseSync {
startupOptions = StartupOptions.latest();
}
- //debezium properties set
+ // debezium properties set
debeziumProperties.put("decimal.handling.mode", "string");
- //date to string
+ // date to string
debeziumProperties.putAll(OracleDateConverter.DEFAULT_PROPS);
for (Map.Entry<String, String> entry : config.toMap().entrySet()) {
@@ -167,43 +181,49 @@ public class OracleDatabaseSync extends DatabaseSync {
JsonDebeziumDeserializationSchema schema =
new JsonDebeziumDeserializationSchema(false, customConverterConfigs);
- if(config.getBoolean(OracleSourceOptions.SCAN_INCREMENTAL_SNAPSHOT_ENABLED, false)){
- JdbcIncrementalSource<String> incrSource = OracleSourceBuilder.OracleIncrementalSource.<String>builder()
- .hostname(hostname)
- .url(url)
- .port(port)
- .databaseList(databaseName)
- .schemaList(schemaName)
- .tableList(tableName)
- .username(username)
- .password(password)
- .includeSchemaChanges(true)
- .startupOptions(startupOptions)
- .deserializer(schema)
- .debeziumProperties(debeziumProperties)
- .splitSize(config.get(SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE))
- .splitMetaGroupSize(config.get(CHUNK_META_GROUP_SIZE))
- .fetchSize(config.get(SCAN_SNAPSHOT_FETCH_SIZE))
- .connectTimeout(config.get(CONNECT_TIMEOUT))
- .connectionPoolSize(config.get(CONNECTION_POOL_SIZE))
- .connectMaxRetries(config.get(CONNECT_MAX_RETRIES))
- .distributionFactorUpper(config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_UPPER_BOUND))
- .distributionFactorLower(config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_LOWER_BOUND))
- .build();
- return env.fromSource(incrSource, WatermarkStrategy.noWatermarks(), "Oracle IncrSource");
- }else{
- DebeziumSourceFunction<String> oracleSource = OracleSource.<String>builder().url(url)
- .hostname(hostname)
- .port(port)
- .username(username)
- .password(password)
- .database(databaseName)
- .schemaList(schemaName)
- .tableList(tableName)
- .debeziumProperties(debeziumProperties)
- .startupOptions(startupOptions)
- .deserializer(schema)
- .build();
+ if (config.getBoolean(OracleSourceOptions.SCAN_INCREMENTAL_SNAPSHOT_ENABLED, false)) {
+ JdbcIncrementalSource<String> incrSource =
+ OracleSourceBuilder.OracleIncrementalSource.<String>builder()
+ .hostname(hostname)
+ .url(url)
+ .port(port)
+ .databaseList(databaseName)
+ .schemaList(schemaName)
+ .tableList(tableName)
+ .username(username)
+ .password(password)
+ .includeSchemaChanges(true)
+ .startupOptions(startupOptions)
+ .deserializer(schema)
+ .debeziumProperties(debeziumProperties)
+ .splitSize(config.get(SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE))
+ .splitMetaGroupSize(config.get(CHUNK_META_GROUP_SIZE))
+ .fetchSize(config.get(SCAN_SNAPSHOT_FETCH_SIZE))
+ .connectTimeout(config.get(CONNECT_TIMEOUT))
+ .connectionPoolSize(config.get(CONNECTION_POOL_SIZE))
+ .connectMaxRetries(config.get(CONNECT_MAX_RETRIES))
+ .distributionFactorUpper(
+ config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_UPPER_BOUND))
+ .distributionFactorLower(
+ config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_LOWER_BOUND))
+ .build();
+ return env.fromSource(
+ incrSource, WatermarkStrategy.noWatermarks(), "Oracle IncrSource");
+ } else {
+ DebeziumSourceFunction<String> oracleSource =
+ OracleSource.<String>builder()
+ .url(url)
+ .hostname(hostname)
+ .port(port)
+ .username(username)
+ .password(password)
+ .database(databaseName)
+ .schemaList(schemaName)
+ .tableList(tableName)
+ .debeziumProperties(debeziumProperties)
+ .startupOptions(startupOptions)
+ .deserializer(schema)
+ .build();
return env.addSource(oracleSource, "Oracle Source");
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java
index d7bd8fd..b847476 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.oracle;
import com.ververica.cdc.connectors.shaded.org.apache.kafka.connect.data.SchemaBuilder;
@@ -36,66 +37,76 @@ import java.util.regex.Pattern;
public class OracleDateConverter implements CustomConverter<SchemaBuilder, RelationalColumn> {
private static final Logger log = LoggerFactory.getLogger(OracleDateConverter.class);
- private static final Pattern TO_DATE = Pattern.compile("TO_DATE\\('(.*)',[ ]*'(.*)'\\)", Pattern.CASE_INSENSITIVE);
- private static final Pattern TO_TIMESTAMP = Pattern.compile("TO_TIMESTAMP\\('(.*)'\\)", Pattern.CASE_INSENSITIVE);
- private static final Pattern TIMESTAMP_OR_DATE_REGEX = Pattern.compile("^TIMESTAMP[(]\\d[)]$|^DATE$", Pattern.CASE_INSENSITIVE);
+ private static final Pattern TO_DATE =
+ Pattern.compile("TO_DATE\\('(.*)',[ ]*'(.*)'\\)", Pattern.CASE_INSENSITIVE);
+ private static final Pattern TO_TIMESTAMP =
+ Pattern.compile("TO_TIMESTAMP\\('(.*)'\\)", Pattern.CASE_INSENSITIVE);
+ private static final Pattern TIMESTAMP_OR_DATE_REGEX =
+ Pattern.compile("^TIMESTAMP[(]\\d[)]$|^DATE$", Pattern.CASE_INSENSITIVE);
private ZoneId timestampZoneId = ZoneId.systemDefault();
- public static Properties DEFAULT_PROPS = new Properties();
- private final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss";
- private final String DATETIMEV2_PATTERN = "yyyy-MM-dd HH:mm:ss.SSSSSS";
- private final DateTimeFormatter dateTimeV2Formatter = DateTimeFormatter.ofPattern(DATETIMEV2_PATTERN);
+ public static final Properties DEFAULT_PROPS = new Properties();
+ private static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss";
+ private static final String DATETIMEV2_PATTERN = "yyyy-MM-dd HH:mm:ss.SSSSSS";
+ private final DateTimeFormatter dateTimeV2Formatter =
+ DateTimeFormatter.ofPattern(DATETIMEV2_PATTERN);
static {
DEFAULT_PROPS.setProperty("converters", "oracleDate");
- DEFAULT_PROPS.setProperty("oracleDate.type", "org.apache.doris.flink.tools.cdc.oracle.OracleDateConverter");
+ DEFAULT_PROPS.setProperty(
+ "oracleDate.type", "org.apache.doris.flink.tools.cdc.oracle.OracleDateConverter");
}
- private static final DateTimeFormatter TIMESTAMP_FORMATTER = new DateTimeFormatterBuilder()
- .parseCaseInsensitive()
- .appendPattern("yyyy-MM-dd HH:mm:ss")
- .optionalStart()
- .appendPattern(".")
- .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, false)
- .optionalEnd()
- .toFormatter();
+ private static final DateTimeFormatter TIMESTAMP_FORMATTER =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .appendPattern("yyyy-MM-dd HH:mm:ss")
+ .optionalStart()
+ .appendPattern(".")
+ .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, false)
+ .optionalEnd()
+ .toFormatter();
@Override
- public void configure(Properties props) {
- }
+ public void configure(Properties props) {}
@Override
- public void converterFor(RelationalColumn column, ConverterRegistration<SchemaBuilder> registration) {
+ public void converterFor(
+ RelationalColumn column, ConverterRegistration<SchemaBuilder> registration) {
String typeName = column.typeName();
if (TIMESTAMP_OR_DATE_REGEX.matcher(typeName).matches()) {
- registration.register(SchemaBuilder.string().optional(), value -> {
- if (value == null) {
- if (column.isOptional()) {
- return null;
- } else if (column.hasDefaultValue()) {
- return column.defaultValue();
- } else {
- return null;
- }
- }
+ registration.register(
+ SchemaBuilder.string().optional(),
+ value -> {
+ if (value == null) {
+ if (column.isOptional()) {
+ return null;
+ } else if (column.hasDefaultValue()) {
+ return column.defaultValue();
+ } else {
+ return null;
+ }
+ }
- if (value instanceof String) {
- return convertStringTimestamp((String) value);
- }
- if(value instanceof Timestamp){
- return dateTimeV2Formatter.format(((Timestamp) value).toLocalDateTime());
- }
+ if (value instanceof String) {
+ return convertStringTimestamp((String) value);
+ }
+ if (value instanceof Timestamp) {
+ return dateTimeV2Formatter.format(
+ ((Timestamp) value).toLocalDateTime());
+ }
- //oracle timestamp
- try{
- if (value instanceof TIMESTAMP){
- return dateTimeV2Formatter.format(((TIMESTAMP) value).timestampValue().toLocalDateTime());
- }
- }catch (SQLException ex){
- log.error("convert timestamp failed, values is {}", value);
- }
+ // oracle timestamp
+ try {
+ if (value instanceof TIMESTAMP) {
+ return dateTimeV2Formatter.format(
+ ((TIMESTAMP) value).timestampValue().toLocalDateTime());
+ }
+ } catch (SQLException ex) {
+ log.error("convert timestamp failed, values is {}", value);
+ }
- return null;
- });
+ return null;
+ });
}
}
@@ -105,28 +116,32 @@ public class OracleDateConverter implements CustomConverter<SchemaBuilder, Relat
final Matcher toTimestampMatcher = TO_TIMESTAMP.matcher(data);
if (toTimestampMatcher.matches()) {
String dateText = toTimestampMatcher.group(1);
- dateTime = LocalDateTime.from(TIMESTAMP_FORMATTER.parse(completeMilliseconds(dateText.trim())));
+ dateTime =
+ LocalDateTime.from(
+ TIMESTAMP_FORMATTER.parse(completeMilliseconds(dateText.trim())));
return dateTimeV2Formatter.format(dateTime.atZone(timestampZoneId));
}
final Matcher toDateMatcher = TO_DATE.matcher(data);
if (toDateMatcher.matches()) {
String date = toDateMatcher.group(1);
- dateTime = LocalDateTime.from(TIMESTAMP_FORMATTER.parse(completeMilliseconds(date.trim())));
+ dateTime =
+ LocalDateTime.from(
+ TIMESTAMP_FORMATTER.parse(completeMilliseconds(date.trim())));
return dateTimeV2Formatter.format(dateTime.atZone(timestampZoneId));
}
return null;
}
private String completeMilliseconds(String stringValue) {
- if(stringValue.length() == DATETIMEV2_PATTERN.length()){
+ if (stringValue.length() == DATETIMEV2_PATTERN.length()) {
return stringValue;
}
StringBuilder sb = new StringBuilder(stringValue);
- if(stringValue.length() == DATETIME_PATTERN.length()){
+ if (stringValue.length() == DATETIME_PATTERN.length()) {
sb.append(".");
}
- while (sb.toString().length() < DATETIMEV2_PATTERN.length()){
+ while (sb.toString().length() < DATETIMEV2_PATTERN.length()) {
sb.append(0);
}
return sb.toString();
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java
index 464c9a3..c085754 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.oracle;
import org.apache.doris.flink.tools.cdc.SourceSchema;
@@ -22,8 +23,14 @@ import java.sql.DatabaseMetaData;
public class OracleSchema extends SourceSchema {
- public OracleSchema(DatabaseMetaData metaData, String databaseName, String schemaName, String tableName, String tableComment) throws Exception {
- super(metaData, databaseName, schemaName, tableName, tableComment);
+ public OracleSchema(
+ DatabaseMetaData metaData,
+ String databaseName,
+ String schemaName,
+ String tableName,
+ String tableComment)
+ throws Exception {
+ super(metaData, databaseName, schemaName, tableName, tableComment);
}
@Override
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
index 2024691..6bd5ce3 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
@@ -14,11 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.oracle;
-import org.apache.doris.flink.catalog.doris.DorisType;
import org.apache.flink.util.Preconditions;
+import org.apache.doris.flink.catalog.doris.DorisType;
+
public class OracleType {
private static final String VARCHAR2 = "VARCHAR2";
private static final String NVARCHAR2 = "NVARCHAR2";
@@ -43,12 +45,12 @@ public class OracleType {
public static String toDorisType(String oracleType, Integer precision, Integer scale) {
oracleType = oracleType.toUpperCase();
- if(oracleType.startsWith(INTERVAL)){
+ if (oracleType.startsWith(INTERVAL)) {
oracleType = oracleType.substring(0, 8);
} else if (oracleType.startsWith(TIMESTAMP)) {
return String.format("%s(%s)", DorisType.DATETIME_V2, 6);
}
- switch (oracleType){
+ switch (oracleType) {
case NUMBER:
if (scale <= 0) {
precision -= scale;
@@ -72,7 +74,11 @@ public class OracleType {
precision = scale;
}
return precision != null && precision <= 38
- ? String.format("%s(%s,%s)", DorisType.DECIMAL_V3, precision, scale != null && scale >= 0 ? scale : 0)
+ ? String.format(
+ "%s(%s,%s)",
+ DorisType.DECIMAL_V3,
+ precision,
+ scale != null && scale >= 0 ? scale : 0)
: DorisType.STRING;
case FLOAT:
return DorisType.DOUBLE;
@@ -84,7 +90,9 @@ public class OracleType {
case NCHAR:
case NVARCHAR2:
Preconditions.checkNotNull(precision);
- return precision * 3 > 65533 ? DorisType.STRING : String.format("%s(%s)", DorisType.VARCHAR, precision * 3);
+ return precision * 3 > 65533
+ ? DorisType.STRING
+ : String.format("%s(%s)", DorisType.VARCHAR, precision * 3);
case LONG:
case RAW:
case LONG_RAW:
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java
index 04b4127..2645d83 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java
@@ -14,8 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.postgres;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.Preconditions;
import com.ververica.cdc.connectors.base.options.SourceOptions;
import com.ververica.cdc.connectors.base.options.StartupOptions;
@@ -29,10 +34,6 @@ import com.ververica.cdc.debezium.table.DebeziumOptions;
import org.apache.doris.flink.catalog.doris.DataModel;
import org.apache.doris.flink.tools.cdc.DatabaseSync;
import org.apache.doris.flink.tools.cdc.SourceSchema;
-import org.apache.flink.api.common.eventtime.WatermarkStrategy;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,13 +74,19 @@ public class PostgresDatabaseSync extends DatabaseSync {
try {
Class.forName("org.postgresql.Driver");
} catch (ClassNotFoundException ex) {
- throw new SQLException("No suitable driver found, can not found class org.postgresql.Driver");
+ throw new SQLException(
+ "No suitable driver found, can not found class org.postgresql.Driver");
}
}
@Override
public Connection getConnection() throws SQLException {
- String jdbcUrl = String.format(JDBC_URL, config.get(PostgresSourceOptions.HOSTNAME), config.get(PostgresSourceOptions.PG_PORT),config.get(PostgresSourceOptions.DATABASE_NAME));
+ String jdbcUrl =
+ String.format(
+ JDBC_URL,
+ config.get(PostgresSourceOptions.HOSTNAME),
+ config.get(PostgresSourceOptions.PG_PORT),
+ config.get(PostgresSourceOptions.DATABASE_NAME));
Properties pro = new Properties();
pro.setProperty("user", config.get(PostgresSourceOptions.USERNAME));
pro.setProperty("password", config.get(PostgresSourceOptions.PASSWORD));
@@ -95,7 +102,7 @@ public class PostgresDatabaseSync extends DatabaseSync {
try (Connection conn = getConnection()) {
DatabaseMetaData metaData = conn.getMetaData();
try (ResultSet tables =
- metaData.getTables(databaseName, schemaName, "%", new String[]{"TABLE"})) {
+ metaData.getTables(databaseName, schemaName, "%", new String[] {"TABLE"})) {
while (tables.next()) {
String tableName = tables.getString("TABLE_NAME");
String tableComment = tables.getString("REMARKS");
@@ -103,8 +110,12 @@ public class PostgresDatabaseSync extends DatabaseSync {
continue;
}
SourceSchema sourceSchema =
- new PostgresSchema(metaData, databaseName, schemaName, tableName, tableComment);
- sourceSchema.setModel(sourceSchema.primaryKeys.size() > 0 ? DataModel.UNIQUE : DataModel.DUPLICATE);
+ new PostgresSchema(
+ metaData, databaseName, schemaName, tableName, tableComment);
+ sourceSchema.setModel(
+ sourceSchema.primaryKeys.size() > 0
+ ? DataModel.UNIQUE
+ : DataModel.DUPLICATE);
schemaList.add(sourceSchema);
}
}
@@ -135,7 +146,7 @@ public class PostgresDatabaseSync extends DatabaseSync {
startupOptions = StartupOptions.latest();
}
- //debezium properties set
+ // debezium properties set
Properties debeziumProperties = new Properties();
debeziumProperties.putAll(PostgresDateConverter.DEFAULT_PROPS);
debeziumProperties.put("decimal.handling.mode", "string");
@@ -153,46 +164,51 @@ public class PostgresDatabaseSync extends DatabaseSync {
JsonDebeziumDeserializationSchema schema =
new JsonDebeziumDeserializationSchema(false, customConverterConfigs);
- if(config.getBoolean(SourceOptions.SCAN_INCREMENTAL_SNAPSHOT_ENABLED, false)){
- JdbcIncrementalSource<String> incrSource = PostgresSourceBuilder.PostgresIncrementalSource.<String>builder()
- .hostname(hostname)
- .port(port)
- .database(databaseName)
- .schemaList(schemaName)
- .tableList(tableName)
- .username(username)
- .password(password)
- .deserializer(schema)
- .slotName(slotName)
- .decodingPluginName(config.get(DECODING_PLUGIN_NAME))
- .includeSchemaChanges(true)
- .debeziumProperties(debeziumProperties)
- .startupOptions(startupOptions)
- .splitSize(config.get(SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE))
- .splitMetaGroupSize(config.get(CHUNK_META_GROUP_SIZE))
- .fetchSize(config.get(SCAN_SNAPSHOT_FETCH_SIZE))
- .connectTimeout(config.get(CONNECT_TIMEOUT))
- .connectionPoolSize(config.get(CONNECTION_POOL_SIZE))
- .connectMaxRetries(config.get(CONNECT_MAX_RETRIES))
- .distributionFactorUpper(config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_UPPER_BOUND))
- .distributionFactorLower(config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_LOWER_BOUND))
- .heartbeatInterval(config.get(HEARTBEAT_INTERVAL))
- .build();
- return env.fromSource(incrSource, WatermarkStrategy.noWatermarks(), "Postgres IncrSource");
- }else{
- DebeziumSourceFunction<String> postgresSource = PostgreSQLSource.<String>builder()
- .hostname(hostname)
- .port(port)
- .database(databaseName)
- .schemaList(schemaName)
- .tableList(tableName)
- .username(username)
- .password(password)
- .debeziumProperties(debeziumProperties)
- .deserializer(schema)
- .slotName(slotName)
- .decodingPluginName(config.get(DECODING_PLUGIN_NAME))
- .build();
+ if (config.getBoolean(SourceOptions.SCAN_INCREMENTAL_SNAPSHOT_ENABLED, false)) {
+ JdbcIncrementalSource<String> incrSource =
+ PostgresSourceBuilder.PostgresIncrementalSource.<String>builder()
+ .hostname(hostname)
+ .port(port)
+ .database(databaseName)
+ .schemaList(schemaName)
+ .tableList(tableName)
+ .username(username)
+ .password(password)
+ .deserializer(schema)
+ .slotName(slotName)
+ .decodingPluginName(config.get(DECODING_PLUGIN_NAME))
+ .includeSchemaChanges(true)
+ .debeziumProperties(debeziumProperties)
+ .startupOptions(startupOptions)
+ .splitSize(config.get(SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE))
+ .splitMetaGroupSize(config.get(CHUNK_META_GROUP_SIZE))
+ .fetchSize(config.get(SCAN_SNAPSHOT_FETCH_SIZE))
+ .connectTimeout(config.get(CONNECT_TIMEOUT))
+ .connectionPoolSize(config.get(CONNECTION_POOL_SIZE))
+ .connectMaxRetries(config.get(CONNECT_MAX_RETRIES))
+ .distributionFactorUpper(
+ config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_UPPER_BOUND))
+ .distributionFactorLower(
+ config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_LOWER_BOUND))
+ .heartbeatInterval(config.get(HEARTBEAT_INTERVAL))
+ .build();
+ return env.fromSource(
+ incrSource, WatermarkStrategy.noWatermarks(), "Postgres IncrSource");
+ } else {
+ DebeziumSourceFunction<String> postgresSource =
+ PostgreSQLSource.<String>builder()
+ .hostname(hostname)
+ .port(port)
+ .database(databaseName)
+ .schemaList(schemaName)
+ .tableList(tableName)
+ .username(username)
+ .password(password)
+ .debeziumProperties(debeziumProperties)
+ .deserializer(schema)
+ .slotName(slotName)
+ .decodingPluginName(config.get(DECODING_PLUGIN_NAME))
+ .build();
return env.addSource(postgresSource, "Postgres Source");
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java
index b91269d..6856238 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java
@@ -39,11 +39,12 @@ public class PostgresDateConverter implements CustomConverter<SchemaBuilder, Rel
private DateTimeFormatter dateFormatter = DateTimeFormatter.ISO_DATE;
private DateTimeFormatter timestampFormatter = DateTimeFormatter.ISO_DATE_TIME;
- public static Properties DEFAULT_PROPS = new Properties();
+ public static final Properties DEFAULT_PROPS = new Properties();
static {
DEFAULT_PROPS.setProperty("converters", "date");
- DEFAULT_PROPS.setProperty("date.type", "org.apache.doris.flink.tools.cdc.postgres.PostgresDateConverter");
+ DEFAULT_PROPS.setProperty(
+ "date.type", "org.apache.doris.flink.tools.cdc.postgres.PostgresDateConverter");
DEFAULT_PROPS.setProperty("date.format.date", "yyyy-MM-dd");
DEFAULT_PROPS.setProperty("date.format.timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS");
}
@@ -51,7 +52,10 @@ public class PostgresDateConverter implements CustomConverter<SchemaBuilder, Rel
@Override
public void configure(Properties props) {
readProps(props, "format.date", p -> dateFormatter = DateTimeFormatter.ofPattern(p));
- readProps(props, "format.timestamp", p -> timestampFormatter = DateTimeFormatter.ofPattern(p));
+ readProps(
+ props,
+ "format.timestamp",
+ p -> timestampFormatter = DateTimeFormatter.ofPattern(p));
}
private void readProps(Properties properties, String settingKey, Consumer<String> callback) {
@@ -68,7 +72,8 @@ public class PostgresDateConverter implements CustomConverter<SchemaBuilder, Rel
}
@Override
- public void converterFor(RelationalColumn column, ConverterRegistration<SchemaBuilder> registration) {
+ public void converterFor(
+ RelationalColumn column, ConverterRegistration<SchemaBuilder> registration) {
String sqlType = column.typeName().toUpperCase();
SchemaBuilder schemaBuilder = null;
Converter converter = null;
@@ -95,7 +100,7 @@ public class PostgresDateConverter implements CustomConverter<SchemaBuilder, Rel
} else if (input instanceof Integer) {
LocalDate date = LocalDate.ofEpochDay((Integer) input);
return dateFormatter.format(date);
- } else if (input instanceof Date){
+ } else if (input instanceof Date) {
return dateFormatter.format(((Date) input).toLocalDate());
}
return null;
@@ -111,8 +116,8 @@ public class PostgresDateConverter implements CustomConverter<SchemaBuilder, Rel
private String convertTimestamp(Object input) {
if (input instanceof Timestamp) {
return timestampFormatter.format(((Timestamp) input).toLocalDateTime());
- } else if (input instanceof Instant){
- LocalDateTime ldt = LocalDateTime.ofInstant(((Instant) input), ZoneOffset.UTC);
+ } else if (input instanceof Instant) {
+ LocalDateTime ldt = LocalDateTime.ofInstant(((Instant) input), ZoneOffset.UTC);
return timestampFormatter.format(ldt);
}
return null;
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java
index de9d869..ecb6edf 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.postgres;
import org.apache.doris.flink.tools.cdc.SourceSchema;
@@ -22,8 +23,14 @@ import java.sql.DatabaseMetaData;
public class PostgresSchema extends SourceSchema {
- public PostgresSchema(DatabaseMetaData metaData, String databaseName, String schemaName, String tableName, String tableComment) throws Exception {
- super(metaData, databaseName, schemaName, tableName, tableComment);
+ public PostgresSchema(
+ DatabaseMetaData metaData,
+ String databaseName,
+ String schemaName,
+ String tableName,
+ String tableComment)
+ throws Exception {
+ super(metaData, databaseName, schemaName, tableName, tableComment);
}
@Override
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
index b77496d..2508b15 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
@@ -14,11 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.postgres;
-import org.apache.doris.flink.catalog.doris.DorisType;
import org.apache.flink.util.Preconditions;
+import org.apache.doris.flink.catalog.doris.DorisType;
+
public class PostgresType {
private static final String INT2 = "int2";
private static final String SMALLSERIAL = "smallserial";
@@ -67,10 +69,10 @@ public class PostgresType {
public static String toDorisType(String postgresType, Integer precision, Integer scale) {
postgresType = postgresType.toLowerCase();
- if(postgresType.startsWith("_")){
- return DorisType.STRING;
+ if (postgresType.startsWith("_")) {
+ return DorisType.STRING;
}
- switch (postgresType){
+ switch (postgresType) {
case INT2:
case SMALLSERIAL:
return DorisType.TINYINT;
@@ -82,7 +84,11 @@ public class PostgresType {
return DorisType.BIGINT;
case NUMERIC:
return precision != null && precision > 0 && precision <= 38
- ? String.format("%s(%s,%s)", DorisType.DECIMAL_V3, precision, scale != null && scale >= 0 ? scale : 0)
+ ? String.format(
+ "%s(%s,%s)",
+ DorisType.DECIMAL_V3,
+ precision,
+ scale != null && scale >= 0 ? scale : 0)
: DorisType.STRING;
case FLOAT4:
return DorisType.FLOAT;
@@ -90,7 +96,9 @@ public class PostgresType {
return DorisType.DOUBLE;
case TIMESTAMP:
case TIMESTAMPTZ:
- return String.format("%s(%s)", DorisType.DATETIME_V2, Math.min(precision == null ? 0 : precision, 6));
+ return String.format(
+ "%s(%s)",
+ DorisType.DATETIME_V2, Math.min(precision == null ? 0 : precision, 6));
case DATE:
return DorisType.DATE_V2;
case BOOL:
@@ -100,7 +108,9 @@ public class PostgresType {
case BPCHAR:
case VARCHAR:
Preconditions.checkNotNull(precision);
- return precision * 3 > 65533 ? DorisType.STRING : String.format("%s(%s)", DorisType.VARCHAR, precision * 3);
+ return precision * 3 > 65533
+ ? DorisType.STRING
+ : String.format("%s(%s)", DorisType.VARCHAR, precision * 3);
case POINT:
case LINE:
case LSEG:
@@ -122,29 +132,30 @@ public class PostgresType {
case JSON:
case JSONB:
return DorisType.JSONB;
- /* Compatible with doris1.2 array type can only be used in dup table,
- and then converted to array in the next version
- case _BOOL:
- return String.format("%s<%s>", DorisType.ARRAY, DorisType.BOOLEAN);
- case _INT2:
- return String.format("%s<%s>", DorisType.ARRAY, DorisType.TINYINT);
- case _INT4:
- return String.format("%s<%s>", DorisType.ARRAY, DorisType.INT);
- case _INT8:
- return String.format("%s<%s>", DorisType.ARRAY, DorisType.BIGINT);
- case _FLOAT4:
- return String.format("%s<%s>", DorisType.ARRAY, DorisType.FLOAT);
- case _FLOAT8:
- return String.format("%s<%s>", DorisType.ARRAY, DorisType.DOUBLE);
- case _TEXT:
- return String.format("%s<%s>", DorisType.ARRAY, DorisType.STRING);
- case _DATE:
- return String.format("%s<%s>", DorisType.ARRAY, DorisType.DATE_V2);
- case _TIMESTAMP:
- return String.format("%s<%s>", DorisType.ARRAY, DorisType.DATETIME_V2);
- **/
+ /* Compatible with doris1.2 array type can only be used in dup table,
+ and then converted to array in the next version
+ case _BOOL:
+ return String.format("%s<%s>", DorisType.ARRAY, DorisType.BOOLEAN);
+ case _INT2:
+ return String.format("%s<%s>", DorisType.ARRAY, DorisType.TINYINT);
+ case _INT4:
+ return String.format("%s<%s>", DorisType.ARRAY, DorisType.INT);
+ case _INT8:
+ return String.format("%s<%s>", DorisType.ARRAY, DorisType.BIGINT);
+ case _FLOAT4:
+ return String.format("%s<%s>", DorisType.ARRAY, DorisType.FLOAT);
+ case _FLOAT8:
+ return String.format("%s<%s>", DorisType.ARRAY, DorisType.DOUBLE);
+ case _TEXT:
+ return String.format("%s<%s>", DorisType.ARRAY, DorisType.STRING);
+ case _DATE:
+ return String.format("%s<%s>", DorisType.ARRAY, DorisType.DATE_V2);
+ case _TIMESTAMP:
+ return String.format("%s<%s>", DorisType.ARRAY, DorisType.DATETIME_V2);
+ **/
default:
- throw new UnsupportedOperationException("Unsupported Postgres Type: " + postgresType);
+ throw new UnsupportedOperationException(
+ "Unsupported Postgres Type: " + postgresType);
}
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java
index 10db3c1..fcdb6b1 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java
@@ -14,8 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.sqlserver;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.Preconditions;
import com.ververica.cdc.connectors.base.options.JdbcSourceOptions;
import com.ververica.cdc.connectors.base.options.SourceOptions;
@@ -29,10 +34,6 @@ import com.ververica.cdc.debezium.table.DebeziumOptions;
import org.apache.doris.flink.catalog.doris.DataModel;
import org.apache.doris.flink.tools.cdc.DatabaseSync;
import org.apache.doris.flink.tools.cdc.SourceSchema;
-import org.apache.flink.api.common.eventtime.WatermarkStrategy;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,13 +71,19 @@ public class SqlServerDatabaseSync extends DatabaseSync {
try {
Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
} catch (ClassNotFoundException ex) {
- throw new SQLException("No suitable driver found, can not found class com.microsoft.sqlserver.jdbc.SQLServerDriver");
+ throw new SQLException(
+ "No suitable driver found, can not found class com.microsoft.sqlserver.jdbc.SQLServerDriver");
}
}
@Override
public Connection getConnection() throws SQLException {
- String jdbcUrl = String.format(JDBC_URL, config.get(JdbcSourceOptions.HOSTNAME), config.getInteger(PORT, 1433),config.get(JdbcSourceOptions.DATABASE_NAME));
+ String jdbcUrl =
+ String.format(
+ JDBC_URL,
+ config.get(JdbcSourceOptions.HOSTNAME),
+ config.getInteger(PORT, 1433),
+ config.get(JdbcSourceOptions.DATABASE_NAME));
Properties pro = new Properties();
pro.setProperty("user", config.get(JdbcSourceOptions.USERNAME));
pro.setProperty("password", config.get(JdbcSourceOptions.PASSWORD));
@@ -92,7 +99,7 @@ public class SqlServerDatabaseSync extends DatabaseSync {
try (Connection conn = getConnection()) {
DatabaseMetaData metaData = conn.getMetaData();
try (ResultSet tables =
- metaData.getTables(databaseName, null, "%", new String[]{"TABLE"})) {
+ metaData.getTables(databaseName, null, "%", new String[] {"TABLE"})) {
while (tables.next()) {
String tableName = tables.getString("TABLE_NAME");
String tableComment = tables.getString("REMARKS");
@@ -100,8 +107,12 @@ public class SqlServerDatabaseSync extends DatabaseSync {
continue;
}
SourceSchema sourceSchema =
- new SqlServerSchema(metaData, databaseName, null, tableName, tableComment);
- sourceSchema.setModel(!sourceSchema.primaryKeys.isEmpty() ? DataModel.UNIQUE : DataModel.DUPLICATE);
+ new SqlServerSchema(
+ metaData, databaseName, null, tableName, tableComment);
+ sourceSchema.setModel(
+ !sourceSchema.primaryKeys.isEmpty()
+ ? DataModel.UNIQUE
+ : DataModel.DUPLICATE);
schemaList.add(sourceSchema);
}
}
@@ -130,7 +141,7 @@ public class SqlServerDatabaseSync extends DatabaseSync {
startupOptions = StartupOptions.latest();
}
- //debezium properties set
+ // debezium properties set
Properties debeziumProperties = new Properties();
debeziumProperties.putAll(SqlServerDateConverter.DEFAULT_PROPS);
debeziumProperties.put("decimal.handling.mode", "string");
@@ -148,40 +159,45 @@ public class SqlServerDatabaseSync extends DatabaseSync {
JsonDebeziumDeserializationSchema schema =
new JsonDebeziumDeserializationSchema(false, customConverterConfigs);
- if(config.getBoolean(SourceOptions.SCAN_INCREMENTAL_SNAPSHOT_ENABLED, false)){
- JdbcIncrementalSource<String> incrSource = SqlServerSourceBuilder.SqlServerIncrementalSource.<String>builder()
- .hostname(hostname)
- .port(port)
- .databaseList(databaseName)
- .tableList(tableName)
- .username(username)
- .password(password)
- .startupOptions(startupOptions)
- .deserializer(schema)
- .includeSchemaChanges(true)
- .debeziumProperties(debeziumProperties)
- .splitSize(config.get(SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE))
- .splitMetaGroupSize(config.get(CHUNK_META_GROUP_SIZE))
- .fetchSize(config.get(SCAN_SNAPSHOT_FETCH_SIZE))
- .connectTimeout(config.get(CONNECT_TIMEOUT))
- .connectionPoolSize(config.get(CONNECTION_POOL_SIZE))
- .connectMaxRetries(config.get(CONNECT_MAX_RETRIES))
- .distributionFactorUpper(config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_UPPER_BOUND))
- .distributionFactorLower(config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_LOWER_BOUND))
- .build();
- return env.fromSource(incrSource, WatermarkStrategy.noWatermarks(), "SqlServer IncrSource");
- }else{
- DebeziumSourceFunction<String> sqlServerSource = SqlServerSource.<String>builder()
- .hostname(hostname)
- .port(port)
- .database(databaseName)
- .tableList(tableName)
- .username(username)
- .password(password)
- .debeziumProperties(debeziumProperties)
- .startupOptions(startupOptions)
- .deserializer(schema)
- .build();
+ if (config.getBoolean(SourceOptions.SCAN_INCREMENTAL_SNAPSHOT_ENABLED, false)) {
+ JdbcIncrementalSource<String> incrSource =
+ SqlServerSourceBuilder.SqlServerIncrementalSource.<String>builder()
+ .hostname(hostname)
+ .port(port)
+ .databaseList(databaseName)
+ .tableList(tableName)
+ .username(username)
+ .password(password)
+ .startupOptions(startupOptions)
+ .deserializer(schema)
+ .includeSchemaChanges(true)
+ .debeziumProperties(debeziumProperties)
+ .splitSize(config.get(SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE))
+ .splitMetaGroupSize(config.get(CHUNK_META_GROUP_SIZE))
+ .fetchSize(config.get(SCAN_SNAPSHOT_FETCH_SIZE))
+ .connectTimeout(config.get(CONNECT_TIMEOUT))
+ .connectionPoolSize(config.get(CONNECTION_POOL_SIZE))
+ .connectMaxRetries(config.get(CONNECT_MAX_RETRIES))
+ .distributionFactorUpper(
+ config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_UPPER_BOUND))
+ .distributionFactorLower(
+ config.get(SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_LOWER_BOUND))
+ .build();
+ return env.fromSource(
+ incrSource, WatermarkStrategy.noWatermarks(), "SqlServer IncrSource");
+ } else {
+ DebeziumSourceFunction<String> sqlServerSource =
+ SqlServerSource.<String>builder()
+ .hostname(hostname)
+ .port(port)
+ .database(databaseName)
+ .tableList(tableName)
+ .username(username)
+ .password(password)
+ .debeziumProperties(debeziumProperties)
+ .startupOptions(startupOptions)
+ .deserializer(schema)
+ .build();
return env.addSource(sqlServerSource, "SqlServer Source");
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java
index 9f8a4a0..abab9f6 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.sqlserver;
import com.ververica.cdc.connectors.shaded.org.apache.kafka.connect.data.SchemaBuilder;
@@ -34,11 +35,12 @@ public class SqlServerDateConverter implements CustomConverter<SchemaBuilder, Re
private DateTimeFormatter dateFormatter = DateTimeFormatter.ISO_DATE;
private DateTimeFormatter timestampFormatter = DateTimeFormatter.ISO_DATE_TIME;
- public static Properties DEFAULT_PROPS = new Properties();
+ public static final Properties DEFAULT_PROPS = new Properties();
static {
DEFAULT_PROPS.setProperty("converters", "date");
- DEFAULT_PROPS.setProperty("date.type", "org.apache.doris.flink.tools.cdc.sqlserver.SqlServerDateConverter");
+ DEFAULT_PROPS.setProperty(
+ "date.type", "org.apache.doris.flink.tools.cdc.sqlserver.SqlServerDateConverter");
DEFAULT_PROPS.setProperty("date.format.date", "yyyy-MM-dd");
DEFAULT_PROPS.setProperty("date.format.timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS");
}
@@ -46,7 +48,10 @@ public class SqlServerDateConverter implements CustomConverter<SchemaBuilder, Re
@Override
public void configure(Properties props) {
readProps(props, "format.date", p -> dateFormatter = DateTimeFormatter.ofPattern(p));
- readProps(props, "format.timestamp", p -> timestampFormatter = DateTimeFormatter.ofPattern(p));
+ readProps(
+ props,
+ "format.timestamp",
+ p -> timestampFormatter = DateTimeFormatter.ofPattern(p));
}
private void readProps(Properties properties, String settingKey, Consumer<String> callback) {
@@ -63,7 +68,9 @@ public class SqlServerDateConverter implements CustomConverter<SchemaBuilder, Re
}
@Override
- public void converterFor(RelationalColumn column, CustomConverter.ConverterRegistration<SchemaBuilder> registration) {
+ public void converterFor(
+ RelationalColumn column,
+ CustomConverter.ConverterRegistration<SchemaBuilder> registration) {
String sqlType = column.typeName().toUpperCase();
SchemaBuilder schemaBuilder = null;
CustomConverter.Converter converter = null;
@@ -71,7 +78,9 @@ public class SqlServerDateConverter implements CustomConverter<SchemaBuilder, Re
schemaBuilder = SchemaBuilder.string().optional();
converter = this::convertDate;
}
- if ("SMALLDATETIME".equals(sqlType) || "DATETIME".equals(sqlType) || "DATETIME2".equals(sqlType)) {
+ if ("SMALLDATETIME".equals(sqlType)
+ || "DATETIME".equals(sqlType)
+ || "DATETIME2".equals(sqlType)) {
schemaBuilder = SchemaBuilder.string().optional();
converter = this::convertDateTime;
}
@@ -88,10 +97,9 @@ public class SqlServerDateConverter implements CustomConverter<SchemaBuilder, Re
}
private String convertDate(Object input) {
- if (input instanceof Date){
+ if (input instanceof Date) {
return dateFormatter.format(((Date) input).toLocalDate());
}
return null;
}
-
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java
index 36ecb42..ce060ea 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.sqlserver;
import org.apache.doris.flink.tools.cdc.SourceSchema;
@@ -22,8 +23,14 @@ import java.sql.DatabaseMetaData;
public class SqlServerSchema extends SourceSchema {
- public SqlServerSchema(DatabaseMetaData metaData, String databaseName, String schemaName, String tableName, String tableComment) throws Exception {
- super(metaData, databaseName, schemaName, tableName, tableComment);
+ public SqlServerSchema(
+ DatabaseMetaData metaData,
+ String databaseName,
+ String schemaName,
+ String tableName,
+ String tableComment)
+ throws Exception {
+ super(metaData, databaseName, schemaName, tableName, tableComment);
}
@Override
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java
index f2895c6..f09eac1 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java
+++ b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package org.apache.doris.flink.tools.cdc.sqlserver;
import org.apache.doris.flink.catalog.doris.DorisType;
@@ -52,7 +53,7 @@ public class SqlServerType {
// For sqlserver IDENTITY type, such as 'INT IDENTITY'
// originSqlServerType is "int identity", so we only get "int".
String sqlServerType = originSqlServerType.split(" ")[0];
- switch (sqlServerType){
+ switch (sqlServerType) {
case BIT:
return DorisType.BOOLEAN;
case TINYINT:
@@ -74,19 +75,26 @@ public class SqlServerType {
case DECIMAL:
case NUMERIC:
return precision != null && precision > 0 && precision <= 38
- ? String.format("%s(%s,%s)", DorisType.DECIMAL_V3, precision, scale != null && scale >= 0 ? scale : 0)
+ ? String.format(
+ "%s(%s,%s)",
+ DorisType.DECIMAL_V3,
+ precision,
+ scale != null && scale >= 0 ? scale : 0)
: DorisType.STRING;
case DATE:
return DorisType.DATE_V2;
case DATETIME:
case DATETIME2:
case SMALLDATETIME:
- return String.format("%s(%s)", DorisType.DATETIME_V2, Math.min(scale == null ? 0 : scale, 6));
+ return String.format(
+ "%s(%s)", DorisType.DATETIME_V2, Math.min(scale == null ? 0 : scale, 6));
case CHAR:
case VARCHAR:
case NCHAR:
case NVARCHAR:
- return precision * 3 > 65533 ? DorisType.STRING : String.format("%s(%s)", DorisType.VARCHAR, precision * 3);
+ return precision * 3 > 65533
+ ? DorisType.STRING
+ : String.format("%s(%s)", DorisType.VARCHAR, precision * 3);
case TEXT:
case NTEXT:
case TIME:
@@ -94,7 +102,8 @@ public class SqlServerType {
case TIMESTAMP:
return DorisType.STRING;
default:
- throw new UnsupportedOperationException("Unsupported SqlServer Type: " + sqlServerType);
+ throw new UnsupportedOperationException(
+ "Unsupported SqlServer Type: " + sqlServerType);
}
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java
index 2e4a9ea..8ad4130 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java
@@ -17,8 +17,7 @@
package org.apache.doris.flink;
-import org.apache.flink.shaded.guava30.com.google.common.collect.Lists;
-
+import com.google.common.collect.Lists;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/CatalogTest.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/CatalogTest.java
index 6e89959..e93dfdd 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/CatalogTest.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/CatalogTest.java
@@ -37,8 +37,7 @@ import org.apache.flink.types.Row;
import org.apache.flink.types.RowKind;
import org.apache.flink.util.CollectionUtil;
-import org.apache.flink.shaded.guava30.com.google.common.collect.Lists;
-
+import com.google.common.collect.Lists;
import org.apache.doris.flink.cfg.DorisConnectionOptions;
import org.junit.Before;
import org.junit.Ignore;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
index c31112a..6c8adfe 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
@@ -19,9 +19,8 @@ package org.apache.doris.flink.serialization;
import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableList;
-import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableMap;
-
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java
index caca9d0..f084a84 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java
@@ -17,8 +17,7 @@
package org.apache.doris.flink.sink.schema;
-import org.apache.flink.shaded.guava30.com.google.common.collect.Maps;
-
+import com.google.common.collect.Maps;
import org.apache.doris.flink.catalog.doris.FieldSchema;
import org.junit.Assert;
import org.junit.Before;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java
index 84cc699..7da3a30 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java
@@ -17,11 +17,10 @@
package org.apache.doris.flink.sink.writer;
-import org.apache.flink.shaded.guava30.com.google.common.collect.Maps;
-
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Maps;
import org.apache.commons.collections.CollectionUtils;
import org.apache.doris.flink.catalog.doris.FieldSchema;
import org.apache.doris.flink.catalog.doris.TableSchema;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataLookupFunctionTest.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataLookupFunctionTest.java
index 9ad084c..423704f 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataLookupFunctionTest.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataLookupFunctionTest.java
@@ -24,8 +24,7 @@ import org.apache.flink.table.data.StringData;
import org.apache.flink.table.types.DataType;
import org.apache.flink.util.Collector;
-import org.apache.flink.shaded.guava30.com.google.common.cache.Cache;
-
+import com.google.common.cache.Cache;
import org.apache.doris.flink.cfg.DorisLookupOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java
index d4d07cd..342e8d1 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java
@@ -47,7 +47,8 @@ public class DateToStringConverter implements CustomConverter<SchemaBuilder, Rel
static {
DEFAULT_PROPS.setProperty("converters", "date");
- DEFAULT_PROPS.setProperty("date.type", "org.apache.doris.flink.utils.DateToStringConverter");
+ DEFAULT_PROPS.setProperty(
+ "date.type", "org.apache.doris.flink.utils.DateToStringConverter");
DEFAULT_PROPS.setProperty("date.format.date", "yyyy-MM-dd");
DEFAULT_PROPS.setProperty("date.format.datetime", "yyyy-MM-dd HH:mm:ss");
DEFAULT_PROPS.setProperty("date.format.timestamp", "yyyy-MM-dd HH:mm:ss");
diff --git a/tools/maven/checkstyle.xml b/tools/maven/checkstyle.xml
index 86cc1c9..a3d8fa3 100644
--- a/tools/maven/checkstyle.xml
+++ b/tools/maven/checkstyle.xml
@@ -201,10 +201,10 @@ This file is based on the checkstyle file of Apache Beam.
<property name="illegalPkgs" value="io.netty"/>
<message key="import.illegal" value="{0}; Use flink-shaded-netty instead."/>
</module>
- <module name="IllegalImport">
- <property name="illegalPkgs" value="com.google.common"/>
- <message key="import.illegal" value="{0}; Use flink-shaded-guava instead."/>
- </module>
+<!-- <module name="IllegalImport">-->
+<!-- <property name="illegalPkgs" value="com.google.common"/>-->
+<!-- <message key="import.illegal" value="{0}; Use flink-shaded-guava instead."/>-->
+<!-- </module>-->
<module name="RedundantModifier">
<!-- Checks for redundant modifiers on various symbol definitions.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org