You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/05/06 10:39:17 UTC
[incubator-doris] branch master updated: [feature][datax]doriswriter support timeZone (#9327)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new c1707ca388 [feature][datax]doriswriter support timeZone (#9327)
c1707ca388 is described below
commit c1707ca38873e6534e614323e45c20d15a3aae7c
Author: Lonre Wang <lo...@gmail.com>
AuthorDate: Fri May 6 18:39:10 2022 +0800
[feature][datax]doriswriter support timeZone (#9327)
---
extension/DataX/doriswriter/doc/doriswriter.md | 6 ++++++
extension/DataX/doriswriter/doc/mysql2doris.json | 1 +
.../alibaba/datax/plugin/writer/doriswriter/DorisCodec.java | 11 ++++++-----
.../datax/plugin/writer/doriswriter/DorisJsonCodec.java | 4 ++--
.../alibaba/datax/plugin/writer/doriswriter/DorisWriter.java | 2 +-
.../java/com/alibaba/datax/plugin/writer/doriswriter/Key.java | 6 ++++++
.../doriswriter/src/main/resources/plugin_job_template.json | 1 +
7 files changed, 23 insertions(+), 8 deletions(-)
diff --git a/extension/DataX/doriswriter/doc/doriswriter.md b/extension/DataX/doriswriter/doc/doriswriter.md
index a8783909e7..777e441bae 100644
--- a/extension/DataX/doriswriter/doc/doriswriter.md
+++ b/extension/DataX/doriswriter/doc/doriswriter.md
@@ -127,6 +127,12 @@ DorisWriter 通过Doris原生支持Stream load方式导入数据， DorisWriter
- 必选：是
- 默认值：否
+* **timeZone**
+
+ - 描述：Doris 的时区。
+ - 必选：否
+ - 默认值：`+08:00`
+
* **preSql**
- 描述：写入数据到目的表前，会先执行这里的标准语句。
diff --git a/extension/DataX/doriswriter/doc/mysql2doris.json b/extension/DataX/doriswriter/doc/mysql2doris.json
index f6e9d9bda5..1422c385c5 100644
--- a/extension/DataX/doriswriter/doc/mysql2doris.json
+++ b/extension/DataX/doriswriter/doc/mysql2doris.json
@@ -27,6 +27,7 @@
"database": "db1",
"table": "t3",
"column": ["k1", "k2", "k3"],
+ "timeZone": "+08:00",
"username": "root",
"password": "12345",
"postSql": [],
diff --git a/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodec.java b/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodec.java
index 9a364f4038..77a34266b4 100644
--- a/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodec.java
+++ b/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodec.java
@@ -20,18 +20,19 @@ package com.alibaba.datax.plugin.writer.doriswriter;
import com.alibaba.datax.common.element.Column;
import com.alibaba.datax.common.element.DateColumn;
import com.alibaba.datax.common.element.Record;
+import java.time.ZoneId;
import org.apache.commons.lang3.time.DateFormatUtils;
import java.util.List;
import java.util.TimeZone;
public abstract class DorisCodec {
- protected static String timeZone = "GMT+8";
- protected static TimeZone timeZoner = TimeZone.getTimeZone(timeZone);
+ protected final TimeZone timeZone;
protected final List<String> fieldNames;
- public DorisCodec(final List<String> fieldNames) {
+ public DorisCodec(final List<String> fieldNames, final String timeZone) {
this.fieldNames = fieldNames;
+ this.timeZone = TimeZone.getTimeZone(ZoneId.of(timeZone));
}
public abstract String serialize(Record row);
@@ -60,9 +61,9 @@ public abstract class DorisCodec {
final DateColumn.DateType dateType = ((DateColumn) col).getSubType();
switch (dateType) {
case DATE:
- return DateFormatUtils.format(col.asDate(), "yyyy-MM-dd", timeZoner);
+ return DateFormatUtils.format(col.asDate(), "yyyy-MM-dd", timeZone);
case DATETIME:
- return DateFormatUtils.format(col.asDate(), "yyyy-MM-dd HH:mm:ss", timeZoner);
+ return DateFormatUtils.format(col.asDate(), "yyyy-MM-dd HH:mm:ss", timeZone);
default:
return col.asString();
}
diff --git a/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonCodec.java b/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonCodec.java
index 8d4568c2d2..664b9d3629 100644
--- a/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonCodec.java
+++ b/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonCodec.java
@@ -28,8 +28,8 @@ import java.util.Map;
public class DorisJsonCodec extends DorisCodec {
private Map<String, Object> rowMap;
- public DorisJsonCodec(final List<String> fieldNames) {
- super(fieldNames);
+ public DorisJsonCodec(final List<String> fieldNames, final String timeZone) {
+ super(fieldNames, timeZone);
this.rowMap = new HashMap<>(this.fieldNames.size());
}
diff --git a/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriter.java b/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriter.java
index 71e2d1ad7f..94a86e82dd 100644
--- a/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriter.java
+++ b/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriter.java
@@ -55,7 +55,7 @@ public class DorisWriter extends Writer {
@Override
public void init() {
this.keys = new Key(super.getPluginJobConf());
- this.rowCodec = new DorisJsonCodec(this.keys.getColumns());
+ this.rowCodec = new DorisJsonCodec(this.keys.getColumns(), this.keys.getTimeZone());
this.dorisWriterEmitter = new DorisWriterEmitter(keys);
}
diff --git a/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/Key.java b/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/Key.java
index cfbef96c5e..fbf892d3ad 100644
--- a/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/Key.java
+++ b/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/Key.java
@@ -34,6 +34,7 @@ public class Key implements Serializable {
public static final String DATABASE = "database";
public static final String TABLE = "table";
public static final String COLUMN = "column";
+ public static final String TIME_ZONE = "timeZone";
public static final String USERNAME = "username";
public static final String PASSWORD = "password";
@@ -54,6 +55,7 @@ public class Key implements Serializable {
private static final long DEFAULT_MAX_BATCH_BYTE_SIZE = 100 * 1024 * 1024; // 100MB
private static final String DEFAULT_LABEL_PREFIX = "datax_doris_writer_";
private static final String DEFAULT_LINE_DELIMITER = "\n";
+ private static final String DEFAULT_TIME_ZONE = "+08:00";
private static final int DEFAULT_CONNECT_TIMEOUT = -1;
public Key(final Configuration options) {
@@ -98,6 +100,10 @@ public class Key implements Serializable {
return this.options.getList(COLUMN, String.class);
}
+ public String getTimeZone() {
+ return this.options.getString(TIME_ZONE, DEFAULT_TIME_ZONE);
+ }
+
public List<String> getPreSqlList() {
return this.options.getList(PRE_SQL, String.class);
}
diff --git a/extension/DataX/doriswriter/src/main/resources/plugin_job_template.json b/extension/DataX/doriswriter/src/main/resources/plugin_job_template.json
index 152f1eeefd..9cd9bb18b6 100644
--- a/extension/DataX/doriswriter/src/main/resources/plugin_job_template.json
+++ b/extension/DataX/doriswriter/src/main/resources/plugin_job_template.json
@@ -6,6 +6,7 @@
"database": "",
"table": "",
"column": [],
+ "timeZone": "",
"preSql": [],
"postSql": [],
"jdbcUrl": "",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org