You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2019/12/19 23:45:39 UTC
[incubator-doris] branch master updated: Timezone variable support
one digital time (#2513)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 11b7800 Timezone variable support one digital time (#2513)
11b7800 is described below
commit 11b78008cdee3582de30edb4e925e251878b3f02
Author: HangyuanLiu <46...@qq.com>
AuthorDate: Fri Dec 20 07:45:29 2019 +0800
Timezone variable support one digital time (#2513)
Support time zone variable like "-8:00","+8:00","8:00"
Time zone variable like "-8:00" is illegal in time-zone ID ,so we mush transfer it to standard format
---
.../apache/doris/analysis/CreateRoutineLoadStmt.java | 3 +--
.../java/org/apache/doris/analysis/LoadStmt.java | 3 ++-
.../java/org/apache/doris/common/util/TimeUtils.java | 14 ++++++++++++--
.../main/java/org/apache/doris/qe/VariableMgr.java | 5 ++++-
.../java/org/apache/doris/task/StreamLoadTask.java | 3 +--
.../doris/analysis/CreateRoutineLoadStmtTest.java | 3 +++
.../org/apache/doris/common/util/TimeUtilsTest.java | 20 ++++++++++++++++++++
.../java/org/apache/doris/qe/VariableMgrTest.java | 12 ++++++++++++
8 files changed, 55 insertions(+), 8 deletions(-)
diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
index 06a51ae..9e83d41 100644
--- a/fe/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
+++ b/fe/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
@@ -328,8 +328,7 @@ public class CreateRoutineLoadStmt extends DdlStmt {
if (ConnectContext.get() != null) {
timezone = ConnectContext.get().getSessionVariable().getTimeZone();
}
- timezone = jobProperties.getOrDefault(LoadStmt.TIMEZONE, timezone);
- TimeUtils.checkTimeZoneValid(timezone);
+ timezone = TimeUtils.checkTimeZoneValidAndStandardize(jobProperties.getOrDefault(LoadStmt.TIMEZONE, timezone));
}
private void checkDataSourceProperties() throws AnalysisException {
diff --git a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java
index 957e0df..a423d9c 100644
--- a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java
+++ b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java
@@ -201,7 +201,8 @@ public class LoadStmt extends DdlStmt {
// time zone
final String timezone = properties.get(TIMEZONE);
if (timezone != null) {
- TimeUtils.checkTimeZoneValid(timezone);
+ properties.put(TIMEZONE, TimeUtils.checkTimeZoneValidAndStandardize(
+ properties.getOrDefault(LoadStmt.TIMEZONE, TimeUtils.DEFAULT_TIME_ZONE)));
}
}
diff --git a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
index 571ca2b..37e1c4f 100644
--- a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
+++ b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
@@ -70,7 +70,7 @@ public class TimeUtils {
+ "((0?[1-9])|([1-2][0-9])|(30)))|(0?2[\\-\\/\\s]?((0?[1-9])|(1[0-9])|(2[0-8]))))))"
+ "(\\s(((0?[0-9])|([1][0-9])|([2][0-3]))\\:([0-5]?[0-9])((\\s)|(\\:([0-5]?[0-9])))))?$");
- private static final Pattern TIMEZONE_OFFSET_FORMAT_REG = Pattern.compile("^[+-]{1}\\d{2}\\:\\d{2}$");
+ private static final Pattern TIMEZONE_OFFSET_FORMAT_REG = Pattern.compile("^[+-]{0,1}\\d{1,2}\\:\\d{2}$");
public static Date MIN_DATE = null;
public static Date MAX_DATE = null;
@@ -235,8 +235,11 @@ public class TimeUtils {
}
// Check if the time zone_value is valid
- public static void checkTimeZoneValid(String value) throws DdlException {
+ public static String checkTimeZoneValidAndStandardize(String value) throws DdlException {
try {
+ if (value == null) {
+ ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TIME_ZONE, "null");
+ }
// match offset type, such as +08:00, -07:00
Matcher matcher = TIMEZONE_OFFSET_FORMAT_REG.matcher(value);
// it supports offset and region timezone type, "CST" use here is compatibility purposes.
@@ -245,6 +248,11 @@ public class TimeUtils {
ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TIME_ZONE, value);
}
if (match) {
+ boolean postive = value.charAt(0) != '-';
+ value = (postive ? "+" : "-") + String.format("%02d:%02d",
+ Integer.parseInt(value.replaceAll("[+-]", "").split(":")[0]),
+ Integer.parseInt(value.replaceAll("[+-]", "").split(":")[1]));
+
// timezone offsets around the world extended from -12:00 to +14:00
int tz = Integer.parseInt(value.substring(1, 3)) * 100 + Integer.parseInt(value.substring(4, 6));
if (value.charAt(0) == '-' && tz > 1200) {
@@ -254,8 +262,10 @@ public class TimeUtils {
}
}
ZoneId.of(value, timeZoneAliasMap);
+ return value;
} catch (DateTimeException ex) {
ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TIME_ZONE, value);
}
+ throw new DdlException("Parse time zone " + value + " error");
}
}
diff --git a/fe/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/src/main/java/org/apache/doris/qe/VariableMgr.java
index a9fdd2a..4bc821a 100644
--- a/fe/src/main/java/org/apache/doris/qe/VariableMgr.java
+++ b/fe/src/main/java/org/apache/doris/qe/VariableMgr.java
@@ -19,6 +19,7 @@ package org.apache.doris.qe;
import org.apache.doris.analysis.SetType;
import org.apache.doris.analysis.SetVar;
+import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.analysis.SysVariableDesc;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Type;
@@ -215,7 +216,9 @@ public class VariableMgr {
checkUpdate(setVar, ctx.getFlag());
// Check variable time_zone value is valid
if (setVar.getVariable().toLowerCase().equals("time_zone")) {
- TimeUtils.checkTimeZoneValid(setVar.getValue().getStringValue());
+ setVar = new SetVar(
+ setVar.getType(), setVar.getVariable(),
+ new StringLiteral(TimeUtils.checkTimeZoneValidAndStandardize(setVar.getValue().getStringValue())));
}
// To modify to default value.
diff --git a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java
index 02b0fd9..3afa2b3 100644
--- a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java
+++ b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java
@@ -160,8 +160,7 @@ public class StreamLoadTask {
strictMode = request.isStrictMode();
}
if (request.isSetTimezone()) {
- timezone = request.getTimezone();
- TimeUtils.checkTimeZoneValid(timezone);
+ timezone = TimeUtils.checkTimeZoneValidAndStandardize(request.getTimezone());
}
if (request.isSetExecMemLimit()) {
execMemLimit = request.getExecMemLimit();
diff --git a/fe/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java b/fe/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java
index 574f515..3336079 100644
--- a/fe/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java
+++ b/fe/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java
@@ -96,6 +96,7 @@ public class CreateRoutineLoadStmtTest {
String topicName = "topic1";
String serverAddress = "127.0.0.1:8080";
String kafkaPartitionString = "1,2,3";
+ String timeZone = "8:00";
List<String> partitionNameString = Lists.newArrayList();
partitionNameString.add("p1");
PartitionNames partitionNames = new PartitionNames(partitionNameString);
@@ -108,6 +109,7 @@ public class CreateRoutineLoadStmtTest {
loadPropertyList.add(partitionNames);
Map<String, String> properties = Maps.newHashMap();
properties.put(CreateRoutineLoadStmt.DESIRED_CONCURRENT_NUMBER_PROPERTY, "2");
+ properties.put(LoadStmt.TIMEZONE, timeZone);
String typeName = LoadDataSourceType.KAFKA.name();
Map<String, String> customProperties = Maps.newHashMap();
@@ -134,6 +136,7 @@ public class CreateRoutineLoadStmtTest {
Assert.assertEquals(0, createRoutineLoadStmt.getMaxErrorNum());
Assert.assertEquals(serverAddress, createRoutineLoadStmt.getKafkaBrokerList());
Assert.assertEquals(topicName, createRoutineLoadStmt.getKafkaTopic());
+ Assert.assertEquals("+08:00", createRoutineLoadStmt.getTimezone());
}
}
diff --git a/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java b/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
index 1cbabec..d3d849f 100644
--- a/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
+++ b/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
@@ -22,6 +22,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
import org.junit.Assert;
import org.junit.Test;
@@ -134,4 +135,23 @@ public class TimeUtilsTest {
Assert.assertEquals(20150301120000L, datetime.getRealValue());
}
+ @Test
+ public void testTimezone() throws AnalysisException {
+ try {
+ Assert.assertEquals("CST", TimeUtils.checkTimeZoneValidAndStandardize("CST"));
+ Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("+08:00"));
+ Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("+8:00"));
+ Assert.assertEquals("-08:00", TimeUtils.checkTimeZoneValidAndStandardize("-8:00"));
+ Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("8:00"));
+ } catch (DdlException ex) {
+ Assert.fail();
+ }
+ try {
+ TimeUtils.checkTimeZoneValidAndStandardize("FOO");
+ Assert.fail();
+ } catch (DdlException ex) {
+ Assert.assertTrue(ex.getMessage().contains("Unknown or incorrect time zone: 'FOO'"));
+ }
+ }
+
}
diff --git a/fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java b/fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java
index 2ee5ee8..b8d56fc 100644
--- a/fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java
+++ b/fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java
@@ -124,6 +124,18 @@ public class VariableMgrTest {
VariableMgr.setVar(var, setVar4);
Assert.assertEquals(2L, var.getSqlMode());
+ // Test checkTimeZoneValidAndStandardize
+ SetVar setVar5 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("+8:00"));
+ VariableMgr.setVar(var, setVar5);
+ Assert.assertEquals("+08:00", VariableMgr.newSessionVariable().getTimeZone());
+
+ SetVar setVar6 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("8:00"));
+ VariableMgr.setVar(var, setVar6);
+ Assert.assertEquals("+08:00", VariableMgr.newSessionVariable().getTimeZone());
+
+ SetVar setVar7 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("-8:00"));
+ VariableMgr.setVar(var, setVar7);
+ Assert.assertEquals("-08:00", VariableMgr.newSessionVariable().getTimeZone());
}
@Test(expected = DdlException.class)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org