You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/09/12 07:04:46 UTC
[kylin] branch master updated (1a2f779 -> 1b71a62)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 1a2f779 Kylin 3392 support sum(null)
new d2907d8 Add JoinedFormatter for Dynamic variables
new 1b71a62 KYLIN-4156 Code reivew
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/kylin/job/JoinedFlatTable.java | 5 +-
.../java/org/apache/kylin/job/JoinedFormatter.java | 127 +++++++++++++++++++++
.../org/apache/kylin/job/JoinedFormatterTest.java | 50 ++++----
.../modelDesigner/conditions_settings.html | 1 +
4 files changed, 157 insertions(+), 26 deletions(-)
create mode 100644 core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java
copy core-cube/src/test/java/org/apache/kylin/cube/ProjectSpecificConfigTest.java => core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java (53%)
[kylin] 02/02: KYLIN-4156 Code reivew
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1b71a6259411ac0d69ff460f5ee3c789498873a5
Author: nichunen <ni...@apache.org>
AuthorDate: Thu Sep 12 15:02:57 2019 +0800
KYLIN-4156 Code reivew
---
.../java/org/apache/kylin/job/JoinedFlatTable.java | 2 +-
.../java/org/apache/kylin/job/JoinedFormatter.java | 46 +++++++++++-----------
.../org/apache/kylin/job/JoinedFormatterTest.java | 4 +-
.../modelDesigner/conditions_settings.html | 1 +
4 files changed, 27 insertions(+), 26 deletions(-)
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 5884421..2838d41 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -226,7 +226,7 @@ public class JoinedFlatTable {
DataModelDesc model = flatDesc.getDataModel();
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
JoinedFormatter formatter = new JoinedFormatter(flatDesc);
- String fmtFilterCondition = formatter.formatSentense(model.getFilterCondition());
+ String fmtFilterCondition = formatter.formatSentence(model.getFilterCondition());
String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc, fmtFilterCondition);
whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug
}
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java
index aa7408c..901ee15 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java
@@ -18,13 +18,6 @@
package org.apache.kylin.job;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.SegmentRange;
-
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -33,30 +26,37 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.SegmentRange;
+
/**
* Joined Formatter for JoinedFlatTable
*/
public class JoinedFormatter {
- private static String REG_SEPARATOR = "\\$\\{(?<KEY>.*?)\\}";
- private static Pattern REG_PATTERN = Pattern.compile(REG_SEPARATOR,
+ private static final String REG_SEPARATOR = "\\$\\{(?<KEY>.*?)\\}";
+ private static final Pattern REG_PATTERN = Pattern.compile(REG_SEPARATOR,
Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
//
- public static String START_DATE = "START_DATE";
- public static String END_DATE = "END_DATE";
- public static String ENV_KEY = "KEY";
+ private static final String START_DATE = "START_DATE";
+ private static final String END_DATE = "END_DATE";
+ private static final String ENV_KEY = "KEY";
//
private Map<String, Object> mapEnv = new HashMap<>();
- public JoinedFormatter() {
+ JoinedFormatter() {
}
- public JoinedFormatter(IJoinedFlatTableDesc flatDesc) {
+ JoinedFormatter(IJoinedFlatTableDesc flatDesc) {
setDateEnv(flatDesc);
}
- public void setDateEnv(IJoinedFlatTableDesc flatDesc) {
+ private void setDateEnv(IJoinedFlatTableDesc flatDesc) {
DataModelDesc model = flatDesc.getDataModel();
PartitionDesc partDesc = model.getPartitionDesc();
SegmentRange segRange = flatDesc.getSegRange();
@@ -83,10 +83,10 @@ public class JoinedFormatter {
return value == null ? "" : value;
}
- public String formatSentense(String sentense) {
- String[] cArray = REG_PATTERN.split(sentense);
+ String formatSentence(String sentence) {
+ String[] cArray = REG_PATTERN.split(sentence);
StringBuilder sbr = new StringBuilder();
- List<String> keys = getKeys(sentense);
+ List<String> keys = getKeys(sentence);
int length = Math.max(cArray.length, keys.size());
for (int i = 0; i < length; i++) {
if (i < cArray.length) {
@@ -99,7 +99,7 @@ public class JoinedFormatter {
return sbr.toString();
}
- public List<String> getKeys(String condition) {
+ private List<String> getKeys(String condition) {
List<String> keys = new ArrayList<>();
Matcher matcher = REG_PATTERN.matcher(condition);
while (matcher.find()) {
@@ -108,20 +108,20 @@ public class JoinedFormatter {
return keys;
}
- public void setKeyValue(String key, Object value) {
+ private void setKeyValue(String key, Object value) {
String fmtKey = StringUtils.trimToEmpty(key).toUpperCase(Locale.ROOT);
mapEnv.put(fmtKey, value);
}
- public void setStartDate(String dateStr) {
+ void setStartDate(String dateStr) {
setKeyValue(START_DATE, dateStr);
}
- public void setEndDate(String dateStr) {
+ void setEndDate(String dateStr) {
setKeyValue(END_DATE, dateStr);
}
- public void printEnv() {
+ void printEnv() {
System.out.println(mapEnv);
}
}
diff --git a/core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java b/core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java
index 6077011..a08a12b 100644
--- a/core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java
@@ -48,7 +48,7 @@ public class JoinedFormatterTest extends LocalFileMetadataTestCase {
public void testConditionFormat() {
String expected = "date_str>='20190710' and date_str<'20190711'";
String condition = "date_str>='${start_date}' and date_str<'${end_date}'";
- String fmtCondition = formatter.formatSentense(condition);
+ String fmtCondition = formatter.formatSentence(condition);
assertEquals(expected, fmtCondition);
}
@@ -56,7 +56,7 @@ public class JoinedFormatterTest extends LocalFileMetadataTestCase {
public void testSqlFormat() {
String expected = "select * from table where date_str>=20190710 and date_str<20190711";
String sql = "select * from table where date_str>=${start_date} and date_str<${end_date}";
- String fmtSql = formatter.formatSentense(sql);
+ String fmtSql = formatter.formatSentence(sql);
assertEquals(expected, fmtSql);
}
}
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index 10aee84..becb327 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -192,6 +192,7 @@
<li>Where clause to filter data from source</li>
<li>Do not include date column which will be used for incremental refresh</li>
<li>Do not include "Where"</li>
+ <li>Use ${start_date} and ${end_date} as dynamic variables for segment's start time and end time which are determined after a building job triggered</li>
</ol>
</script>
<script type="text/ng-template" id="separateTimeColumnTip.html">
[kylin] 01/02: Add JoinedFormatter for Dynamic variables
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d2907d8f017c316376d5ad5e0206b49c4dba0118
Author: liuzx32 <li...@163.com>
AuthorDate: Wed Aug 28 14:13:43 2019 +0800
Add JoinedFormatter for Dynamic variables
---
.../java/org/apache/kylin/job/JoinedFlatTable.java | 5 +-
.../java/org/apache/kylin/job/JoinedFormatter.java | 127 +++++++++++++++++++++
.../org/apache/kylin/job/JoinedFormatterTest.java | 62 ++++++++++
3 files changed, 192 insertions(+), 2 deletions(-)
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index f729d59..5884421 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -225,8 +225,9 @@ public class JoinedFlatTable {
DataModelDesc model = flatDesc.getDataModel();
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
- String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc,
- model.getFilterCondition());
+ JoinedFormatter formatter = new JoinedFormatter(flatDesc);
+ String fmtFilterCondition = formatter.formatSentense(model.getFilterCondition());
+ String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc, fmtFilterCondition);
whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug
}
if (flatDesc.getSegment() != null) {
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java
new file mode 100644
index 0000000..aa7408c
--- /dev/null
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.job;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.SegmentRange;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Joined Formatter for JoinedFlatTable
+ */
+
+public class JoinedFormatter {
+
+ private static String REG_SEPARATOR = "\\$\\{(?<KEY>.*?)\\}";
+ private static Pattern REG_PATTERN = Pattern.compile(REG_SEPARATOR,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+ //
+ public static String START_DATE = "START_DATE";
+ public static String END_DATE = "END_DATE";
+ public static String ENV_KEY = "KEY";
+ //
+ private Map<String, Object> mapEnv = new HashMap<>();
+
+ public JoinedFormatter() {
+ }
+
+ public JoinedFormatter(IJoinedFlatTableDesc flatDesc) {
+ setDateEnv(flatDesc);
+ }
+
+ public void setDateEnv(IJoinedFlatTableDesc flatDesc) {
+ DataModelDesc model = flatDesc.getDataModel();
+ PartitionDesc partDesc = model.getPartitionDesc();
+ SegmentRange segRange = flatDesc.getSegRange();
+ long startInclusive = (Long) segRange.start.v;
+ long endExclusive = (Long) segRange.end.v;
+ //
+ String startDate = "";
+ String endDate = "";
+ String partitionColumnDateFormat = partDesc.getPartitionDateFormat();
+ if (StringUtils.isBlank(partitionColumnDateFormat)) {
+ startDate = String.valueOf(startInclusive);
+ endDate = String.valueOf(endExclusive);
+ } else {
+ startDate = DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat);
+ endDate = DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat);
+ }
+ setKeyValue(START_DATE, startDate);
+ setKeyValue(END_DATE, endDate);
+ }
+
+ public Object getValue(String key) {
+ String fmtKey = StringUtils.trimToEmpty(key).toUpperCase(Locale.ROOT);
+ Object value = mapEnv.get(fmtKey);
+ return value == null ? "" : value;
+ }
+
+ public String formatSentense(String sentense) {
+ String[] cArray = REG_PATTERN.split(sentense);
+ StringBuilder sbr = new StringBuilder();
+ List<String> keys = getKeys(sentense);
+ int length = Math.max(cArray.length, keys.size());
+ for (int i = 0; i < length; i++) {
+ if (i < cArray.length) {
+ sbr.append(cArray[i]);
+ }
+ if (i < keys.size()) {
+ sbr.append(getValue(keys.get(i)));
+ }
+ }
+ return sbr.toString();
+ }
+
+ public List<String> getKeys(String condition) {
+ List<String> keys = new ArrayList<>();
+ Matcher matcher = REG_PATTERN.matcher(condition);
+ while (matcher.find()) {
+ keys.add(matcher.group(ENV_KEY));
+ }
+ return keys;
+ }
+
+ public void setKeyValue(String key, Object value) {
+ String fmtKey = StringUtils.trimToEmpty(key).toUpperCase(Locale.ROOT);
+ mapEnv.put(fmtKey, value);
+ }
+
+ public void setStartDate(String dateStr) {
+ setKeyValue(START_DATE, dateStr);
+ }
+
+ public void setEndDate(String dateStr) {
+ setKeyValue(END_DATE, dateStr);
+ }
+
+ public void printEnv() {
+ System.out.println(mapEnv);
+ }
+}
diff --git a/core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java b/core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java
new file mode 100644
index 0000000..6077011
--- /dev/null
+++ b/core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.job;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author Zhixin Liu (liuzx32)
+ */
+public class JoinedFormatterTest extends LocalFileMetadataTestCase {
+
+ private JoinedFormatter formatter;
+
+ @Before
+ public void setUp() throws Exception {
+ formatter = new JoinedFormatter();
+ formatter.setStartDate("20190710");
+ formatter.setEndDate("20190711");
+ }
+
+ @After
+ public void after() throws Exception {
+ formatter.printEnv();
+ }
+
+ @Test
+ public void testConditionFormat() {
+ String expected = "date_str>='20190710' and date_str<'20190711'";
+ String condition = "date_str>='${start_date}' and date_str<'${end_date}'";
+ String fmtCondition = formatter.formatSentense(condition);
+ assertEquals(expected, fmtCondition);
+ }
+
+ @Test
+ public void testSqlFormat() {
+ String expected = "select * from table where date_str>=20190710 and date_str<20190711";
+ String sql = "select * from table where date_str>=${start_date} and date_str<${end_date}";
+ String fmtSql = formatter.formatSentense(sql);
+ assertEquals(expected, fmtSql);
+ }
+}