You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2020/02/02 09:13:48 UTC
[kylin] branch master updated (33c9f75 -> d018b04)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 33c9f75 Minor, add class TableDesc.TableProject to avoid the use of Pair
new 609ce65 KYLIN-4365 fix RDBMS pushdown with clause bug
new d018b04 KYLIN-4365 Fix sql starts with "with" cannot be queried on web
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/kylin/query/util/PushDownUtil.java | 27 +++++++++++++++------
.../org/apache/kylin/query/util/QueryUtil.java | 28 +++++++++++-----------
.../apache/kylin/query/util/PushDownUtilTest.java | 7 ++++++
3 files changed, 41 insertions(+), 21 deletions(-)
[kylin] 01/02: KYLIN-4365 fix RDBMS pushdown with clause bug
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 609ce65d5bfa18ee4021fde3d96c0b8b22c5382e
Author: woyumen4597 <wo...@gmail.com>
AuthorDate: Sat Feb 1 13:56:53 2020 +0800
KYLIN-4365 fix RDBMS pushdown with clause bug
---
.../org/apache/kylin/query/util/PushDownUtil.java | 27 ++++++++++++++++------
.../apache/kylin/query/util/PushDownUtilTest.java | 7 ++++++
2 files changed, 27 insertions(+), 7 deletions(-)
diff --git a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 4d69272..2167fdd 100644
--- a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -24,7 +24,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
-
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
@@ -41,16 +40,14 @@ import org.apache.calcite.sql.SqlWith;
import org.apache.calcite.sql.SqlWithItem;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.util.SqlVisitor;
-
import org.apache.commons.lang.text.StrBuilder;
-
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
public class PushDownUtil {
private static final Logger logger = LoggerFactory.getLogger(PushDownUtil.class);
@@ -103,8 +100,7 @@ public class PushDownUtil {
StrBuilder afterConvert = new StrBuilder(inputSql);
for (Pair<Integer, Integer> pos : tablesPos) {
- String tableWithSchema = schema + "." + inputSql.substring(pos.getFirst(),
- pos.getSecond());
+ String tableWithSchema = schema + "." + inputSql.substring(pos.getFirst(), pos.getSecond());
afterConvert.replace(pos.getFirst(), pos.getSecond(), tableWithSchema);
}
return afterConvert.toString();
@@ -116,13 +112,26 @@ public class PushDownUtil {
*/
static class FromTablesVisitor implements SqlVisitor<SqlNode> {
private List<SqlNode> tables;
+ private List<SqlNode> withTables;
FromTablesVisitor() {
this.tables = new ArrayList<>();
+ this.withTables = new ArrayList<>();
}
List<SqlNode> getTablesWithoutSchema() {
- return tables;
+ List<SqlNode> sqlNodes = Lists.newArrayList();
+ List<String> withs = Lists.newArrayList();
+ for (SqlNode withTable : withTables) {
+ withs.add(((SqlIdentifier) withTable).names.get(0)); // with clause not allow database.table pattern
+ }
+ for (SqlNode table : tables) {
+ SqlIdentifier identifier = (SqlIdentifier) table;
+ if (!withs.contains(identifier.names.get(0))) {
+ sqlNodes.add(identifier);
+ }
+ }
+ return sqlNodes;
}
@Override
@@ -156,6 +165,10 @@ public class PushDownUtil {
}
if (call instanceof SqlWith) {
SqlWith sqlWith = (SqlWith) call;
+ List<SqlNode> list = sqlWith.withList.getList();
+ for (SqlNode sqlNode : list) {
+ withTables.add(((SqlWithItem) sqlNode).name);
+ }
sqlWith.body.accept(this);
sqlWith.withList.accept(this);
}
diff --git a/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java b/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
index f23010e..81fb68b 100644
--- a/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
@@ -124,4 +124,11 @@ public class PushDownUtilTest {
"ORDER BY c_customer_id limit 5"; //
Assert.assertEquals(expected, PushDownUtil.schemaCompletion(ori, "EDW"));
}
+
+ @Test
+ public void testWithSyntax2() throws SqlParseException {
+ String origin = "with tmp as (select id from a),tmp2 as (select * from b) select * from tmp,tmp2 where tmp.id = tmp2.id";
+ String expected = "with tmp as (select id from ssb.a),tmp2 as (select * from ssb.b) select * from tmp,tmp2 where tmp.id = tmp2.id";
+ Assert.assertEquals(expected, PushDownUtil.schemaCompletion(origin, "ssb"));
+ }
}
[kylin] 02/02: KYLIN-4365 Fix sql starts with "with" cannot be
queried on web
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d018b046a29bce12a1ddf3806e65d2e5b22f48bc
Author: nichunen <ni...@apache.org>
AuthorDate: Sun Feb 2 15:01:32 2020 +0800
KYLIN-4365 Fix sql starts with "with" cannot be queried on web
---
.../org/apache/kylin/query/util/QueryUtil.java | 28 +++++++++++-----------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index aa6c345..3d5632c 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -23,6 +23,7 @@ import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -37,26 +38,20 @@ import com.google.common.collect.Lists;
public class QueryUtil {
protected static final Logger logger = LoggerFactory.getLogger(QueryUtil.class);
-
- private QueryUtil() {
- throw new IllegalStateException("Class QueryUtil is an utility class !");
- }
-
- private static List<IQueryTransformer> queryTransformers;
-
- public interface IQueryTransformer {
- String transform(String sql, String project, String defaultSchema);
- }
-
private static final String KEYWORD_SELECT = "select";
private static final String KEYWORD_WITH = "with";
private static final String KEYWORD_EXPLAIN = "explain";
+ private static List<IQueryTransformer> queryTransformers;
+ private QueryUtil() {
+ throw new IllegalStateException("Class QueryUtil is an utility class !");
+ }
public static String appendLimitOffsetToSql(String sql, int limit, int offset) {
String retSql = sql;
String prefixSql = "select * from (";
String suffixSql = ")";
- if (sql.startsWith(KEYWORD_EXPLAIN)) {
+ if (StringUtils.startsWithIgnoreCase(sql, KEYWORD_EXPLAIN)
+ || StringUtils.startsWithIgnoreCase(sql, KEYWORD_WITH)) {
prefixSql = "";
suffixSql = "";
}
@@ -126,7 +121,8 @@ public class QueryUtil {
/**
* add remove catalog step at final
*/
- public static String massageSql(String sql, String project, int limit, int offset, String defaultSchema, String catalog) {
+ public static String massageSql(String sql, String project, int limit, int offset, String defaultSchema,
+ String catalog) {
String correctedSql = massageSql(sql, project, limit, offset, defaultSchema);
correctedSql = removeCatalog(correctedSql, catalog);
return correctedSql;
@@ -219,7 +215,7 @@ public class QueryUtil {
public static String removeCommentInSql(String sql1) {
// match two patterns, one is "-- comment", the other is "/* comment */"
- final String[] commentPatterns = new String[]{"--(?!.*\\*/).*?[\r\n]", "/\\*(.|\r|\n)*?\\*/"};
+ final String[] commentPatterns = new String[] { "--(?!.*\\*/).*?[\r\n]", "/\\*(.|\r|\n)*?\\*/" };
for (int i = 0; i < commentPatterns.length; i++) {
sql1 = sql1.replaceAll(commentPatterns[i], "");
@@ -229,4 +225,8 @@ public class QueryUtil {
return sql1;
}
+
+ public interface IQueryTransformer {
+ String transform(String sql, String project, String defaultSchema);
+ }
}