You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/07/03 07:44:12 UTC
[01/26] kylin git commit: fix ci
Repository: kylin
Updated Branches:
refs/heads/kylin-2.1.x [created] 82b578a56
fix ci
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f7aa456
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f7aa456
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f7aa456
Branch: refs/heads/kylin-2.1.x
Commit: 2f7aa456640885dfde622966a6d487579a117931
Parents: 8603297
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 29 12:34:03 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 13:51:45 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/util/AdHocUtilTest.java | 45 ++++++++++----------
1 file changed, 22 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f7aa456/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
index b93e2d3..724844c 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
@@ -30,36 +30,28 @@ public class AdHocUtilTest {
@Test
public void testReplaceIdentifierInExpr() {
{
- String ret = AdHocUtil.replaceIdentifierInExpr("a.b.x * a.b.y", null, false);
- Assert.assertEquals("b.x * b.y", ret);
+ String ret = AdHocUtil.replaceIdentifierInExpr("x * y", null, false);
+ Assert.assertEquals("x * y", ret);
}
{
- String ret = AdHocUtil.replaceIdentifierInExpr("a_1.b_2.x_3 * a_1.b_2.y_3", null, false);
+ String ret = AdHocUtil.replaceIdentifierInExpr("x_3 * y_3", "b_2", false);
Assert.assertEquals("b_2.x_3 * b_2.y_3", ret);
}
{
- String ret = AdHocUtil.replaceIdentifierInExpr("a.b.x * a.b.y", "c", false);
- Assert.assertEquals("c.x * c.y", ret);
+ String ret = AdHocUtil.replaceIdentifierInExpr("substr(x,1,3)>y", "c", true);
+ Assert.assertEquals("substr(c.x,1,3)>c.y", ret);
}
{
- String ret = AdHocUtil.replaceIdentifierInExpr("a.b.x * a.b.y", "c", true);
- Assert.assertEquals("\"C\".\"X\" * \"C\".\"Y\"", ret);
+ String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", "c", true);
+ Assert.assertEquals("strcmp(substr(c.x,1,3),c.y)", ret);
}
{
- String ret = AdHocUtil.replaceIdentifierInExpr("substr(a.b.x,1,3)>a.b.y", "c", true);
- Assert.assertEquals("substr(\"C\".\"X\",1,3)>\"C\".\"Y\"", ret);
+ String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, true);
+ Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
}
{
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(a.b.x,1,3),a.b.y) > 0", "c", true);
- Assert.assertEquals("strcmp(substr(\"C\".\"X\",1,3),\"C\".\"Y\") > 0", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(a.b.x,1,3),a.b.y) > 0", null, true);
- Assert.assertEquals("strcmp(substr(\"B\".\"X\",1,3),\"B\".\"Y\") > 0", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(a.b.x, 1, 3),a.b.y) > 0", null, false);
- Assert.assertEquals("strcmp(substr(b.x, 1, 3),b.y) > 0", ret);
+ String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, false);
+ Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
}
}
@@ -68,7 +60,7 @@ public class AdHocUtilTest {
ComputedColumnDesc computedColumnDesc = mock(ComputedColumnDesc.class);
when(computedColumnDesc.getColumnName()).thenReturn("DEAL_AMOUNT");
- when(computedColumnDesc.getExpression()).thenReturn("DB.TABLE.price * DB.TABLE.number");
+ when(computedColumnDesc.getExpression()).thenReturn("price * number");
CCInfo ccInfo = mock(CCInfo.class);
when(ccInfo.getComputedColumnDesc()).thenReturn(computedColumnDesc);
@@ -77,22 +69,29 @@ public class AdHocUtilTest {
String ret = AdHocUtil.restoreComputedColumnToExpr(
"select DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
Assert.assertEquals(
- "select (TABLE.price * TABLE.number) from DB.TABLE group by date order by (TABLE.price * TABLE.number)",
+ "select (price * number) from DB.TABLE group by date order by (price * number)",
ret);
}
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
"select DEAL_AMOUNT as DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
Assert.assertEquals(
- "select (TABLE.price * TABLE.number) as DEAL_AMOUNT from DB.TABLE group by date order by (TABLE.price * TABLE.number)",
+ "select (price * number) as DEAL_AMOUNT from DB.TABLE group by date order by (price * number)",
ret);
}
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
"select \"DEAL_AMOUNT\" AS deal_amount from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
Assert.assertEquals(
- "select (\"TABLE\".\"PRICE\" * \"TABLE\".\"NUMBER\") AS deal_amount from DB.TABLE group by date order by (TABLE.price * TABLE.number)",
+ "select (price * number) AS deal_amount from DB.TABLE group by date order by (price * number)",
ret);
}
+ {
+ String ret = AdHocUtil.restoreComputedColumnToExpr(
+ "select x.DEAL_AMOUNT AS deal_amount from DB.TABLE x group by date order by x.DEAL_AMOUNT", ccInfo);
+ Assert.assertEquals(
+ "select (x.price * x.number) AS deal_amount from DB.TABLE x group by date order by (x.price * x.number)",
+ ret);
+ }
}
}
[03/26] kylin git commit: minor, enhance computed column checks
Posted by li...@apache.org.
minor, enhance computed column checks
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d61f867f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d61f867f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d61f867f
Branch: refs/heads/kylin-2.1.x
Commit: d61f867ff9fa170e155310edeb03db6059433f2f
Parents: 69532cd
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jun 28 21:13:25 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 13:51:45 2017 +0800
----------------------------------------------------------------------
core-metadata/pom.xml | 16 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 15 +-
.../metadata/model/ComputedColumnDesc.java | 3 +
.../kylin/metadata/model/DataModelDesc.java | 30 ++-
.../apache/kylin/metadata/model/TblColRef.java | 2 +-
.../metadata/model/tool/CalciteParser.java | 203 +++++++++++++++++++
.../kylin/model/tool/CalciteParserTest.java | 111 ++++++++++
.../model_desc/ci_inner_join_model.json | 6 +-
.../model_desc/ci_left_join_model.json | 6 +-
.../query/util/ConvertToComputedColumn.java | 111 ++--------
.../query/util/ConvertToComputedColumnTest.java | 22 --
11 files changed, 379 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index f5ab12a..d45a838 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -17,7 +17,8 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -60,6 +61,19 @@
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>atopcalcite</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-core</artifactId>
+ </exclusion>
+ </exclusions>
+ <!--set to provided on purpose, so that dependant on metadata module will not depend on calcite unnecessarily-->
+ <!--https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html-->
+ <scope>provided</scope>
+ </dependency>
<!-- Compiled -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 5d15d56..3c84c96 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -19,9 +19,9 @@
package org.apache.kylin.metadata.model;
import java.io.Serializable;
-import java.util.regex.Pattern;
import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
@@ -82,7 +82,8 @@ public class ColumnDesc implements Serializable {
this.index = other.index;
}
- public ColumnDesc(String id, String name, String datatype, String comment, String dataGen, String index, String computedColumnExpr) {
+ public ColumnDesc(String id, String name, String datatype, String comment, String dataGen, String index,
+ String computedColumnExpr) {
this.id = id;
this.name = name;
this.datatype = datatype;
@@ -193,11 +194,12 @@ public class ColumnDesc implements Serializable {
return index;
}
- public String getComputedColumnExpr(String tableAlias, String tableIdentity) {
+ public String getComputedColumnExpr(String tableAlias) {
Preconditions.checkState(computedColumnExpr != null);
+ Preconditions.checkState(tableAlias != null);
- //http://stackoverflow.com/questions/5054995/how-to-replace-case-insensitive-literal-substrings-in-java
- return computedColumnExpr.replaceAll("(?i)" + Pattern.quote(tableIdentity), tableAlias);
+ String s = CalciteParser.insertAliasInExpr(computedColumnExpr, tableAlias);
+ return s;
}
public boolean isComputedColumnn() {
@@ -257,6 +259,7 @@ public class ColumnDesc implements Serializable {
@Override
public String toString() {
- return "ColumnDesc{" + "id='" + id + '\'' + ", name='" + name + '\'' + ", datatype='" + datatype + '\'' + ", comment='" + comment + '\'' + '}';
+ return "ColumnDesc{" + "id='" + id + '\'' + ", name='" + name + '\'' + ", datatype='" + datatype + '\''
+ + ", comment='" + comment + '\'' + '}';
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 013cb8c..540b5fc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class ComputedColumnDesc {
@@ -42,6 +43,8 @@ public class ComputedColumnDesc {
tableIdentity = tableIdentity.toUpperCase();
columnName = columnName.toUpperCase();
+
+ CalciteParser.ensureNoTableNameExists(expression);
}
public String getFullName() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 7de955e..91802f7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -489,17 +489,11 @@ public class DataModelDesc extends RootPersistentEntity {
}
CCInfo other = ccInfoMap.get(thisCCName);
- if (other == null || (other.getDataModelDescs().size() == 1 && other.getDataModelDescs().contains(this))) {
- //check whether two computer columns's definition is the same.
- for (CCInfo sysCCInfo : ccInfoMap.values()) {
- String definition0 = thisCCDesc.getExpression();
- String definition1 = sysCCInfo.getComputedColumnDesc().getExpression();
- if (isTwoCCDefinitionEquals(definition0, definition1)) {
- throw new IllegalStateException(String.format(
- "Computed column %s'definition: %s is already defined in other models: %s. Please change another definition, or try to keep consistent definition",
- thisCCName, definition0, sysCCInfo.getDataModelDescs()));
- }
- }
+
+ if (other == null) {
+ checkSameCCDefinition(ccInfoMap, thisCCDesc, thisCCName);
+ ccInfoMap.put(thisCCName, new CCInfo(thisCCDesc, Sets.<DataModelDesc> newHashSet(this)));
+ } else if (other.getDataModelDescs().size() == 1 && other.getDataModelDescs().contains(this)) {
ccInfoMap.put(thisCCName, new CCInfo(thisCCDesc, Sets.<DataModelDesc> newHashSet(this)));
} else if (other.getComputedColumnDesc().equals(thisCCDesc)) {
other.getDataModelDescs().add(this);
@@ -511,6 +505,20 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
+ private void checkSameCCDefinition(Map<String, CCInfo> ccInfoMap, ComputedColumnDesc thisCCDesc,
+ String thisCCName) {
+ //check whether two computer columns's definition is the same.
+ for (CCInfo sysCCInfo : ccInfoMap.values()) {
+ String definition0 = thisCCDesc.getExpression();
+ String definition1 = sysCCInfo.getComputedColumnDesc().getExpression();
+ if (isTwoCCDefinitionEquals(definition0, definition1)) {
+ throw new IllegalStateException(String.format(
+ "Computed column %s's definition: %s is already defined in other models: %s. Please change another definition, or try to keep consistent definition",
+ thisCCName, definition0, sysCCInfo.getDataModelDescs()));
+ }
+ }
+ }
+
private boolean isTwoCCDefinitionEquals(String definition0, String definition1) {
definition0 = definition0.replaceAll("\\s*", "");
definition1 = definition1.replaceAll("\\s*", "");
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 5a28e8b..5563856 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -158,7 +158,7 @@ public class TblColRef implements Serializable {
if (!column.isComputedColumnn()) {
return getIdentity();
} else {
- return column.getComputedColumnExpr(getTableAlias(), table.getTableIdentity());
+ return column.getComputedColumnExpr(getTableAlias());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
new file mode 100644
index 0000000..47a90bf
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model.tool;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.util.SqlBasicVisitor;
+import org.apache.calcite.sql.util.SqlVisitor;
+import org.apache.commons.lang3.tuple.Pair;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class CalciteParser {
+ public static SqlNode parse(String sql) throws SqlParseException {
+ SqlParser.ConfigBuilder parserBuilder = SqlParser.configBuilder();
+ SqlParser sqlParser = SqlParser.create(sql, parserBuilder.build());
+ return sqlParser.parseQuery();
+ }
+
+ public static SqlNode getOnlySelectNode(String sql) {
+ SqlNodeList selectList = null;
+ try {
+ selectList = ((SqlSelect) CalciteParser.parse(sql)).getSelectList();
+ } catch (SqlParseException e) {
+ throw new RuntimeException("Failed to parse expression \'" + sql
+ + "\', please make sure the expression is valid and no table name exists in the expression");
+ }
+
+ Preconditions.checkArgument(selectList.size() == 1,
+ "Expression is invalid because size of select list exceeds one");
+
+ return selectList.get(0);
+ }
+
+ public static SqlNode getExpNode(String expr) {
+ return getOnlySelectNode("select " + expr + " from t");
+
+ }
+
+ public static boolean isNodeEqual(SqlNode node0, SqlNode node1) {
+ if (node0 == null) {
+ return node1 == null;
+ } else if (node1 == null) {
+ return false;
+ }
+
+ if (!Objects.equals(node0.getClass().getSimpleName(), node1.getClass().getSimpleName())) {
+ return false;
+ }
+
+ if (node0 instanceof SqlCall) {
+ SqlCall thisNode = (SqlCall) node0;
+ SqlCall thatNode = (SqlCall) node1;
+ if (!thisNode.getOperator().getName().equalsIgnoreCase(thatNode.getOperator().getName())) {
+ return false;
+ }
+ return isNodeEqual(thisNode.getOperandList(), thatNode.getOperandList());
+ }
+ if (node0 instanceof SqlLiteral) {
+ SqlLiteral thisNode = (SqlLiteral) node0;
+ SqlLiteral thatNode = (SqlLiteral) node1;
+ return Objects.equals(thisNode.getValue(), thatNode.getValue());
+ }
+ if (node0 instanceof SqlNodeList) {
+ SqlNodeList thisNode = (SqlNodeList) node0;
+ SqlNodeList thatNode = (SqlNodeList) node1;
+ if (thisNode.getList().size() != thatNode.getList().size()) {
+ return false;
+ }
+ for (int i = 0; i < thisNode.getList().size(); i++) {
+ SqlNode thisChild = thisNode.getList().get(i);
+ final SqlNode thatChild = thatNode.getList().get(i);
+ if (!isNodeEqual(thisChild, thatChild)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ if (node0 instanceof SqlIdentifier) {
+ SqlIdentifier thisNode = (SqlIdentifier) node0;
+ SqlIdentifier thatNode = (SqlIdentifier) node1;
+ // compare ignore table alias.eg: expression like "a.b + a.c + a.d" ,alias a will be ignored when compared
+ String name0 = thisNode.names.get(thisNode.names.size() - 1).replace("\"", "");
+ String name1 = thatNode.names.get(thatNode.names.size() - 1).replace("\"", "");
+ return name0.equalsIgnoreCase(name1);
+ }
+
+ return false;
+ }
+
+ private static boolean isNodeEqual(List<SqlNode> operands0, List<SqlNode> operands1) {
+ if (operands0.size() != operands1.size()) {
+ return false;
+ }
+ for (int i = 0; i < operands0.size(); i++) {
+ if (!isNodeEqual(operands0.get(i), operands1.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static void ensureNoTableNameExists(String expr) {
+ SqlNode sqlNode = getExpNode(expr);
+
+ SqlVisitor sqlVisitor = new SqlBasicVisitor() {
+ @Override
+ public Object visit(SqlIdentifier id) {
+ if (id.names.size() > 1) {
+ throw new IllegalArgumentException("SqlIdentifier " + id + " contains DB/Table name");
+ }
+ return null;
+ }
+ };
+
+ sqlNode.accept(sqlVisitor);
+ }
+
+ public static String insertAliasInExpr(String expr, String alias) {
+ String prefix = "select ";
+ String suffix = " from t";
+ String sql = prefix + expr + suffix;
+ SqlNode sqlNode = getOnlySelectNode(sql);
+
+ final List<SqlIdentifier> sqlIdentifiers = Lists.newArrayList();
+ SqlVisitor sqlVisitor = new SqlBasicVisitor() {
+ @Override
+ public Object visit(SqlIdentifier id) {
+ if (id.names.size() > 1) {
+ throw new IllegalArgumentException("SqlIdentifier " + id + " contains DB/Table name");
+ }
+ sqlIdentifiers.add(id);
+ return null;
+ }
+ };
+
+ sqlNode.accept(sqlVisitor);
+
+ Collections.sort(sqlIdentifiers, new Comparator<SqlIdentifier>() {
+ @Override
+ public int compare(SqlIdentifier o1, SqlIdentifier o2) {
+ int linegap = o2.getParserPosition().getLineNum() - o1.getParserPosition().getLineNum();
+ if (linegap != 0)
+ return linegap;
+
+ return o2.getParserPosition().getColumnNum() - o1.getParserPosition().getColumnNum();
+ }
+ });
+
+ for (SqlIdentifier sqlIdentifier : sqlIdentifiers) {
+ Pair<Integer, Integer> replacePos = getReplacePos(sqlIdentifier, sql);
+ int start = replacePos.getLeft();
+ sql = sql.substring(0, start) + alias + "." + sql.substring(start);
+ }
+
+ return sql.substring(prefix.length(), sql.length() - suffix.length());
+ }
+
+ public static Pair<Integer, Integer> getReplacePos(SqlNode node, String... lines) {
+ SqlParserPos pos = node.getParserPosition();
+ int lineStart = pos.getLineNum();
+ int lineEnd = pos.getEndLineNum();
+ int columnStart = pos.getColumnNum() - 1;
+ int columnEnd = pos.getEndColumnNum();
+ //for the case that sql is multi lines
+ for (int i = 0; i < lineStart - 1; i++) {
+ columnStart += lines[i].length() + 1;
+ }
+ for (int i = 0; i < lineEnd - 1; i++) {
+ columnEnd += lines[i].length() + 1;
+ }
+ return Pair.of(columnStart, columnEnd);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
new file mode 100644
index 0000000..acf7b5a
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.model.tool;
+
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import com.google.common.base.Preconditions;
+
+public class CalciteParserTest {
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testNoTableNameExists() throws SqlParseException {
+ String expr1 = "a + b";
+ Assert.assertEquals("x.a + x.b", CalciteParser.insertAliasInExpr(expr1, "x"));
+
+ String expr2 = "a + year(b)";
+ Assert.assertEquals("x.a + year(x.b)", CalciteParser.insertAliasInExpr(expr2, "x"));
+
+ String expr3 = "a + hiveudf(b)";
+ Assert.assertEquals("x.a + hiveudf(x.b)", CalciteParser.insertAliasInExpr(expr3, "x"));
+ }
+
+ @Test
+ public void testTableNameExists1() throws SqlParseException {
+ String expr1 = "a + x.b";
+
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("SqlIdentifier X.B contains DB/Table name");
+ CalciteParser.insertAliasInExpr(expr1, "x");
+ }
+
+ @Test
+ public void testTableNameExists2() throws SqlParseException {
+ String expr1 = "a + year(x.b)";
+
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("SqlIdentifier X.B contains DB/Table name");
+ CalciteParser.insertAliasInExpr(expr1, "x");
+ }
+
+ @Test
+ public void testTableNameExists3() throws SqlParseException {
+ String expr1 = "a + hiveudf(x.b)";
+
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("SqlIdentifier X.B contains DB/Table name");
+ CalciteParser.insertAliasInExpr(expr1, "x");
+ }
+
+ @Test
+ public void testPos() throws SqlParseException {
+ String[] sqls = new String[] { "select \n a \n + \n b \n from t", //
+ "select\na\n+\nb\nfrom t", //
+ "select \r\n a \r\n + \r\n b \r\n from t", //
+ "select\r\na\r\n+\r\nb\r\nfrom t" };
+
+ for (String sql : sqls) {
+ SqlNode parse = ((SqlSelect) CalciteParser.parse(sql)).getSelectList().get(0);
+ String[] lines = sql.split("\n");
+ Pair<Integer, Integer> replacePos = CalciteParser.getReplacePos(parse, lines);
+ String substring = sql.substring(replacePos.getLeft(), replacePos.getRight());
+ Preconditions.checkArgument(substring.startsWith("a"));
+ Preconditions.checkArgument(substring.endsWith("b"));
+ }
+
+ }
+
+ @Test
+ public void testEqual() throws SqlParseException {
+ String sql0 = "select a.a + a.b + a.c from t as a";
+ String sql1 = "select (((a . a + a.b + a.c))) from t as a";
+ String sql2 = "select (a + b) + c from t";
+ String sql3 = "select a.a + (a.b + a.c) from t as a";
+
+ SqlNode sn0 = CalciteParser.getOnlySelectNode(sql0);
+ SqlNode sn1 = CalciteParser.getOnlySelectNode(sql1);
+ SqlNode sn2 = CalciteParser.getOnlySelectNode(sql2);
+ SqlNode sn3 = CalciteParser.getOnlySelectNode(sql3);
+
+ Assert.assertEquals(true, CalciteParser.isNodeEqual(sn0, sn1));
+ Assert.assertEquals(true, CalciteParser.isNodeEqual(sn0, sn2));
+ Assert.assertEquals(false, CalciteParser.isNodeEqual(sn0, sn3));
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
index d084ce2..5225cf3 100644
--- a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
+++ b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
@@ -125,21 +125,21 @@
{
"tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
"columnName": "DEAL_AMOUNT",
- "expression": "DEFAULT.TEST_KYLIN_FACT.PRICE * DEFAULT.TEST_KYLIN_FACT.ITEM_COUNT",
+ "expression": "PRICE * ITEM_COUNT",
"datatype": "decimal",
"comment": "deal amount of inner join model"
},
{
"tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
"columnName": "DEAL_YEAR",
- "expression": "year(DEFAULT.TEST_KYLIN_FACT.CAL_DT)",
+ "expression": "year(CAL_DT)",
"datatype": "integer",
"comment": "the year of the deal of the inner join model"
},
{
"tableIdentity": "DEFAULT.TEST_ACCOUNT",
"columnName": "COUNTRY_ABBR",
- "expression": "SUBSTR(DEFAULT.TEST_ACCOUNT.ACCOUNT_COUNTRY,0,1)",
+ "expression": "SUBSTR(ACCOUNT_COUNTRY,0,1)",
"datatype": "string",
"comment": "first char of country of the inner join model"
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
index 4a10899..0058425 100644
--- a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
+++ b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
@@ -125,21 +125,21 @@
{
"tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
"columnName": "DEAL_AMOUNT",
- "expression": "DEFAULT.TEST_KYLIN_FACT.PRICE * DEFAULT.TEST_KYLIN_FACT.ITEM_COUNT",
+ "expression": "PRICE * ITEM_COUNT",
"datatype": "decimal",
"comment": "deal amount of left join model"
},
{
"tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
"columnName": "DEAL_YEAR",
- "expression": "year(DEFAULT.TEST_KYLIN_FACT.CAL_DT)",
+ "expression": "year(CAL_DT)",
"datatype": "integer",
"comment": "the year of the deal of the left join model"
},
{
"tableIdentity": "DEFAULT.TEST_ACCOUNT",
"columnName": "COUNTRY_ABBR",
- "expression": "SUBSTR(DEFAULT.TEST_ACCOUNT.ACCOUNT_COUNTRY,0,1)",
+ "expression": "SUBSTR(ACCOUNT_COUNTRY,0,1)",
"datatype": "string",
"comment": "first char of country of the left join model"
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
index d8f1220..7f188c1 100644
--- a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++ b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -23,7 +23,6 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import javax.annotation.Nullable;
@@ -35,15 +34,14 @@ import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
-import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.parser.SqlParseException;
-import org.apache.calcite.sql.parser.SqlParser;
-import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.util.SqlVisitor;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
@@ -65,7 +63,11 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
return sql;
}
ImmutableSortedMap<String, String> computedColumns = getSortedComputedColumnWithProject(project);
- return replaceComputedColumn(sql, computedColumns);
+ String s = replaceComputedColumn(sql, computedColumns);
+ if (!StringUtils.equals(sql, s)) {
+ logger.debug("change sql to " + s);
+ }
+ return s;
}
static String replaceComputedColumn(String inputSql, ImmutableSortedMap<String, String> computedColumn) {
@@ -88,7 +90,7 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
logger.error("Convert to computedColumn Fail,parse sql fail ", e.getMessage());
}
for (SqlNode node : matchedNodes) {
- Pair<Integer, Integer> startEndPos = getReplacePos(lines, node);
+ Pair<Integer, Integer> startEndPos = CalciteParser.getReplacePos(node, lines);
int start = startEndPos.getLeft();
int end = startEndPos.getRight();
//add table alias like t1.column,if exists alias
@@ -104,32 +106,18 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
return result;
}
- private static Pair<Integer, Integer> getReplacePos(String[] lines, SqlNode node) {
- SqlParserPos pos = node.getParserPosition();
- int lineStart = pos.getLineNum();
- int columnStart = pos.getColumnNum() - 1;
- int columnEnd = pos.getEndColumnNum();
- //for the case that sql is multi lines
- for (int i = 0; i < lineStart - 1; i++) {
- int offset = lines[i].length();
- columnStart += offset + 1;
- columnEnd += offset + 1;
- }
- return Pair.of(columnStart, columnEnd);
- }
-
//Return matched node's position and its alias(if exists).If can not find matches, return an empty capacity list
private static List<SqlNode> getMatchedNodes(String inputSql, String ccExp) throws SqlParseException {
if (ccExp == null || ccExp.equals("")) {
return new ArrayList<>();
}
ArrayList<SqlNode> toBeReplacedNodes = new ArrayList<>();
- SqlNode ccNode = getCCExpNode(ccExp);
+ SqlNode ccNode = CalciteParser.getExpNode(ccExp);
List<SqlNode> inputNodes = getInputTreeNodes(inputSql);
// find whether user input sql's tree node equals computed columns's define expression
for (SqlNode inputNode : inputNodes) {
- if (isNodeEqual(inputNode, ccNode)) {
+ if (CalciteParser.isNodeEqual(inputNode, ccNode)) {
toBeReplacedNodes.add(inputNode);
}
}
@@ -138,85 +126,10 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
private static List<SqlNode> getInputTreeNodes(String inputSql) throws SqlParseException {
SqlTreeVisitor stv = new SqlTreeVisitor();
- parse(inputSql).accept(stv);
+ CalciteParser.parse(inputSql).accept(stv);
return stv.getSqlNodes();
}
- private static SqlNode getCCExpNode(String ccExp) throws SqlParseException {
- ccExp = "select " + ccExp + " from t";
- return ((SqlSelect) parse(ccExp)).getSelectList().get(0);
- }
-
- static SqlNode parse(String sql) throws SqlParseException {
- SqlParser.ConfigBuilder parserBuilder = SqlParser.configBuilder();
- SqlParser sqlParser = SqlParser.create(sql, parserBuilder.build());
- return sqlParser.parseQuery();
- }
-
- static boolean isNodeEqual(SqlNode node0, SqlNode node1) {
- if (node0 == null) {
- return node1 == null;
- } else if (node1 == null) {
- return false;
- }
-
- if (!Objects.equals(node0.getClass().getSimpleName(), node1.getClass().getSimpleName())) {
- return false;
- }
-
- if (node0 instanceof SqlCall) {
- SqlCall thisNode = (SqlCall) node0;
- SqlCall thatNode = (SqlCall) node1;
- if (!thisNode.getOperator().getName().equalsIgnoreCase(thatNode.getOperator().getName())) {
- return false;
- }
- return isNodeEqual(thisNode.getOperandList(), thatNode.getOperandList());
- }
- if (node0 instanceof SqlLiteral) {
- SqlLiteral thisNode = (SqlLiteral) node0;
- SqlLiteral thatNode = (SqlLiteral) node1;
- return Objects.equals(thisNode.getValue(), thatNode.getValue());
- }
- if (node0 instanceof SqlNodeList) {
- SqlNodeList thisNode = (SqlNodeList) node0;
- SqlNodeList thatNode = (SqlNodeList) node1;
- if (thisNode.getList().size() != thatNode.getList().size()) {
- return false;
- }
- for (int i = 0; i < thisNode.getList().size(); i++) {
- SqlNode thisChild = thisNode.getList().get(i);
- final SqlNode thatChild = thatNode.getList().get(i);
- if (!isNodeEqual(thisChild, thatChild)) {
- return false;
- }
- }
- return true;
- }
- if (node0 instanceof SqlIdentifier) {
- SqlIdentifier thisNode = (SqlIdentifier) node0;
- SqlIdentifier thatNode = (SqlIdentifier) node1;
- // compare ignore table alias.eg: expression like "a.b + a.c + a.d" ,alias a will be ignored when compared
- String name0 = thisNode.names.get(thisNode.names.size() - 1).replace("\"", "");
- String name1 = thatNode.names.get(thatNode.names.size() - 1).replace("\"", "");
- return name0.equalsIgnoreCase(name1);
- }
-
- logger.error("Convert to computed column fail,failed to compare two nodes,unknown instance type");
- return false;
- }
-
- private static boolean isNodeEqual(List<SqlNode> operands0, List<SqlNode> operands1) {
- if (operands0.size() != operands1.size()) {
- return false;
- }
- for (int i = 0; i < operands0.size(); i++) {
- if (!isNodeEqual(operands0.get(i), operands1.get(i))) {
- return false;
- }
- }
- return true;
- }
-
private static String getTableAlias(SqlNode node) {
if (node instanceof SqlCall) {
SqlCall call = (SqlCall) node;
@@ -350,4 +263,4 @@ class SqlTreeVisitor implements SqlVisitor<SqlNode> {
public SqlNode visit(SqlIntervalQualifier intervalQualifier) {
return null;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java b/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
index c3efe8d..f9b8d8b 100644
--- a/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
@@ -21,8 +21,6 @@ package org.apache.kylin.query.util;
import java.util.HashMap;
import java.util.Map;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.parser.SqlParseException;
import org.junit.Assert;
import org.junit.Test;
@@ -30,22 +28,6 @@ import org.junit.Test;
import com.google.common.collect.ImmutableSortedMap;
public class ConvertToComputedColumnTest {
- @Test
- public void testEqual() throws SqlParseException {
- String sql0 = "select a.a + a.b + a.c from t as a";
- String sql1 = "select (((a . a + a.b + a.c))) from t as a";
- String sql2 = "select (a + b) + c from t";
- String sql3 = "select a.a + (a.b + a.c) from t as a";
-
- SqlNode sn0 = getSelectNode(sql0);
- SqlNode sn1 = getSelectNode(sql1);
- SqlNode sn2 = getSelectNode(sql2);
- SqlNode sn3 = getSelectNode(sql3);
-
- Assert.assertEquals(true, ConvertToComputedColumn.isNodeEqual(sn0, sn1));
- Assert.assertEquals(true, ConvertToComputedColumn.isNodeEqual(sn0, sn2));
- Assert.assertEquals(false, ConvertToComputedColumn.isNodeEqual(sn0, sn3));
- }
@Test
public void testErrorCase() {
@@ -101,10 +83,6 @@ public class ConvertToComputedColumnTest {
}
- private static SqlNode getSelectNode(String sql) throws SqlParseException {
- return ((SqlSelect) ConvertToComputedColumn.parse(sql)).getSelectList().get(0);
- }
-
@Test
public void testTwoCCHasSameSubExp() {
String sql0 = "select a + b + c from t order by a + b";
[13/26] kylin git commit: minor, fix bug "skip all steps after resume"
Posted by li...@apache.org.
minor, fix bug "skip all steps after resume"
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7f9dab7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7f9dab7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7f9dab7
Branch: refs/heads/kylin-2.1.x
Commit: e7f9dab77f087ce1701974abaae1b8f541448de4
Parents: 5eca7f6
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 23:08:27 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 23:11:00 2017 +0800
----------------------------------------------------------------------
.../kylin/job/execution/DefaultChainedExecutable.java | 1 +
.../org/apache/kylin/job/execution/ExecutableManager.java | 9 +++++++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7f9dab7/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index ae129ab..4f5c02b 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -75,6 +75,7 @@ public class DefaultChainedExecutable extends AbstractExecutable implements Chai
final long endTime = getEndTime();
if (endTime > 0) {
long interruptTime = System.currentTimeMillis() - endTime + getInterruptTime();
+ info.putAll(getManager().getJobOutput(getId()).getInfo());
info.put(START_TIME, Long.toString(startTime));
info.put(INTERRUPT_TIME, Long.toString(interruptTime));
getManager().updateJobOutput(getId(), ExecutableState.RUNNING, info, null);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7f9dab7/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index f16cfde..f8fb980 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -389,6 +389,15 @@ public class ExecutableManager {
updateJobOutput(jobId, ExecutableState.STOPPED, null, null);
}
+ public ExecutableOutputPO getJobOutput(String jobId) {
+ try {
+ return executableDao.getJobOutput(jobId);
+ } catch (PersistentException e) {
+ logger.error("Can't get output of Job " + jobId);
+ throw new RuntimeException(e);
+ }
+ }
+
public void updateJobOutput(String jobId, ExecutableState newStatus, Map<String, String> info, String output) {
try {
final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
[24/26] kylin git commit: minor, rename adhoc to pushdown in KYLIN
Posted by li...@apache.org.
minor, rename adhoc to pushdown in KYLIN
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7002dd86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7002dd86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7002dd86
Branch: refs/heads/kylin-2.1.x
Commit: 7002dd864a3f43a2e97a022d1d348773609f446d
Parents: d91f522
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Jun 30 21:12:54 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 21:18:10 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 28 +-
.../main/resources/kylin-defaults.properties | 18 +-
.../kylin/metadata/project/ProjectManager.java | 8 +-
.../source/adhocquery/HiveAdhocConverter.java | 297 -------------------
.../adhocquery/HivePushDownConverter.java | 297 +++++++++++++++++++
.../source/adhocquery/IAdHocConverter.java | 25 --
.../kylin/source/adhocquery/IAdHocRunner.java | 39 ---
.../source/adhocquery/IPushDownConverter.java | 25 ++
.../source/adhocquery/IPushDownRunner.java | 39 +++
.../adhocquery/HiveAdhocConverterTest.java | 78 -----
.../adhocquery/HivePushDownConverterTest.java | 78 +++++
.../test_case_data/sandbox/kylin.properties | 18 +-
.../org/apache/kylin/query/KylinTestBase.java | 4 +-
.../kylin/query/adhoc/AdHocRunnerJdbcImpl.java | 122 --------
.../query/adhoc/PushDownRunnerJdbcImpl.java | 122 ++++++++
.../apache/kylin/rest/response/SQLResponse.java | 17 +-
.../apache/kylin/rest/service/QueryService.java | 150 +++++++---
.../org/apache/kylin/rest/util/AdHocUtil.java | 178 -----------
.../apache/kylin/rest/util/PushDownUtil.java | 177 +++++++++++
.../apache/kylin/rest/util/AdHocUtilTest.java | 94 ------
.../kylin/rest/util/PushDownUtilTest.java | 94 ++++++
21 files changed, 986 insertions(+), 922 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4a21ded..179d61f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1022,45 +1022,45 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
}
- public boolean isAdhocEnabled() {
- return StringUtils.isNotEmpty(getAdHocRunnerClassName());
+ public boolean isPushDownEnabled() {
+ return StringUtils.isNotEmpty(getPushDownRunnerClassName());
}
- public String getAdHocRunnerClassName() {
- return getOptional("kylin.query.ad-hoc.runner-class-name", "");
+ public String getPushDownRunnerClassName() {
+ return getOptional("kylin.query.pushdown.runner-class-name", "");
}
- public String getAdHocConverterClassName() {
- return getOptional("kylin.query.ad-hoc.converter-class-name",
- "org.apache.kylin.source.adhocquery.HiveAdhocConverter");
+ public String getPushDownConverterClassName() {
+ return getOptional("kylin.query.pushdown.converter-class-name",
+ "org.apache.kylin.source.adhocquery.HivePushDownConverter");
}
public String getJdbcUrl() {
- return getOptional("kylin.query.ad-hoc.jdbc.url", "");
+ return getOptional("kylin.query.pushdown.jdbc.url", "");
}
public String getJdbcDriverClass() {
- return getOptional("kylin.query.ad-hoc.jdbc.driver", "");
+ return getOptional("kylin.query.pushdown.jdbc.driver", "");
}
public String getJdbcUsername() {
- return getOptional("kylin.query.ad-hoc.jdbc.username", "");
+ return getOptional("kylin.query.pushdown.jdbc.username", "");
}
public String getJdbcPassword() {
- return getOptional("kylin.query.ad-hoc.jdbc.password", "");
+ return getOptional("kylin.query.pushdown.jdbc.password", "");
}
public int getPoolMaxTotal() {
- return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-max-total", "8"));
+ return Integer.parseInt(this.getOptional("kylin.query.pushdown.jdbc.pool-max-total", "8"));
}
public int getPoolMaxIdle() {
- return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-max-idle", "8"));
+ return Integer.parseInt(this.getOptional("kylin.query.pushdown.jdbc.pool-max-idle", "8"));
}
public int getPoolMinIdle() {
- return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-min-idle", "0"));
+ return Integer.parseInt(this.getOptional("kylin.query.pushdown.jdbc.pool-min-idle", "0"));
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index beac63c..cb511e7 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -237,15 +237,15 @@ kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-histo
#kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current
-### AD-HOC QUERY ###
+### QUERY PUSH DOWN ###
-#kylin.query.ad-hoc.runner-class-name=org.apache.kylin.query.adhoc.AdHocRunnerJdbcImpl
+#kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl
-#kylin.query.ad-hoc.jdbc.url=jdbc:hive2://sandbox:10000/default
-#kylin.query.ad-hoc.jdbc.driver=org.apache.hive.jdbc.HiveDriver
-#kylin.query.ad-hoc.jdbc.username=hive
-#kylin.query.ad-hoc.jdbc.password=
+#kylin.query.pushdown.jdbc.url=jdbc:hive2://sandbox:10000/default
+#kylin.query.pushdown.jdbc.driver=org.apache.hive.jdbc.HiveDriver
+#kylin.query.pushdown.jdbc.username=hive
+#kylin.query.pushdown.jdbc.password=
-#kylin.query.ad-hoc.jdbc.pool-max-total=8
-#kylin.query.ad-hoc.jdbc.pool-max-idle=8
-#kylin.query.ad-hoc.jdbc.pool-min-idle=0
+#kylin.query.pushdown.jdbc.pool-max-total=8
+#kylin.query.pushdown.jdbc.pool-max-idle=8
+#kylin.query.pushdown.jdbc.pool-min-idle=0
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index ea03c3c..b801d8c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -437,7 +437,7 @@ public class ProjectManager {
}
public Collection<TableDesc> listExposedTables(String project) {
- return config.isAdhocEnabled() ? //
+ return config.isPushDownEnabled() ? //
this.listDefinedTables(project) : //
l2Cache.listExposedTables(norm(project));
}
@@ -445,7 +445,7 @@ public class ProjectManager {
public List<ColumnDesc> listExposedColumns(String project, TableDesc tableDesc) {
Set<ColumnDesc> exposedColumns = l2Cache.listExposedColumns(norm(project), tableDesc.getIdentity());
- if (config.isAdhocEnabled()) {
+ if (config.isPushDownEnabled()) {
// take care of computed columns
Set<ColumnDesc> dedup = Sets.newHashSet(tableDesc.getColumns());
dedup.addAll(exposedColumns);
@@ -456,13 +456,13 @@ public class ProjectManager {
}
public boolean isExposedTable(String project, String table) {
- return config.isAdhocEnabled() ? //
+ return config.isPushDownEnabled() ? //
l2Cache.isDefinedTable(norm(project), table) : //
l2Cache.isExposedTable(norm(project), table);
}
public boolean isExposedColumn(String project, String table, String col) {
- return config.isAdhocEnabled() ? //
+ return config.isPushDownEnabled() ? //
l2Cache.isDefinedColumn(norm(project), table, col) || l2Cache.isExposedColumn(norm(project), table, col)
: //
l2Cache.isExposedColumn(norm(project), table, col);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
deleted file mode 100644
index e8c06ff..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.source.adhocquery;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Stack;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.ImmutableSet;
-
-//TODO: Some workaround ways to make sql readable by hive parser, should replaced it with a more well-designed way
-public class HiveAdhocConverter implements IAdHocConverter {
-
- @SuppressWarnings("unused")
- private static final Logger logger = LoggerFactory.getLogger(HiveAdhocConverter.class);
-
- private static final Pattern EXTRACT_PATTERN = Pattern.compile("extract\\s*(\\()\\s*(.*?)\\s*from(\\s+)",
- Pattern.CASE_INSENSITIVE);
- private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\()\\s*select\\s",
- Pattern.CASE_INSENSITIVE);
- private static final Pattern ALIAS_PATTERN = Pattern.compile("\\s+([`'_a-z0-9A-Z]+)", Pattern.CASE_INSENSITIVE);
- private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\((.*?) (?i)AS\\s*(.*?)\\s*\\)",
- Pattern.CASE_INSENSITIVE);
- private static final Pattern CONCAT_PATTERN = Pattern.compile("(['_a-z0-9A-Z]+)\\|\\|(['_a-z0-9A-Z]+)",
- Pattern.CASE_INSENSITIVE);
- private static final Pattern TIMESTAMPADD_PATTERN = Pattern.compile("timestampadd\\s*\\(\\s*(.*?)\\s*,",
- Pattern.CASE_INSENSITIVE);
- private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("A", "ABS", "ABSOLUTE", "ACTION",
- "ADA", "ADD", "ADMIN", "AFTER", "ALL", "ALLOCATE", "ALLOW", "ALTER", "ALWAYS", "AND", "ANY", "APPLY", "ARE",
- "ARRAY", "ARRAY_MAX_CARDINALITY", "ASC", "ASENSITIVE", "ASSERTION", "ASSIGNMENT", "ASYMMETRIC", "AT",
- "ATOMIC", "ATTRIBUTE", "ATTRIBUTES", "AUTHORIZATION", "AVG", "BEFORE", "BEGIN", "BEGIN_FRAME",
- "BEGIN_PARTITION", "BERNOULLI", "BETWEEN", "BIGINT", "BINARY", "BIT", "BLOB", "BOOLEAN", "BOTH", "BREADTH",
- "BY", "C", "CALL", "CALLED", "CARDINALITY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG",
- "CATALOG_NAME", "CEIL", "CEILING", "CENTURY", "CHAIN", "CHAR", "CHARACTER", "CHARACTERISTICS", "CHARACTERS",
- "CHARACTER_LENGTH", "CHARACTER_SET_CATALOG", "CHARACTER_SET_NAME", "CHARACTER_SET_SCHEMA", "CHAR_LENGTH",
- "CHECK", "CLASSIFIER", "CLASS_ORIGIN", "CLOB", "CLOSE", "COALESCE", "COBOL", "COLLATE", "COLLATION",
- "COLLATION_CATALOG", "COLLATION_NAME", "COLLATION_SCHEMA", "COLLECT", "COLUMN", "COLUMN_NAME",
- "COMMAND_FUNCTION", "COMMAND_FUNCTION_CODE", "COMMIT", "COMMITTED", "CONDITION", "CONDITION_NUMBER",
- "CONNECT", "CONNECTION", "CONNECTION_NAME", "CONSTRAINT", "CONSTRAINTS", "CONSTRAINT_CATALOG",
- "CONSTRAINT_NAME", "CONSTRAINT_SCHEMA", "CONSTRUCTOR", "CONTAINS", "CONTINUE", "CONVERT", "CORR",
- "CORRESPONDING", "COUNT", "COVAR_POP", "COVAR_SAMP", "CREATE", "CROSS", "CUBE", "CUME_DIST", "CURRENT",
- "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_DEFAULT_TRANSFORM_GROUP", "CURRENT_PATH", "CURRENT_ROLE",
- "CURRENT_ROW", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_TRANSFORM_GROUP_FOR_TYPE",
- "CURRENT_USER", "CURSOR", "CURSOR_NAME", "CYCLE", "DATA", "DATABASE", "DATE", "DATETIME_INTERVAL_CODE",
- "DATETIME_INTERVAL_PRECISION", "DAY", "DEALLOCATE", "DEC", "DECADE", "DECIMAL", "DECLARE", "DEFAULT",
- "DEFAULTS", "DEFERRABLE", "DEFERRED", "DEFINE", "DEFINED", "DEFINER", "DEGREE", "DELETE", "DENSE_RANK",
- "DEPTH", "DEREF", "DERIVED", "DESC", "DESCRIBE", "DESCRIPTION", "DESCRIPTOR", "DETERMINISTIC",
- "DIAGNOSTICS", "DISALLOW", "DISCONNECT", "DISPATCH", "DISTINCT", "DOMAIN", "DOUBLE", "DOW", "DOY", "DROP",
- "DYNAMIC", "DYNAMIC_FUNCTION", "DYNAMIC_FUNCTION_CODE", "EACH", "ELEMENT", "ELSE", "EMPTY", "END",
- "END-EXEC", "END_FRAME", "END_PARTITION", "EPOCH", "EQUALS", "ESCAPE", "EVERY", "EXCEPT", "EXCEPTION",
- "EXCLUDE", "EXCLUDING", "EXEC", "EXECUTE", "EXISTS", "EXP", "EXPLAIN", "EXTEND", "EXTERNAL", "EXTRACT",
- "FALSE", "FETCH", "FILTER", "FINAL", "FIRST", "FIRST_VALUE", "FLOAT", "FLOOR", "FOLLOWING", "FOR",
- "FOREIGN", "FORTRAN", "FOUND", "FRAC_SECOND", "FRAME_ROW", "FREE", "FROM", "FULL", "FUNCTION", "FUSION",
- "G", "GENERAL", "GENERATED", "GET", "GLOBAL", "GO", "GOTO", "GRANT", "GRANTED", "GROUP", "GROUPING",
- "GROUPS", "HAVING", "HIERARCHY", "HOLD", "HOUR", "IDENTITY", "IMMEDIATE", "IMMEDIATELY", "IMPLEMENTATION",
- "IMPORT", "IN", "INCLUDING", "INCREMENT", "INDICATOR", "INITIAL", "INITIALLY", "INNER", "INOUT", "INPUT",
- "INSENSITIVE", "INSERT", "INSTANCE", "INSTANTIABLE", "INT", "INTEGER", "INTERSECT", "INTERSECTION",
- "INTERVAL", "INTO", "INVOKER", "IS", "ISOLATION", "JAVA", "JOIN", "JSON", "K", "KEY", "KEY_MEMBER",
- "KEY_TYPE", "LABEL", "LAG", "LANGUAGE", "LARGE", "LAST", "LAST_VALUE", "LATERAL", "LEAD", "LEADING", "LEFT",
- "LENGTH", "LEVEL", "LIBRARY", "LIKE", "LIKE_REGEX", "LIMIT", "LN", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP",
- "LOCATOR", "LOWER", "M", "MAP", "MATCH", "MATCHED", "MATCHES", "MATCH_NUMBER", "MATCH_RECOGNIZE", "MAX",
- "MAXVALUE", "MEASURES", "MEMBER", "MERGE", "MESSAGE_LENGTH", "MESSAGE_OCTET_LENGTH", "MESSAGE_TEXT",
- "METHOD", "MICROSECOND", "MILLENNIUM", "MIN", "MINUS", "MINUTE", "MINVALUE", "MOD", "MODIFIES", "MODULE",
- "MONTH", "MORE", "MULTISET", "MUMPS", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NCLOB", "NESTING",
- "NEW", "NEXT", "NO", "NONE", "NORMALIZE", "NORMALIZED", "NOT", "NTH_VALUE", "NTILE", "NULL", "NULLABLE",
- "NULLIF", "NULLS", "NUMBER", "NUMERIC", "OBJECT", "OCCURRENCES_REGEX", "OCTETS", "OCTET_LENGTH", "OF",
- "OFFSET", "OLD", "OMIT", "ON", "ONE", "ONLY", "OPEN", "OPTION", "OPTIONS", "OR", "ORDER", "ORDERING",
- "ORDINALITY", "OTHERS", "OUT", "OUTER", "OUTPUT", "OVER", "OVERLAPS", "OVERLAY", "OVERRIDING", "PAD",
- "PARAMETER", "PARAMETER_MODE", "PARAMETER_NAME", "PARAMETER_ORDINAL_POSITION", "PARAMETER_SPECIFIC_CATALOG",
- "PARAMETER_SPECIFIC_NAME", "PARAMETER_SPECIFIC_SCHEMA", "PARTIAL", "PARTITION", "PASCAL", "PASSTHROUGH",
- "PAST", "PATH", "PATTERN", "PER", "PERCENT", "PERCENTILE_CONT", "PERCENTILE_DISC", "PERCENT_RANK", "PERIOD",
- "PERMUTE", "PLACING", "PLAN", "PLI", "PORTION", "POSITION", "POSITION_REGEX", "POWER", "PRECEDES",
- "PRECEDING", "PRECISION", "PREPARE", "PRESERVE", "PREV", "PRIMARY", "PRIOR", "PRIVILEGES", "PROCEDURE",
- "PUBLIC", "QUARTER", "RANGE", "RANK", "READ", "READS", "REAL", "RECURSIVE", "REF", "REFERENCES",
- "REFERENCING", "REGR_AVGX", "REGR_AVGY", "REGR_COUNT", "REGR_INTERCEPT", "REGR_R2", "REGR_SLOPE",
- "REGR_SXX", "REGR_SXY", "REGR_SYY", "RELATIVE", "RELEASE", "REPEATABLE", "REPLACE", "RESET", "RESTART",
- "RESTRICT", "RESULT", "RETURN", "RETURNED_CARDINALITY", "RETURNED_LENGTH", "RETURNED_OCTET_LENGTH",
- "RETURNED_SQLSTATE", "RETURNS", "REVOKE", "RIGHT", "ROLE", "ROLLBACK", "ROLLUP", "ROUTINE",
- "ROUTINE_CATALOG", "ROUTINE_NAME", "ROUTINE_SCHEMA", "ROW", "ROWS", "ROW_COUNT", "ROW_NUMBER", "RUNNING",
- "SAVEPOINT", "SCALE", "SCHEMA", "SCHEMA_NAME", "SCOPE", "SCOPE_CATALOGS", "SCOPE_NAME", "SCOPE_SCHEMA",
- "SCROLL", "SEARCH", "SECOND", "SECTION", "SECURITY", "SEEK", "SELECT", "SELF", "SENSITIVE", "SEQUENCE",
- "SERIALIZABLE", "SERVER", "SERVER_NAME", "SESSION", "SESSION_USER", "SET", "SETS", "SHOW", "SIMILAR",
- "SIMPLE", "SIZE", "SKIP", "SMALLINT", "SOME", "SOURCE", "SPACE", "SPECIFIC", "SPECIFICTYPE",
- "SPECIFIC_NAME", "SQL", "SQLEXCEPTION", "SQLSTATE", "SQLWARNING", "SQL_BIGINT", "SQL_BINARY", "SQL_BIT",
- "SQL_BLOB", "SQL_BOOLEAN", "SQL_CHAR", "SQL_CLOB", "SQL_DATE", "SQL_DECIMAL", "SQL_DOUBLE", "SQL_FLOAT",
- "SQL_INTEGER", "SQL_INTERVAL_DAY", "SQL_INTERVAL_DAY_TO_HOUR", "SQL_INTERVAL_DAY_TO_MINUTE",
- "SQL_INTERVAL_DAY_TO_SECOND", "SQL_INTERVAL_HOUR", "SQL_INTERVAL_HOUR_TO_MINUTE",
- "SQL_INTERVAL_HOUR_TO_SECOND", "SQL_INTERVAL_MINUTE", "SQL_INTERVAL_MINUTE_TO_SECOND", "SQL_INTERVAL_MONTH",
- "SQL_INTERVAL_SECOND", "SQL_INTERVAL_YEAR", "SQL_INTERVAL_YEAR_TO_MONTH", "SQL_LONGVARBINARY",
- "SQL_LONGVARCHAR", "SQL_LONGVARNCHAR", "SQL_NCHAR", "SQL_NCLOB", "SQL_NUMERIC", "SQL_NVARCHAR", "SQL_REAL",
- "SQL_SMALLINT", "SQL_TIME", "SQL_TIMESTAMP", "SQL_TINYINT", "SQL_TSI_DAY", "SQL_TSI_FRAC_SECOND",
- "SQL_TSI_HOUR", "SQL_TSI_MICROSECOND", "SQL_TSI_MINUTE", "SQL_TSI_MONTH", "SQL_TSI_QUARTER",
- "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SQL_VARBINARY", "SQL_VARCHAR", "SQRT", "START", "STATE",
- "STATEMENT", "STATIC", "STDDEV_POP", "STDDEV_SAMP", "STREAM", "STRUCTURE", "STYLE", "SUBCLASS_ORIGIN",
- "SUBMULTISET", "SUBSET", "SUBSTITUTE", "SUBSTRING", "SUBSTRING_REGEX", "SUCCEEDS", "SUM", "SYMMETRIC",
- "SYSTEM", "SYSTEM_TIME", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TABLE_NAME", "TEMPORARY", "THEN", "TIES",
- "TIME", "TIMESTAMP", "TIMESTAMPADD", "TIMESTAMPDIFF", "TIMEZONE_HOUR", "TIMEZONE_MINUTE", "TINYINT", "TO",
- "TOP_LEVEL_COUNT", "TRAILING", "TRANSACTION", "TRANSACTIONS_ACTIVE", "TRANSACTIONS_COMMITTED",
- "TRANSACTIONS_ROLLED_BACK", "TRANSFORM", "TRANSFORMS", "TRANSLATE", "TRANSLATE_REGEX", "TRANSLATION",
- "TREAT", "TRIGGER", "TRIGGER_CATALOG", "TRIGGER_NAME", "TRIGGER_SCHEMA", "TRIM", "TRIM_ARRAY", "TRUE",
- "TRUNCATE", "TYPE", "UESCAPE", "UNBOUNDED", "UNCOMMITTED", "UNDER", "UNION", "UNIQUE", "UNKNOWN", "UNNAMED",
- "UNNEST", "UPDATE", "UPPER", "UPSERT", "USAGE", "USER", "USER_DEFINED_TYPE_CATALOG",
- "USER_DEFINED_TYPE_CODE", "USER_DEFINED_TYPE_NAME", "USER_DEFINED_TYPE_SCHEMA", "USING", "VALUE", "VALUES",
- "VALUE_OF", "VARBINARY", "VARCHAR", "VARYING", "VAR_POP", "VAR_SAMP", "VERSION", "VERSIONING", "VIEW",
- "WEEK", "WHEN", "WHENEVER", "WHERE", "WIDTH_BUCKET", "WINDOW", "WITH", "WITHIN", "WITHOUT", "WORK",
- "WRAPPER", "WRITE", "XML", "YEAR", "ZONE");
-
- public static String replaceString(String originString, String fromString, String toString) {
- return originString.replace(fromString, toString);
- }
-
- public static String extractReplace(String originString) {
- Matcher extractMatcher = EXTRACT_PATTERN.matcher(originString);
- String replacedString = originString;
- Map<Integer, Integer> parenthesesPairs = null;
-
- while (extractMatcher.find()) {
- if (parenthesesPairs == null) {
- parenthesesPairs = findParenthesesPairs(originString);
- }
-
- String functionStr = extractMatcher.group(2);
- int startIdx = extractMatcher.end(3);
- int endIdx = parenthesesPairs.get(extractMatcher.start(1));
- String extractInner = originString.substring(startIdx, endIdx);
- int originStart = extractMatcher.start(0);
- int originEnd = endIdx + 1;
-
- replacedString = replaceString(replacedString, originString.substring(originStart, originEnd),
- functionStr + "(" + extractInner + ")");
- }
-
- return replacedString;
- }
-
- public static String castReplace(String originString) {
- Matcher castMatcher = CAST_PATTERN.matcher(originString);
- String replacedString = originString;
-
- while (castMatcher.find()) {
- String castStr = castMatcher.group();
- String type = castMatcher.group(2);
- String supportedType = "";
- switch (type.toUpperCase()) {
- case "INTEGER":
- supportedType = "int";
- break;
- case "SHORT":
- supportedType = "smallint";
- break;
- case "LONG":
- supportedType = "bigint";
- break;
- default:
- supportedType = type;
- }
-
- if (!supportedType.equals(type)) {
- String replacedCastStr = castStr.replace(type, supportedType);
- replacedString = replaceString(replacedString, castStr, replacedCastStr);
- }
- }
-
- return replacedString;
- }
-
- public static String subqueryReplace(String originString) {
- Matcher subqueryMatcher = FROM_PATTERN.matcher(originString);
- String replacedString = originString;
- Map<Integer, Integer> parenthesesPairs = null;
-
- while (subqueryMatcher.find()) {
- if (parenthesesPairs == null) {
- parenthesesPairs = findParenthesesPairs(originString);
- }
-
- int startIdx = subqueryMatcher.start(1);
- int endIdx = parenthesesPairs.get(startIdx) + 1;
-
- Matcher aliasMatcher = ALIAS_PATTERN.matcher(originString.substring(endIdx));
- if (aliasMatcher.find()) {
- String aliasCandidate = aliasMatcher.group(1);
-
- if (aliasCandidate != null && !sqlKeyWordsExceptAS.contains(aliasCandidate.toUpperCase())) {
- continue;
- }
-
- replacedString = replaceString(replacedString, originString.substring(startIdx, endIdx),
- originString.substring(startIdx, endIdx) + " as alias");
- }
- }
-
- return replacedString;
- }
-
- public static String timestampaddReplace(String originString) {
- Matcher timestampaddMatcher = TIMESTAMPADD_PATTERN.matcher(originString);
- String replacedString = originString;
-
- while (timestampaddMatcher.find()) {
- String interval = timestampaddMatcher.group(1);
- String timestampaddStr = replaceString(timestampaddMatcher.group(), interval, "'" + interval + "'");
- replacedString = replaceString(replacedString, timestampaddMatcher.group(), timestampaddStr);
- }
-
- return replacedString;
- }
-
- public static String concatReplace(String originString) {
- Matcher concatMatcher = CONCAT_PATTERN.matcher(originString);
- String replacedString = originString;
-
- while (concatMatcher.find()) {
- String leftString = concatMatcher.group(1);
- String rightString = concatMatcher.group(2);
- replacedString = replaceString(replacedString, leftString + "||" + rightString,
- "concat(" + leftString + "," + rightString + ")");
- }
-
- return replacedString;
- }
-
- public static String doConvert(String originStr) {
- // Step1.Replace " with `
- String convertedSql = replaceString(originStr, "\"", "`");
-
- // Step2.Replace extract functions
- convertedSql = extractReplace(convertedSql);
-
- // Step3.Replace cast type string
- convertedSql = castReplace(convertedSql);
-
- // Step4.Replace sub query
- convertedSql = subqueryReplace(convertedSql);
-
- // Step5.Replace char_length with length
- convertedSql = replaceString(convertedSql, "char_length", "length");
-
- // Step6.Replace "||" with concat
- convertedSql = concatReplace(convertedSql);
-
- // Step7.Add quote for interval in timestampadd
- convertedSql = timestampaddReplace(convertedSql);
-
- return convertedSql;
- }
-
- private static Map<Integer, Integer> findParenthesesPairs(String sql) {
- Map<Integer, Integer> result = new HashMap<>();
- if (sql.length() > 1) {
- Stack<Integer> lStack = new Stack<>();
- boolean inStrVal = false;
- for (int i = 0; i < sql.length(); i++) {
- switch (sql.charAt(i)) {
- case '(':
- if (!inStrVal) {
- lStack.push(i);
- }
- break;
- case ')':
- if (!inStrVal && !lStack.empty()) {
- result.put(lStack.pop(), i);
- }
- break;
- default:
- break;
- }
- }
- }
- return result;
- }
-
- @Override
- public String convert(String originSql) {
- return doConvert(originSql);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
new file mode 100644
index 0000000..8d89294
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.source.adhocquery;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Stack;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ImmutableSet;
+
+//TODO: Some workaround ways to make sql readable by hive parser, should replaced it with a more well-designed way
+public class HivePushDownConverter implements IPushDownConverter {
+
+ @SuppressWarnings("unused")
+ private static final Logger logger = LoggerFactory.getLogger(HivePushDownConverter.class);
+
+ private static final Pattern EXTRACT_PATTERN = Pattern.compile("extract\\s*(\\()\\s*(.*?)\\s*from(\\s+)",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\()\\s*select\\s",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern ALIAS_PATTERN = Pattern.compile("\\s+([`'_a-z0-9A-Z]+)", Pattern.CASE_INSENSITIVE);
+ private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\((.*?) (?i)AS\\s*(.*?)\\s*\\)",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern CONCAT_PATTERN = Pattern.compile("(['_a-z0-9A-Z]+)\\|\\|(['_a-z0-9A-Z]+)",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern TIMESTAMPADD_PATTERN = Pattern.compile("timestampadd\\s*\\(\\s*(.*?)\\s*,",
+ Pattern.CASE_INSENSITIVE);
+ private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("A", "ABS", "ABSOLUTE", "ACTION",
+ "ADA", "ADD", "ADMIN", "AFTER", "ALL", "ALLOCATE", "ALLOW", "ALTER", "ALWAYS", "AND", "ANY", "APPLY", "ARE",
+ "ARRAY", "ARRAY_MAX_CARDINALITY", "ASC", "ASENSITIVE", "ASSERTION", "ASSIGNMENT", "ASYMMETRIC", "AT",
+ "ATOMIC", "ATTRIBUTE", "ATTRIBUTES", "AUTHORIZATION", "AVG", "BEFORE", "BEGIN", "BEGIN_FRAME",
+ "BEGIN_PARTITION", "BERNOULLI", "BETWEEN", "BIGINT", "BINARY", "BIT", "BLOB", "BOOLEAN", "BOTH", "BREADTH",
+ "BY", "C", "CALL", "CALLED", "CARDINALITY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG",
+ "CATALOG_NAME", "CEIL", "CEILING", "CENTURY", "CHAIN", "CHAR", "CHARACTER", "CHARACTERISTICS", "CHARACTERS",
+ "CHARACTER_LENGTH", "CHARACTER_SET_CATALOG", "CHARACTER_SET_NAME", "CHARACTER_SET_SCHEMA", "CHAR_LENGTH",
+ "CHECK", "CLASSIFIER", "CLASS_ORIGIN", "CLOB", "CLOSE", "COALESCE", "COBOL", "COLLATE", "COLLATION",
+ "COLLATION_CATALOG", "COLLATION_NAME", "COLLATION_SCHEMA", "COLLECT", "COLUMN", "COLUMN_NAME",
+ "COMMAND_FUNCTION", "COMMAND_FUNCTION_CODE", "COMMIT", "COMMITTED", "CONDITION", "CONDITION_NUMBER",
+ "CONNECT", "CONNECTION", "CONNECTION_NAME", "CONSTRAINT", "CONSTRAINTS", "CONSTRAINT_CATALOG",
+ "CONSTRAINT_NAME", "CONSTRAINT_SCHEMA", "CONSTRUCTOR", "CONTAINS", "CONTINUE", "CONVERT", "CORR",
+ "CORRESPONDING", "COUNT", "COVAR_POP", "COVAR_SAMP", "CREATE", "CROSS", "CUBE", "CUME_DIST", "CURRENT",
+ "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_DEFAULT_TRANSFORM_GROUP", "CURRENT_PATH", "CURRENT_ROLE",
+ "CURRENT_ROW", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_TRANSFORM_GROUP_FOR_TYPE",
+ "CURRENT_USER", "CURSOR", "CURSOR_NAME", "CYCLE", "DATA", "DATABASE", "DATE", "DATETIME_INTERVAL_CODE",
+ "DATETIME_INTERVAL_PRECISION", "DAY", "DEALLOCATE", "DEC", "DECADE", "DECIMAL", "DECLARE", "DEFAULT",
+ "DEFAULTS", "DEFERRABLE", "DEFERRED", "DEFINE", "DEFINED", "DEFINER", "DEGREE", "DELETE", "DENSE_RANK",
+ "DEPTH", "DEREF", "DERIVED", "DESC", "DESCRIBE", "DESCRIPTION", "DESCRIPTOR", "DETERMINISTIC",
+ "DIAGNOSTICS", "DISALLOW", "DISCONNECT", "DISPATCH", "DISTINCT", "DOMAIN", "DOUBLE", "DOW", "DOY", "DROP",
+ "DYNAMIC", "DYNAMIC_FUNCTION", "DYNAMIC_FUNCTION_CODE", "EACH", "ELEMENT", "ELSE", "EMPTY", "END",
+ "END-EXEC", "END_FRAME", "END_PARTITION", "EPOCH", "EQUALS", "ESCAPE", "EVERY", "EXCEPT", "EXCEPTION",
+ "EXCLUDE", "EXCLUDING", "EXEC", "EXECUTE", "EXISTS", "EXP", "EXPLAIN", "EXTEND", "EXTERNAL", "EXTRACT",
+ "FALSE", "FETCH", "FILTER", "FINAL", "FIRST", "FIRST_VALUE", "FLOAT", "FLOOR", "FOLLOWING", "FOR",
+ "FOREIGN", "FORTRAN", "FOUND", "FRAC_SECOND", "FRAME_ROW", "FREE", "FROM", "FULL", "FUNCTION", "FUSION",
+ "G", "GENERAL", "GENERATED", "GET", "GLOBAL", "GO", "GOTO", "GRANT", "GRANTED", "GROUP", "GROUPING",
+ "GROUPS", "HAVING", "HIERARCHY", "HOLD", "HOUR", "IDENTITY", "IMMEDIATE", "IMMEDIATELY", "IMPLEMENTATION",
+ "IMPORT", "IN", "INCLUDING", "INCREMENT", "INDICATOR", "INITIAL", "INITIALLY", "INNER", "INOUT", "INPUT",
+ "INSENSITIVE", "INSERT", "INSTANCE", "INSTANTIABLE", "INT", "INTEGER", "INTERSECT", "INTERSECTION",
+ "INTERVAL", "INTO", "INVOKER", "IS", "ISOLATION", "JAVA", "JOIN", "JSON", "K", "KEY", "KEY_MEMBER",
+ "KEY_TYPE", "LABEL", "LAG", "LANGUAGE", "LARGE", "LAST", "LAST_VALUE", "LATERAL", "LEAD", "LEADING", "LEFT",
+ "LENGTH", "LEVEL", "LIBRARY", "LIKE", "LIKE_REGEX", "LIMIT", "LN", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP",
+ "LOCATOR", "LOWER", "M", "MAP", "MATCH", "MATCHED", "MATCHES", "MATCH_NUMBER", "MATCH_RECOGNIZE", "MAX",
+ "MAXVALUE", "MEASURES", "MEMBER", "MERGE", "MESSAGE_LENGTH", "MESSAGE_OCTET_LENGTH", "MESSAGE_TEXT",
+ "METHOD", "MICROSECOND", "MILLENNIUM", "MIN", "MINUS", "MINUTE", "MINVALUE", "MOD", "MODIFIES", "MODULE",
+ "MONTH", "MORE", "MULTISET", "MUMPS", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NCLOB", "NESTING",
+ "NEW", "NEXT", "NO", "NONE", "NORMALIZE", "NORMALIZED", "NOT", "NTH_VALUE", "NTILE", "NULL", "NULLABLE",
+ "NULLIF", "NULLS", "NUMBER", "NUMERIC", "OBJECT", "OCCURRENCES_REGEX", "OCTETS", "OCTET_LENGTH", "OF",
+ "OFFSET", "OLD", "OMIT", "ON", "ONE", "ONLY", "OPEN", "OPTION", "OPTIONS", "OR", "ORDER", "ORDERING",
+ "ORDINALITY", "OTHERS", "OUT", "OUTER", "OUTPUT", "OVER", "OVERLAPS", "OVERLAY", "OVERRIDING", "PAD",
+ "PARAMETER", "PARAMETER_MODE", "PARAMETER_NAME", "PARAMETER_ORDINAL_POSITION", "PARAMETER_SPECIFIC_CATALOG",
+ "PARAMETER_SPECIFIC_NAME", "PARAMETER_SPECIFIC_SCHEMA", "PARTIAL", "PARTITION", "PASCAL", "PASSTHROUGH",
+ "PAST", "PATH", "PATTERN", "PER", "PERCENT", "PERCENTILE_CONT", "PERCENTILE_DISC", "PERCENT_RANK", "PERIOD",
+ "PERMUTE", "PLACING", "PLAN", "PLI", "PORTION", "POSITION", "POSITION_REGEX", "POWER", "PRECEDES",
+ "PRECEDING", "PRECISION", "PREPARE", "PRESERVE", "PREV", "PRIMARY", "PRIOR", "PRIVILEGES", "PROCEDURE",
+ "PUBLIC", "QUARTER", "RANGE", "RANK", "READ", "READS", "REAL", "RECURSIVE", "REF", "REFERENCES",
+ "REFERENCING", "REGR_AVGX", "REGR_AVGY", "REGR_COUNT", "REGR_INTERCEPT", "REGR_R2", "REGR_SLOPE",
+ "REGR_SXX", "REGR_SXY", "REGR_SYY", "RELATIVE", "RELEASE", "REPEATABLE", "REPLACE", "RESET", "RESTART",
+ "RESTRICT", "RESULT", "RETURN", "RETURNED_CARDINALITY", "RETURNED_LENGTH", "RETURNED_OCTET_LENGTH",
+ "RETURNED_SQLSTATE", "RETURNS", "REVOKE", "RIGHT", "ROLE", "ROLLBACK", "ROLLUP", "ROUTINE",
+ "ROUTINE_CATALOG", "ROUTINE_NAME", "ROUTINE_SCHEMA", "ROW", "ROWS", "ROW_COUNT", "ROW_NUMBER", "RUNNING",
+ "SAVEPOINT", "SCALE", "SCHEMA", "SCHEMA_NAME", "SCOPE", "SCOPE_CATALOGS", "SCOPE_NAME", "SCOPE_SCHEMA",
+ "SCROLL", "SEARCH", "SECOND", "SECTION", "SECURITY", "SEEK", "SELECT", "SELF", "SENSITIVE", "SEQUENCE",
+ "SERIALIZABLE", "SERVER", "SERVER_NAME", "SESSION", "SESSION_USER", "SET", "SETS", "SHOW", "SIMILAR",
+ "SIMPLE", "SIZE", "SKIP", "SMALLINT", "SOME", "SOURCE", "SPACE", "SPECIFIC", "SPECIFICTYPE",
+ "SPECIFIC_NAME", "SQL", "SQLEXCEPTION", "SQLSTATE", "SQLWARNING", "SQL_BIGINT", "SQL_BINARY", "SQL_BIT",
+ "SQL_BLOB", "SQL_BOOLEAN", "SQL_CHAR", "SQL_CLOB", "SQL_DATE", "SQL_DECIMAL", "SQL_DOUBLE", "SQL_FLOAT",
+ "SQL_INTEGER", "SQL_INTERVAL_DAY", "SQL_INTERVAL_DAY_TO_HOUR", "SQL_INTERVAL_DAY_TO_MINUTE",
+ "SQL_INTERVAL_DAY_TO_SECOND", "SQL_INTERVAL_HOUR", "SQL_INTERVAL_HOUR_TO_MINUTE",
+ "SQL_INTERVAL_HOUR_TO_SECOND", "SQL_INTERVAL_MINUTE", "SQL_INTERVAL_MINUTE_TO_SECOND", "SQL_INTERVAL_MONTH",
+ "SQL_INTERVAL_SECOND", "SQL_INTERVAL_YEAR", "SQL_INTERVAL_YEAR_TO_MONTH", "SQL_LONGVARBINARY",
+ "SQL_LONGVARCHAR", "SQL_LONGVARNCHAR", "SQL_NCHAR", "SQL_NCLOB", "SQL_NUMERIC", "SQL_NVARCHAR", "SQL_REAL",
+ "SQL_SMALLINT", "SQL_TIME", "SQL_TIMESTAMP", "SQL_TINYINT", "SQL_TSI_DAY", "SQL_TSI_FRAC_SECOND",
+ "SQL_TSI_HOUR", "SQL_TSI_MICROSECOND", "SQL_TSI_MINUTE", "SQL_TSI_MONTH", "SQL_TSI_QUARTER",
+ "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SQL_VARBINARY", "SQL_VARCHAR", "SQRT", "START", "STATE",
+ "STATEMENT", "STATIC", "STDDEV_POP", "STDDEV_SAMP", "STREAM", "STRUCTURE", "STYLE", "SUBCLASS_ORIGIN",
+ "SUBMULTISET", "SUBSET", "SUBSTITUTE", "SUBSTRING", "SUBSTRING_REGEX", "SUCCEEDS", "SUM", "SYMMETRIC",
+ "SYSTEM", "SYSTEM_TIME", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TABLE_NAME", "TEMPORARY", "THEN", "TIES",
+ "TIME", "TIMESTAMP", "TIMESTAMPADD", "TIMESTAMPDIFF", "TIMEZONE_HOUR", "TIMEZONE_MINUTE", "TINYINT", "TO",
+ "TOP_LEVEL_COUNT", "TRAILING", "TRANSACTION", "TRANSACTIONS_ACTIVE", "TRANSACTIONS_COMMITTED",
+ "TRANSACTIONS_ROLLED_BACK", "TRANSFORM", "TRANSFORMS", "TRANSLATE", "TRANSLATE_REGEX", "TRANSLATION",
+ "TREAT", "TRIGGER", "TRIGGER_CATALOG", "TRIGGER_NAME", "TRIGGER_SCHEMA", "TRIM", "TRIM_ARRAY", "TRUE",
+ "TRUNCATE", "TYPE", "UESCAPE", "UNBOUNDED", "UNCOMMITTED", "UNDER", "UNION", "UNIQUE", "UNKNOWN", "UNNAMED",
+ "UNNEST", "UPDATE", "UPPER", "UPSERT", "USAGE", "USER", "USER_DEFINED_TYPE_CATALOG",
+ "USER_DEFINED_TYPE_CODE", "USER_DEFINED_TYPE_NAME", "USER_DEFINED_TYPE_SCHEMA", "USING", "VALUE", "VALUES",
+ "VALUE_OF", "VARBINARY", "VARCHAR", "VARYING", "VAR_POP", "VAR_SAMP", "VERSION", "VERSIONING", "VIEW",
+ "WEEK", "WHEN", "WHENEVER", "WHERE", "WIDTH_BUCKET", "WINDOW", "WITH", "WITHIN", "WITHOUT", "WORK",
+ "WRAPPER", "WRITE", "XML", "YEAR", "ZONE");
+
+ public static String replaceString(String originString, String fromString, String toString) {
+ return originString.replace(fromString, toString);
+ }
+
+ public static String extractReplace(String originString) {
+ Matcher extractMatcher = EXTRACT_PATTERN.matcher(originString);
+ String replacedString = originString;
+ Map<Integer, Integer> parenthesesPairs = null;
+
+ while (extractMatcher.find()) {
+ if (parenthesesPairs == null) {
+ parenthesesPairs = findParenthesesPairs(originString);
+ }
+
+ String functionStr = extractMatcher.group(2);
+ int startIdx = extractMatcher.end(3);
+ int endIdx = parenthesesPairs.get(extractMatcher.start(1));
+ String extractInner = originString.substring(startIdx, endIdx);
+ int originStart = extractMatcher.start(0);
+ int originEnd = endIdx + 1;
+
+ replacedString = replaceString(replacedString, originString.substring(originStart, originEnd),
+ functionStr + "(" + extractInner + ")");
+ }
+
+ return replacedString;
+ }
+
+ public static String castReplace(String originString) {
+ Matcher castMatcher = CAST_PATTERN.matcher(originString);
+ String replacedString = originString;
+
+ while (castMatcher.find()) {
+ String castStr = castMatcher.group();
+ String type = castMatcher.group(2);
+ String supportedType = "";
+ switch (type.toUpperCase()) {
+ case "INTEGER":
+ supportedType = "int";
+ break;
+ case "SHORT":
+ supportedType = "smallint";
+ break;
+ case "LONG":
+ supportedType = "bigint";
+ break;
+ default:
+ supportedType = type;
+ }
+
+ if (!supportedType.equals(type)) {
+ String replacedCastStr = castStr.replace(type, supportedType);
+ replacedString = replaceString(replacedString, castStr, replacedCastStr);
+ }
+ }
+
+ return replacedString;
+ }
+
+ public static String subqueryReplace(String originString) {
+ Matcher subqueryMatcher = FROM_PATTERN.matcher(originString);
+ String replacedString = originString;
+ Map<Integer, Integer> parenthesesPairs = null;
+
+ while (subqueryMatcher.find()) {
+ if (parenthesesPairs == null) {
+ parenthesesPairs = findParenthesesPairs(originString);
+ }
+
+ int startIdx = subqueryMatcher.start(1);
+ int endIdx = parenthesesPairs.get(startIdx) + 1;
+
+ Matcher aliasMatcher = ALIAS_PATTERN.matcher(originString.substring(endIdx));
+ if (aliasMatcher.find()) {
+ String aliasCandidate = aliasMatcher.group(1);
+
+ if (aliasCandidate != null && !sqlKeyWordsExceptAS.contains(aliasCandidate.toUpperCase())) {
+ continue;
+ }
+
+ replacedString = replaceString(replacedString, originString.substring(startIdx, endIdx),
+ originString.substring(startIdx, endIdx) + " as alias");
+ }
+ }
+
+ return replacedString;
+ }
+
+ public static String timestampaddReplace(String originString) {
+ Matcher timestampaddMatcher = TIMESTAMPADD_PATTERN.matcher(originString);
+ String replacedString = originString;
+
+ while (timestampaddMatcher.find()) {
+ String interval = timestampaddMatcher.group(1);
+ String timestampaddStr = replaceString(timestampaddMatcher.group(), interval, "'" + interval + "'");
+ replacedString = replaceString(replacedString, timestampaddMatcher.group(), timestampaddStr);
+ }
+
+ return replacedString;
+ }
+
+ public static String concatReplace(String originString) {
+ Matcher concatMatcher = CONCAT_PATTERN.matcher(originString);
+ String replacedString = originString;
+
+ while (concatMatcher.find()) {
+ String leftString = concatMatcher.group(1);
+ String rightString = concatMatcher.group(2);
+ replacedString = replaceString(replacedString, leftString + "||" + rightString,
+ "concat(" + leftString + "," + rightString + ")");
+ }
+
+ return replacedString;
+ }
+
+ public static String doConvert(String originStr) {
+ // Step1.Replace " with `
+ String convertedSql = replaceString(originStr, "\"", "`");
+
+ // Step2.Replace extract functions
+ convertedSql = extractReplace(convertedSql);
+
+ // Step3.Replace cast type string
+ convertedSql = castReplace(convertedSql);
+
+ // Step4.Replace sub query
+ convertedSql = subqueryReplace(convertedSql);
+
+ // Step5.Replace char_length with length
+ convertedSql = replaceString(convertedSql, "char_length", "length");
+
+ // Step6.Replace "||" with concat
+ convertedSql = concatReplace(convertedSql);
+
+ // Step7.Add quote for interval in timestampadd
+ convertedSql = timestampaddReplace(convertedSql);
+
+ return convertedSql;
+ }
+
+ private static Map<Integer, Integer> findParenthesesPairs(String sql) {
+ Map<Integer, Integer> result = new HashMap<>();
+ if (sql.length() > 1) {
+ Stack<Integer> lStack = new Stack<>();
+ boolean inStrVal = false;
+ for (int i = 0; i < sql.length(); i++) {
+ switch (sql.charAt(i)) {
+ case '(':
+ if (!inStrVal) {
+ lStack.push(i);
+ }
+ break;
+ case ')':
+ if (!inStrVal && !lStack.empty()) {
+ result.put(lStack.pop(), i);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public String convert(String originSql) {
+ return doConvert(originSql);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java
deleted file mode 100644
index c4b87f8..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.source.adhocquery;
-
-/**
- * convert the query to satisfy the parser of adhoc query engine
- */
-public interface IAdHocConverter {
- String convert(String originSql);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java
deleted file mode 100644
index 369325c..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.source.adhocquery;
-
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
-
-public interface IAdHocRunner {
-
- void init(KylinConfig config);
-
- /**
- * Run an ad-hoc query in the source database in case Kylin cannot serve using cube.
- *
- * @param query the query statement
- * @param returnRows an empty list to collect returning rows
- * @param returnColumnMeta an empty list to collect metadata of returning columns
- * @throws Exception if running ad-hoc query fails
- */
- void executeQuery(String query, List<List<String>> returnRows, List<SelectedColumnMeta> returnColumnMeta) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java
new file mode 100644
index 0000000..c4e7515
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.source.adhocquery;
+
+/**
+ * convert the query to satisfy the parser of push down query engine
+ */
+public interface IPushDownConverter {
+ String convert(String originSql);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
new file mode 100644
index 0000000..c8d18aa
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.adhocquery;
+
+import java.util.List;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
+
+public interface IPushDownRunner {
+
+ void init(KylinConfig config);
+
+ /**
+ * Run an pushdown query in the source database in case Kylin cannot serve using cube.
+ *
+ * @param query the query statement
+ * @param returnRows an empty list to collect returning rows
+ * @param returnColumnMeta an empty list to collect metadata of returning columns
+ * @throws Exception if running pushdown query fails
+ */
+ void executeQuery(String query, List<List<String>> returnRows, List<SelectedColumnMeta> returnColumnMeta) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java
deleted file mode 100644
index cfb0f32..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.adhocquery;
-
-import org.junit.Test;
-
-import junit.framework.TestCase;
-
-public class HiveAdhocConverterTest extends TestCase {
- @Test
- public void testStringReplace() {
- String originString = "select count(*) as cnt from test_kylin_fact where char_length(lstg_format_name) < 10";
- String replacedString = HiveAdhocConverter.replaceString(originString, "char_length", "length");
- assertEquals("select count(*) as cnt from test_kylin_fact where length(lstg_format_name) < 10", replacedString);
- }
-
- @Test
- public void testExtractReplace() {
- String originString = "ignore EXTRACT(YEAR FROM KYLIN_CAL_DT.CAL_DT) ignore";
- String replacedString = HiveAdhocConverter.extractReplace(originString);
- assertEquals("ignore YEAR(KYLIN_CAL_DT.CAL_DT) ignore", replacedString);
- }
-
- @Test
- public void testCastReplace() {
- String originString = "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS INTEGER)) ignore";
- String replacedString = HiveAdhocConverter.castReplace(originString);
- assertEquals("ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS int)) ignore", replacedString);
- }
-
- @Test
- public void testSubqueryReplace1() {
- String originString = "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) group by seller_id,lstg_format_name";
- String replacedString = HiveAdhocConverter.subqueryReplace(originString);
- assertEquals(
- "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) as alias group by seller_id,lstg_format_name",
- replacedString);
- }
-
- @Test
- public void testSubqueryReplace2() {
- String originString = "select count(*) from ( select test_kylin_fact.lstg_format_name from test_kylin_fact where test_kylin_fact.lstg_format_name='FP-GTC' group by test_kylin_fact.lstg_format_name ) t ";
- String replacedString = HiveAdhocConverter.subqueryReplace(originString);
- assertEquals(originString, replacedString);
- }
-
- @Test
- public void testSubqueryReplace3() {
- String originString = "select fact.lstg_format_name from (select * from test_kylin_fact where cal_dt > date'2010-01-01' ) as fact group by fact.lstg_format_name order by CASE WHEN fact.lstg_format_name IS NULL THEN 'sdf' ELSE fact.lstg_format_name END ";
- String replacedString = HiveAdhocConverter.subqueryReplace(originString);
- assertEquals(originString, replacedString);
- }
-
- @Test
- public void testConcatReplace() {
- String originString = "select count(*) as cnt from test_kylin_fact where lstg_format_name||'a'='ABINa'";
- String replacedString = HiveAdhocConverter.concatReplace(originString);
- assertEquals("select count(*) as cnt from test_kylin_fact where concat(lstg_format_name,'a')='ABINa'",
- replacedString);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java
new file mode 100644
index 0000000..42c628a
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.adhocquery;
+
+import org.junit.Test;
+
+import junit.framework.TestCase;
+
+public class HivePushDownConverterTest extends TestCase {
+ @Test
+ public void testStringReplace() {
+ String originString = "select count(*) as cnt from test_kylin_fact where char_length(lstg_format_name) < 10";
+ String replacedString = HivePushDownConverter.replaceString(originString, "char_length", "length");
+ assertEquals("select count(*) as cnt from test_kylin_fact where length(lstg_format_name) < 10", replacedString);
+ }
+
+ @Test
+ public void testExtractReplace() {
+ String originString = "ignore EXTRACT(YEAR FROM KYLIN_CAL_DT.CAL_DT) ignore";
+ String replacedString = HivePushDownConverter.extractReplace(originString);
+ assertEquals("ignore YEAR(KYLIN_CAL_DT.CAL_DT) ignore", replacedString);
+ }
+
+ @Test
+ public void testCastReplace() {
+ String originString = "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS INTEGER)) ignore";
+ String replacedString = HivePushDownConverter.castReplace(originString);
+ assertEquals("ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS int)) ignore", replacedString);
+ }
+
+ @Test
+ public void testSubqueryReplace1() {
+ String originString = "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) group by seller_id,lstg_format_name";
+ String replacedString = HivePushDownConverter.subqueryReplace(originString);
+ assertEquals(
+ "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) as alias group by seller_id,lstg_format_name",
+ replacedString);
+ }
+
+ @Test
+ public void testSubqueryReplace2() {
+ String originString = "select count(*) from ( select test_kylin_fact.lstg_format_name from test_kylin_fact where test_kylin_fact.lstg_format_name='FP-GTC' group by test_kylin_fact.lstg_format_name ) t ";
+ String replacedString = HivePushDownConverter.subqueryReplace(originString);
+ assertEquals(originString, replacedString);
+ }
+
+ @Test
+ public void testSubqueryReplace3() {
+ String originString = "select fact.lstg_format_name from (select * from test_kylin_fact where cal_dt > date'2010-01-01' ) as fact group by fact.lstg_format_name order by CASE WHEN fact.lstg_format_name IS NULL THEN 'sdf' ELSE fact.lstg_format_name END ";
+ String replacedString = HivePushDownConverter.subqueryReplace(originString);
+ assertEquals(originString, replacedString);
+ }
+
+ @Test
+ public void testConcatReplace() {
+ String originString = "select count(*) as cnt from test_kylin_fact where lstg_format_name||'a'='ABINa'";
+ String replacedString = HivePushDownConverter.concatReplace(originString);
+ assertEquals("select count(*) as cnt from test_kylin_fact where concat(lstg_format_name,'a')='ABINa'",
+ replacedString);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 35386de..6a571df 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -186,14 +186,14 @@ kylin.engine.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current
kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current
-### AD-HOC QUERY ###
-#kylin.query.ad-hoc.runner-class-name=org.apache.kylin.query.adhoc.AdHocRunnerJdbcImpl
+### QUERY PUSH DOWN ###
+#kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl
-#kylin.query.ad-hoc.jdbc.url=jdbc:hive2://sandbox:10000/default
-#kylin.query.ad-hoc.jdbc.driver=org.apache.hive.jdbc.HiveDriver
-#kylin.query.ad-hoc.jdbc.username=hive
-#kylin.query.ad-hoc.jdbc.password=
+#kylin.query.pushdown.jdbc.url=jdbc:hive2://sandbox:10000/default
+#kylin.query.pushdown.jdbc.driver=org.apache.hive.jdbc.HiveDriver
+#kylin.query.pushdown.jdbc.username=hive
+#kylin.query.pushdown.jdbc.password=
-#kylin.query.ad-hoc.jdbc.pool-max-total=8
-#kylin.query.ad-hoc.jdbc.pool-max-idle=8
-#kylin.query.ad-hoc.jdbc.pool-min-idle=0
+#kylin.query.pushdown.jdbc.pool-max-total=8
+#kylin.query.pushdown.jdbc.pool-max-idle=8
+#kylin.query.pushdown.jdbc.pool-min-idle=0
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index af05237..3634100 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -51,7 +51,7 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
-import org.apache.kylin.rest.util.AdHocUtil;
+import org.apache.kylin.rest.util.PushDownUtil;
import org.dbunit.DatabaseUnitException;
import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
@@ -263,7 +263,7 @@ public class KylinTestBase {
} catch (SQLException sqlException) {
List<List<String>> results = Lists.newArrayList();
List<SelectedColumnMeta> columnMetas = Lists.newArrayList();
- AdHocUtil.doAdHocQuery(ProjectInstance.DEFAULT_PROJECT_NAME, sql, results, columnMetas, sqlException);
+ PushDownUtil.doPushDownQuery(ProjectInstance.DEFAULT_PROJECT_NAME, sql, results, columnMetas, sqlException);
return results.size();
} finally {
if (resultSet != null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java
deleted file mode 100644
index 852cebf..0000000
--- a/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.adhoc;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.pool.impl.GenericObjectPool;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
-import org.apache.kylin.source.adhocquery.IAdHocRunner;
-
-public class AdHocRunnerJdbcImpl implements IAdHocRunner {
-
- private static org.apache.kylin.query.adhoc.JdbcConnectionPool pool = null;
-
- @Override
- public void init(KylinConfig config) {
- if (pool == null) {
- pool = new JdbcConnectionPool();
- JdbcConnectionFactory factory = new JdbcConnectionFactory(config.getJdbcUrl(), config.getJdbcDriverClass(),
- config.getJdbcUsername(), config.getJdbcPassword());
- GenericObjectPool.Config poolConfig = new GenericObjectPool.Config();
- poolConfig.maxActive = config.getPoolMaxTotal();
- poolConfig.maxIdle = config.getPoolMaxIdle();
- poolConfig.minIdle = config.getPoolMinIdle();
-
- try {
- pool.createPool(factory, poolConfig);
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage(), e);
- }
- }
- }
-
- @Override
- public void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas)
- throws Exception {
- Statement statement = null;
- Connection connection = this.getConnection();
- ResultSet resultSet = null;
-
- try {
- statement = connection.createStatement();
- resultSet = statement.executeQuery(query);
- extractResults(resultSet, results);
- } catch (SQLException sqlException) {
- throw sqlException;
- }
-
- //extract column metadata
- ResultSetMetaData metaData = null;
- int columnCount = 0;
- try {
- metaData = resultSet.getMetaData();
- columnCount = metaData.getColumnCount();
-
- // fill in selected column meta
- for (int i = 1; i <= columnCount; ++i) {
- columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false,
- metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i),
- metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null,
- metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i),
- metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false));
- }
-
- } catch (SQLException sqlException) {
- throw sqlException;
- }
-
- closeConnection(connection);
- }
-
- private Connection getConnection() {
- return pool.getConnection();
- }
-
- private void closeConnection(Connection connection) {
- pool.returnConnection(connection);
- }
-
- static void extractResults(ResultSet resultSet, List<List<String>> results) throws SQLException {
- List<String> oneRow = new LinkedList<String>();
-
- try {
- while (resultSet.next()) {
- //logger.debug("resultSet value: " + resultSet.getString(1));
- for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) {
- oneRow.add((resultSet.getString(i + 1)));
- }
-
- results.add(new LinkedList<String>(oneRow));
- oneRow.clear();
- }
- } catch (SQLException sqlException) {
- throw sqlException;
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
new file mode 100644
index 0000000..8001880
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.adhoc;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.pool.impl.GenericObjectPool;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
+import org.apache.kylin.source.adhocquery.IPushDownRunner;
+
+public class PushDownRunnerJdbcImpl implements IPushDownRunner {
+
+ private static org.apache.kylin.query.adhoc.JdbcConnectionPool pool = null;
+
+ @Override
+ public void init(KylinConfig config) {
+ if (pool == null) {
+ pool = new JdbcConnectionPool();
+ JdbcConnectionFactory factory = new JdbcConnectionFactory(config.getJdbcUrl(), config.getJdbcDriverClass(),
+ config.getJdbcUsername(), config.getJdbcPassword());
+ GenericObjectPool.Config poolConfig = new GenericObjectPool.Config();
+ poolConfig.maxActive = config.getPoolMaxTotal();
+ poolConfig.maxIdle = config.getPoolMaxIdle();
+ poolConfig.minIdle = config.getPoolMinIdle();
+
+ try {
+ pool.createPool(factory, poolConfig);
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
+ }
+
+ @Override
+ public void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas)
+ throws Exception {
+ Statement statement = null;
+ Connection connection = this.getConnection();
+ ResultSet resultSet = null;
+
+ try {
+ statement = connection.createStatement();
+ resultSet = statement.executeQuery(query);
+ extractResults(resultSet, results);
+ } catch (SQLException sqlException) {
+ throw sqlException;
+ }
+
+ //extract column metadata
+ ResultSetMetaData metaData = null;
+ int columnCount = 0;
+ try {
+ metaData = resultSet.getMetaData();
+ columnCount = metaData.getColumnCount();
+
+ // fill in selected column meta
+ for (int i = 1; i <= columnCount; ++i) {
+ columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false,
+ metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i),
+ metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null,
+ metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i),
+ metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false));
+ }
+
+ } catch (SQLException sqlException) {
+ throw sqlException;
+ }
+
+ closeConnection(connection);
+ }
+
+ private Connection getConnection() {
+ return pool.getConnection();
+ }
+
+ private void closeConnection(Connection connection) {
+ pool.returnConnection(connection);
+ }
+
+ static void extractResults(ResultSet resultSet, List<List<String>> results) throws SQLException {
+ List<String> oneRow = new LinkedList<String>();
+
+ try {
+ while (resultSet.next()) {
+ //logger.debug("resultSet value: " + resultSet.getString(1));
+ for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) {
+ oneRow.add((resultSet.getString(i + 1)));
+ }
+
+ results.add(new LinkedList<String>(oneRow));
+ oneRow.clear();
+ }
+ } catch (SQLException sqlException) {
+ throw sqlException;
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index d841dee..79a2c05 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@ -61,12 +61,13 @@ public class SQLResponse implements Serializable {
protected boolean storageCacheUsed = false;
- protected boolean queryAdHoc = false;
+ protected boolean queryPushDown = false;
public SQLResponse() {
}
- public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, int affectedRowCount, boolean isException, String exceptionMessage) {
+ public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, int affectedRowCount,
+ boolean isException, String exceptionMessage) {
this.columnMetas = columnMetas;
this.results = results;
this.affectedRowCount = affectedRowCount;
@@ -74,7 +75,8 @@ public class SQLResponse implements Serializable {
this.exceptionMessage = exceptionMessage;
}
- public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube, int affectedRowCount, boolean isException, String exceptionMessage) {
+ public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube,
+ int affectedRowCount, boolean isException, String exceptionMessage) {
this.columnMetas = columnMetas;
this.results = results;
this.cube = cube;
@@ -83,7 +85,8 @@ public class SQLResponse implements Serializable {
this.exceptionMessage = exceptionMessage;
}
- public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube, int affectedRowCount, boolean isException, String exceptionMessage, boolean isPartial, boolean isAdhoc) {
+ public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube,
+ int affectedRowCount, boolean isException, String exceptionMessage, boolean isPartial, boolean isPushDown) {
this.columnMetas = columnMetas;
this.results = results;
this.cube = cube;
@@ -91,7 +94,7 @@ public class SQLResponse implements Serializable {
this.isException = isException;
this.exceptionMessage = exceptionMessage;
this.isPartial = isPartial;
- this.queryAdHoc = isAdhoc;
+ this.queryPushDown = isPushDown;
}
public List<SelectedColumnMeta> getColumnMetas() {
@@ -147,8 +150,8 @@ public class SQLResponse implements Serializable {
return isPartial;
}
- public boolean isAdHoc() {
- return queryAdHoc;
+ public boolean isPushDown() {
+ return queryPushDown;
}
public long getTotalScanCount() {
[08/26] kylin git commit: minor, refine acl
Posted by li...@apache.org.
minor, refine acl
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c490e6cf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c490e6cf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c490e6cf
Branch: refs/heads/kylin-2.1.x
Commit: c490e6cf7c01e0e1443fa93fac681300f96680d6
Parents: 8e1a7d5
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 16:34:52 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 16:36:57 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/CubeController.java | 15 ++-
.../rest/controller2/AccessControllerV2.java | 16 ++-
.../kylin/rest/service/AccessService.java | 26 +++--
.../apache/kylin/rest/service/CubeService.java | 109 +++++++++++++++----
4 files changed, 131 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c490e6cf/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 6916dd8..a370292 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -352,7 +352,7 @@ public class CubeController extends BasicController {
@ResponseBody
public CubeInstance cloneCube(@PathVariable String cubeName, @RequestBody CubeRequest cubeRequest) {
String newCubeName = cubeRequest.getCubeName();
- String project = cubeRequest.getProject();
+ String projectName = cubeRequest.getProject();
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
if (cube == null) {
@@ -366,6 +366,11 @@ public class CubeController extends BasicController {
throw new BadRequestException("Invalid Cube name, only letters, numbers and underline supported.");
}
+ ProjectInstance project = cubeService.getProjectManager().getProject(projectName);
+ if (project == null) {
+ throw new BadRequestException("Project " + projectName + " doesn't exist");
+ }
+
CubeDesc cubeDesc = cube.getDescriptor();
CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
@@ -373,7 +378,7 @@ public class CubeController extends BasicController {
CubeInstance newCube;
try {
- newCube = cubeService.createCubeAndDesc(newCubeName, project, newCubeDesc);
+ newCube = cubeService.createCubeAndDesc(project, newCubeDesc);
//reload to avoid shallow clone
cubeService.getCubeDescManager().reloadCubeDescLocal(newCubeName);
@@ -449,7 +454,11 @@ public class CubeController extends BasicController {
try {
desc.setUuid(UUID.randomUUID().toString());
String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
- cubeService.createCubeAndDesc(name, projectName, desc);
+ ProjectInstance project = cubeService.getProjectManager().getProject(projectName);
+ if (project == null) {
+ throw new BadRequestException("Project " + projectName + " doesn't exist");
+ }
+ cubeService.createCubeAndDesc(project, desc);
} catch (Exception e) {
logger.error("Failed to deal with the request.", e);
throw new InternalErrorException(e.getLocalizedMessage(), e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c490e6cf/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
index 255e312..a61e321 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
@@ -58,7 +58,6 @@ public class AccessControllerV2 extends BasicController {
* @return
* @throws IOException
*/
-
@RequestMapping(value = "/{type}/{uuid}", method = { RequestMethod.GET }, produces = {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
@@ -70,6 +69,21 @@ public class AccessControllerV2 extends BasicController {
}
/**
+ * List access entry list of a domain object including its parent
+ * @param type
+ * @param uuid
+ * @return
+ */
+ @RequestMapping(value = "all/{type}/{uuid}", method = { RequestMethod.GET }, produces = {
+ "application/vnd.apache.kylin-v2+json" })
+ @ResponseBody
+ public EnvelopeResponse listAccessEntitiesV2(@PathVariable String type, @PathVariable String uuid) {
+ AclEntity ae = accessService.getAclEntity(type, uuid);
+ Acl acl = accessService.getAcl(ae);
+ return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, accessService.generateListAceResponses(acl), "");
+ }
+
+ /**
* Grant a new access on a domain object to a user/role
*
* @param accessRequest
http://git-wip-us.apache.org/repos/asf/kylin/blob/c490e6cf/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index e5cd793..3a6d723 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -20,9 +20,7 @@ package org.apache.kylin.rest.service;
import java.util.ArrayList;
import java.util.List;
-import java.util.Set;
-import com.google.common.collect.Sets;
import org.apache.kylin.common.persistence.AclEntity;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.rest.constant.Constant;
@@ -300,16 +298,9 @@ public class AccessService {
public List<AccessEntryResponse> generateAceResponses(Acl acl) {
List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
- Set<Sid> sidSet = Sets.newHashSet();
- while (acl != null) {
- for (AccessControlEntry ace : acl.getEntries()) {
- if (!sidSet.contains(ace.getSid())) {
- result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
- sidSet.add(ace.getSid());
- }
- }
- acl = acl.getParentAcl();
+ for (AccessControlEntry ace : acl.getEntries()) {
+ result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
}
return result;
@@ -330,4 +321,17 @@ public class AccessService {
throw new ForbiddenException(msg.getREVOKE_ADMIN_PERMISSION());
}
}
+
+ public Object generateListAceResponses(Acl acl) {
+ List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
+
+ while (acl != null) {
+ for (AccessControlEntry ace : acl.getEntries()) {
+ result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
+ }
+ acl = acl.getParentAcl();
+ }
+
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c490e6cf/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a5fc36a..b2e44f6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -58,6 +58,7 @@ import org.apache.kylin.rest.request.MetricsRequest;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.security.AclPermission;
+import org.apache.kylin.rest.util.AclUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -96,6 +97,9 @@ public class CubeService extends BasicService {
@Qualifier("modelMgmtService")
private ModelService modelService;
+ @Autowired
+ private AclUtil aclUtil;
+
@PostFilter(Constant.ACCESS_POST_FILTER_READ)
public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName, boolean exactMatch) {
List<CubeInstance> cubeInstances = null;
@@ -153,9 +157,11 @@ public class CubeService extends BasicService {
return getCubeManager().updateCube(cubeBuilder);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
- public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeDesc desc) throws IOException {
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
+ public CubeInstance createCubeAndDesc(ProjectInstance project, CubeDesc desc) throws IOException {
Message msg = MsgPicker.getMsg();
+ String cubeName = desc.getName();
if (getCubeManager().getCube(cubeName) != null) {
throw new BadRequestException(String.format(msg.getCUBE_ALREADY_EXIST(), cubeName));
@@ -178,10 +184,9 @@ public class CubeService extends BasicService {
int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc, false);
logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids");
- createdCube = getCubeManager().createCube(cubeName, projectName, createdDesc, owner);
+ createdCube = getCubeManager().createCube(cubeName, project.getName(), createdDesc, owner);
accessService.init(createdCube, AclPermission.ADMINISTRATION);
- ProjectInstance project = getProjectManager().getProject(projectName);
accessService.inherit(createdCube, project);
return createdCube;
@@ -611,32 +616,69 @@ public class CubeService extends BasicService {
}
}
- public CubeDesc updateCubeToResourceStore(CubeDesc desc, String projectName) throws IOException {
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
+ public CubeDesc saveCube(CubeDesc desc, ProjectInstance project) throws IOException {
Message msg = MsgPicker.getMsg();
desc.setDraft(false);
if (desc.getUuid() == null)
desc.updateRandomUuid();
- String cubeName = desc.getName();
try {
- if (desc.getLastModified() == 0) {
- // new
- createCubeAndDesc(cubeName, projectName, desc);
- } else {
- // update
- CubeInstance cube = getCubeManager().getCube(desc.getName());
+ createCubeAndDesc(project, desc);
+ } catch (AccessDeniedException accessDeniedException) {
+ throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
+ }
- if (cube == null) {
- throw new BadRequestException(String.format(msg.getCUBE_NOT_FOUND(), desc.getName()));
- }
+ if (!desc.getError().isEmpty()) {
+ throw new BadRequestException(desc.getErrorMsg());
+ }
- if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) {
- throw new BadRequestException(String.format(msg.getINCONSISTENT_CUBE_DESC(), desc.getName()));
- }
+ return desc;
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+ public void saveDraft(ProjectInstance project, CubeInstance cube, String uuid, RootPersistentEntity... entities)
+ throws IOException {
+ Draft draft = new Draft();
+ draft.setProject(project.getName());
+ draft.setUuid(uuid);
+ draft.setEntities(entities);
+ getDraftManager().save(draft);
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
+ public void saveDraft(ProjectInstance project, String uuid, RootPersistentEntity... entities) throws IOException {
+ Draft draft = new Draft();
+ draft.setProject(project.getName());
+ draft.setUuid(uuid);
+ draft.setEntities(entities);
+ getDraftManager().save(draft);
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#draft, 'ADMINISTRATION') or hasPermission(#draft, 'MANAGEMENT')")
+ public void deleteDraft(Draft draft) throws IOException {
+ getDraftManager().delete(draft.getUuid());
+ }
- desc = updateCubeAndDesc(cube, desc, projectName, true);
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+ public CubeDesc updateCube(CubeInstance cube, CubeDesc desc, ProjectInstance project) throws IOException {
+ Message msg = MsgPicker.getMsg();
+ String projectName = project.getName();
+
+ desc.setDraft(false);
+
+ try {
+ if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) {
+ throw new BadRequestException(String.format(msg.getINCONSISTENT_CUBE_DESC(), desc.getName()));
}
+
+ desc = updateCubeAndDesc(cube, desc, projectName, true);
} catch (AccessDeniedException accessDeniedException) {
throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
}
@@ -671,6 +713,33 @@ public class CubeService extends BasicService {
}
}
- return result;
+ List<Draft> filtered = new ArrayList<>();
+
+ // if cube's there, follow cube permission. otherwise follow project permission
+ for (Draft d : result) {
+ CubeDesc desc = (CubeDesc) d.getEntity();
+ CubeInstance cube = getCubeManager().getCube(desc.getName());
+
+ if (cube == null) {
+ try {
+ project = project == null ? d.getProject() : project;
+ if (aclUtil.hasProjectReadPermission(getProjectManager().getProject(project))) {
+ filtered.add(d);
+ }
+ } catch (Exception e) {
+ // do nothing
+ }
+ } else {
+ try {
+ if (aclUtil.hasCubeReadPermission(cube)) {
+ filtered.add(d);
+ }
+ } catch (Exception e) {
+ // do nothing
+ }
+ }
+ }
+
+ return filtered;
}
}
[18/26] kylin git commit: minor,
validate join key should be a dimension
Posted by li...@apache.org.
minor, validate join key should be a dimension
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6228f886
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6228f886
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6228f886
Branch: refs/heads/kylin-2.1.x
Commit: 6228f886ecf697a41fd658b6eb46dde149cd0e2a
Parents: cf1ba95
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Jun 30 18:09:18 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 18:20:45 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 31 ++++++++++++++++++++
.../localmeta/model_desc/ssb.json | 20 ++++++-------
2 files changed, 41 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6228f886/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index bc35e2a..abee71b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -617,7 +617,38 @@ public class DataModelDesc extends RootPersistentEntity {
+ fkCols[i].getDatatype());
}
}
+
+ if (!isSpecialTestModel()) {
+ for (TblColRef t : fkCols) {
+ if (!isJoinKeyDimension(t.getIdentity())) {
+ throw new IllegalStateException(
+ "Foreigner key: " + t.getIdentity() + " should be a dimension.");
+ }
+ }
+
+ for (TblColRef t : pkCols) {
+ if (!isJoinKeyDimension(t.getIdentity())) {
+ throw new IllegalStateException("Primary key: " + t.getIdentity() + " should be a dimension.");
+ }
+ }
+ }
+ }
+ }
+
+ private boolean isJoinKeyDimension(String key) {
+ for (ModelDimensionDesc d : dimensions) {
+ for (String col : d.getColumns()) {
+ if (key.equals(d.getTable() + '.' + col))
+ return true;
+ }
}
+ return false;
+ }
+
+ private boolean isSpecialTestModel() {
+ if (config.isDevEnv() && (name.startsWith("test_kylin_") || name.startsWith("test_streaming")))
+ return true;
+ return false;
}
private void initJoinsTree() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6228f886/examples/test_case_data/localmeta/model_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ssb.json b/examples/test_case_data/localmeta/model_desc/ssb.json
index 52b330f..bbac99d 100644
--- a/examples/test_case_data/localmeta/model_desc/ssb.json
+++ b/examples/test_case_data/localmeta/model_desc/ssb.json
@@ -33,20 +33,20 @@
}
} ],
"dimensions" : [ {
- "table" : "SSB.V_LINEORDER",
- "columns" : [ "LO_DATE" ]
+ "table" : "V_LINEORDER",
+ "columns" : [ "LO_DATE", "LO_PARTKEY", "LO_CUSTKEY", "LO_ORDERDATE", "LO_SUPPKEY" ]
}, {
- "table" : "SSB.PART",
- "columns" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+ "table" : "PART",
+ "columns" : [ "P_MFGR", "P_CATEGORY", "P_BRAND", "P_PARTKEY" ]
}, {
- "table" : "SSB.CUSTOMER",
- "columns" : [ "C_CITY", "C_NATION", "C_REGION" ]
+ "table" : "CUSTOMER",
+ "columns" : [ "C_CITY", "C_NATION", "C_REGION", "C_CUSTKEY" ]
}, {
- "table" : "SSB.SUPPLIER",
- "columns" : [ "S_CITY", "S_NATION", "S_REGION" ]
+ "table" : "SUPPLIER",
+ "columns" : [ "S_CITY", "S_NATION", "S_REGION", "S_SUPPKEY" ]
}, {
- "table" : "SSB.DATES",
- "columns" : [ "D_YEAR", "D_YEARMONTHNUM", "D_YEARMONTH", "D_WEEKNUMINYEAR" ]
+ "table" : "DATES",
+ "columns" : [ "D_YEAR", "D_YEARMONTHNUM", "D_YEARMONTH", "D_WEEKNUMINYEAR", "D_DATEKEY" ]
} ],
"metrics" : [ "LO_REVENUE", "LO_SUPPLYCOST", "V_REVENUE" ],
"last_modified" : 1457444314662,
[11/26] kylin git commit: minor, remove unused API in QueryUtil
Posted by li...@apache.org.
minor, remove unused API in QueryUtil
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/40054dd4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/40054dd4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/40054dd4
Branch: refs/heads/kylin-2.1.x
Commit: 40054dd4b263a44bfacf809d614fb9a9d5cce77e
Parents: 052d609
Author: lidongsjtu <li...@apache.org>
Authored: Thu Jun 29 21:29:43 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 21:33:04 2017 +0800
----------------------------------------------------------------------
query/src/main/java/org/apache/kylin/query/util/QueryUtil.java | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/40054dd4/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 7794f94..eff9f6d 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -41,11 +41,6 @@ public class QueryUtil {
String transform(String sql, String project);
}
- // for mockup test
- public static String massageSql(String sql) {
- return massageSql(sql, null, 0, 0);
- }
-
public static String massageSql(String sql, String project, int limit, int offset) {
sql = sql.trim();
sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator"));
[15/26] kylin git commit: minor, make calcite work with unicode
Posted by li...@apache.org.
minor, make calcite work with unicode
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f52eb0c5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f52eb0c5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f52eb0c5
Branch: refs/heads/kylin-2.1.x
Commit: f52eb0c5c667b655d9a20f6226af2a3f5cbe0f4e
Parents: 2546e00
Author: Li Yang <li...@apache.org>
Authored: Fri Jun 30 13:42:21 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 30 13:46:14 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 21 ++++++++++++++++++++
.../apache/kylin/query/ITCombinationTest.java | 3 +--
.../org/apache/kylin/query/KylinTestBase.java | 4 ++--
.../kylin/query/schema/OLAPSchemaFactory.java | 19 ------------------
4 files changed, 24 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 1298807..cc08056 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -28,6 +28,7 @@ import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringReader;
import java.net.URL;
+import java.nio.ByteOrder;
import java.nio.charset.Charset;
import java.util.Enumeration;
import java.util.Map;
@@ -58,6 +59,26 @@ public class KylinConfig extends KylinConfigBase {
// thread-local instances, will override SYS_ENV_INSTANCE
private static transient ThreadLocal<KylinConfig> THREAD_ENV_INSTANCE = new ThreadLocal<>();
+
+ static {
+ /*
+ * Make Calcite to work with Unicode.
+ *
+ * Sets default char set for string literals in SQL and row types of
+ * RelNode. This is more a label used to compare row type equality. For
+ * both SQL string and row record, they are passed to Calcite in String
+ * object and does not require additional codec.
+ *
+ * Ref SaffronProperties.defaultCharset
+ * Ref SqlUtil.translateCharacterSetName()
+ * Ref NlsString constructor()
+ */
+ // copied from org.apache.calcite.util.ConversionUtil.NATIVE_UTF16_CHARSET_NAME
+ String NATIVE_UTF16_CHARSET_NAME = (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) ? "UTF-16BE" : "UTF-16LE";
+ System.setProperty("saffron.default.charset", NATIVE_UTF16_CHARSET_NAME);
+ System.setProperty("saffron.default.nationalcharset", NATIVE_UTF16_CHARSET_NAME);
+ System.setProperty("saffron.default.collation.name", NATIVE_UTF16_CHARSET_NAME + "$en_US");
+ }
public static KylinConfig getInstanceFromEnv() {
synchronized (KylinConfig.class) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 0bed5ed..5051db1 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -18,7 +18,6 @@
package org.apache.kylin.query;
-import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
@@ -43,7 +42,7 @@ public class ITCombinationTest extends ITKylinQueryTest {
private static final Logger logger = LoggerFactory.getLogger(ITCombinationTest.class);
@BeforeClass
- public static void setUp() throws SQLException {
+ public static void setUp() {
Map<RealizationType, Integer> priorities = Maps.newHashMap();
priorities.put(RealizationType.HYBRID, 0);
priorities.put(RealizationType.CUBE, 0);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index a05d0c3..af05237 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -21,9 +21,9 @@ package org.apache.kylin.query;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -166,7 +166,7 @@ public class KylinTestBase {
}
public static String getTextFromFile(File file) throws IOException {
- BufferedReader reader = new BufferedReader(new FileReader(file));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
String line = null;
StringBuilder stringBuilder = new StringBuilder();
String ls = System.getProperty("line.separator");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
index bfe49cf..c098b23 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
@@ -31,7 +31,6 @@ import java.util.Map.Entry;
import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.SchemaFactory;
import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.util.ConversionUtil;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -50,24 +49,6 @@ import com.google.common.collect.Maps;
public class OLAPSchemaFactory implements SchemaFactory {
public static final Logger logger = LoggerFactory.getLogger(OLAPSchemaFactory.class);
- static {
- /*
- * Tricks Optiq to work with Unicode.
- *
- * Sets default char set for string literals in SQL and row types of
- * RelNode. This is more a label used to compare row type equality. For
- * both SQL string and row record, they are passed to Optiq in String
- * object and does not require additional codec.
- *
- * Ref SaffronProperties.defaultCharset
- * Ref SqlUtil.translateCharacterSetName()
- * Ref NlsString constructor()
- */
- System.setProperty("saffron.default.charset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
- System.setProperty("saffron.default.nationalcharset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
- System.setProperty("saffron.default.collation.name", ConversionUtil.NATIVE_UTF16_CHARSET_NAME + "$en_US");
- }
-
private final static String SCHEMA_PROJECT = "project";
@Override
[10/26] kylin git commit: minor, fix ACL NPE
Posted by li...@apache.org.
minor, fix ACL NPE
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/052d6096
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/052d6096
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/052d6096
Branch: refs/heads/kylin-2.1.x
Commit: 052d6096779d5826ba24517b0b4cc9a45b78bf4a
Parents: 8ca0d32
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 17:23:13 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 18:24:43 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/rest/service/AccessService.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/052d6096/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index affa508..ae7ac6e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.service;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.persistence.AclEntity;
@@ -297,8 +298,11 @@ public class AccessService {
}
public List<AccessEntryResponse> generateAceResponses(Acl acl) {
- List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
+ if (null == acl) {
+ return Collections.emptyList();
+ }
+ List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
for (AccessControlEntry ace : acl.getEntries()) {
result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
}
[22/26] kylin git commit: minor, refine acl
Posted by li...@apache.org.
minor, refine acl
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d91f5229
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d91f5229
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d91f5229
Branch: refs/heads/kylin-2.1.x
Commit: d91f522904424f59d817bbcde10c47cc68f04d9d
Parents: 3ae8ca7
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Jun 30 20:52:03 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 30 20:57:18 2017 +0800
----------------------------------------------------------------------
.../hbase/ITAclTableMigrationToolTest.java | 10 ++--
.../rest/controller2/ProjectControllerV2.java | 4 ++
.../org/apache/kylin/rest/msg/CnMessage.java | 2 +-
.../java/org/apache/kylin/rest/msg/Message.java | 2 +-
.../apache/kylin/rest/security/ManagedUser.java | 54 +++++++++++++++++---
.../rest/service/AclTableMigrationTool.java | 2 +-
.../kylin/rest/service/ServiceTestBase.java | 10 ++--
tool-assembly/pom.xml | 1 +
tool/pom.xml | 12 +++++
9 files changed, 79 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
index 2cb671e..05f437d 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
@@ -44,7 +44,6 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclConstant;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.AclTableMigrationTool;
-import org.apache.kylin.rest.service.UserGrantedAuthority;
import org.apache.kylin.rest.service.UserService;
import org.apache.kylin.rest.util.Serializer;
import org.junit.After;
@@ -53,6 +52,7 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.authority.SimpleGrantedAuthority;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -70,7 +70,7 @@ public class ITAclTableMigrationToolTest extends HBaseMetadataTestCase {
private TableName userTable = TableName.valueOf(STORE_WITH_OLD_TABLE + AclConstant.USER_TABLE_NAME);
- private Serializer<UserGrantedAuthority[]> ugaSerializer = new Serializer<UserGrantedAuthority[]>(UserGrantedAuthority[].class);
+ private Serializer<SimpleGrantedAuthority[]> ugaSerializer = new Serializer<>(SimpleGrantedAuthority[].class);
private AclTableMigrationTool aclTableMigrationJob;
@@ -192,13 +192,13 @@ public class ITAclTableMigrationToolTest extends HBaseMetadataTestCase {
if (authorities == null)
authorities = Collections.emptyList();
- UserGrantedAuthority[] serializing = new UserGrantedAuthority[authorities.size() + 1];
+ SimpleGrantedAuthority[] serializing = new SimpleGrantedAuthority[authorities.size() + 1];
// password is stored as the [0] authority
- serializing[0] = new UserGrantedAuthority(AclConstant.PWD_PREFIX + "password");
+ serializing[0] = new SimpleGrantedAuthority(AclConstant.PWD_PREFIX + "password");
int i = 1;
for (GrantedAuthority a : authorities) {
- serializing[i++] = new UserGrantedAuthority(a.getAuthority());
+ serializing[i++] = new SimpleGrantedAuthority(a.getAuthority());
}
byte[] value = ugaSerializer.serialize(serializing);
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
index a25e5b1..d6ac8f2 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
@@ -142,6 +142,10 @@ public class ProjectControllerV2 extends BasicController {
if (projectDesc.getName().equals(currentProject.getName())) {
updatedProj = projectService.updateProject(projectDesc, currentProject);
} else {
+ if (!isProjectEmpty(formerProjectName)) {
+ throw new BadRequestException(msg.getDELETE_PROJECT_NOT_EMPTY());
+ }
+ // disable project rename
updatedProj = projectService.renameProject(projectDesc, currentProject);
}
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, updatedProj, "");
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
index a828aa0..e086b68 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
@@ -246,7 +246,7 @@ public class CnMessage extends Message {
}
public String getDELETE_PROJECT_NOT_EMPTY() {
- return "不能删除该项目，如需要删除请先清空其中的Cube和Model";
+ return "不能修改该项目，如需要修改请先清空其中的Cube和Model";
}
public String getRENAME_PROJECT_NOT_EMPTY() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
index 45c1a65..f48a217 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
@@ -246,7 +246,7 @@ public class Message {
}
public String getDELETE_PROJECT_NOT_EMPTY() {
- return "Cannot delete non-empty project";
+ return "Cannot modify non-empty project";
}
// Table
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java b/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
index 280339e..69326a7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
@@ -18,6 +18,7 @@
package org.apache.kylin.rest.security;
+import java.io.IOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -25,10 +26,20 @@ import java.util.List;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.rest.service.UserGrantedAuthority;
import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.authority.SimpleGrantedAuthority;
import org.springframework.security.core.userdetails.UserDetails;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.google.common.collect.Lists;
@SuppressWarnings("serial")
@@ -40,7 +51,9 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
@JsonProperty
private String password;
@JsonProperty
- private List<UserGrantedAuthority> authorities = Lists.newArrayList();
+ @JsonSerialize(using = SimpleGrantedAuthoritySerializer.class)
+ @JsonDeserialize(using = SimpleGrantedAuthorityDeserializer.class)
+ private List<SimpleGrantedAuthority> authorities = Lists.newArrayList();
@JsonProperty
private boolean disabled = false;
@JsonProperty
@@ -60,7 +73,7 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
}
public ManagedUser(@JsonProperty String username, @JsonProperty String password,
- @JsonProperty List<UserGrantedAuthority> authorities, @JsonProperty boolean disabled,
+ @JsonProperty List<SimpleGrantedAuthority> authorities, @JsonProperty boolean disabled,
@JsonProperty boolean defaultPassword, @JsonProperty boolean locked, @JsonProperty long lockedTime,
@JsonProperty int wrongTime) {
this.username = username;
@@ -82,7 +95,7 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
this.authorities = Lists.newArrayList();
for (String a : authoritiesStr) {
- authorities.add(new UserGrantedAuthority(a));
+ authorities.add(new SimpleGrantedAuthority(a));
}
caterLegacy();
@@ -116,7 +129,7 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
}
private void caterLegacy() {
- Iterator<UserGrantedAuthority> iterator = authorities.iterator();
+ Iterator<SimpleGrantedAuthority> iterator = authorities.iterator();
while (iterator.hasNext()) {
if (DISABLED_ROLE.equals(iterator.next().getAuthority())) {
iterator.remove();
@@ -125,14 +138,14 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
}
}
- public List<UserGrantedAuthority> getAuthorities() {
+ public List<SimpleGrantedAuthority> getAuthorities() {
return this.authorities;
}
public void setGrantedAuthorities(Collection<? extends GrantedAuthority> grantedAuthorities) {
this.authorities = Lists.newArrayList();
for (GrantedAuthority grantedAuthority : grantedAuthorities) {
- this.authorities.add(new UserGrantedAuthority(grantedAuthority.getAuthority()));
+ this.authorities.add(new SimpleGrantedAuthority(grantedAuthority.getAuthority()));
}
}
@@ -228,4 +241,33 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
public String toString() {
return "ManagedUser [username=" + username + ", authorities=" + authorities + "]";
}
+
+ private static class SimpleGrantedAuthoritySerializer extends JsonSerializer<List<SimpleGrantedAuthority>> {
+
+ @Override
+ public void serialize(List<SimpleGrantedAuthority> value, JsonGenerator gen, SerializerProvider serializers)
+ throws IOException, JsonProcessingException {
+ List<UserGrantedAuthority> ugaList = Lists.newArrayList();
+ for (SimpleGrantedAuthority sga : value) {
+ ugaList.add(new UserGrantedAuthority(sga.getAuthority()));
+ }
+
+ gen.writeObject(ugaList);
+ }
+ }
+
+ private static class SimpleGrantedAuthorityDeserializer extends JsonDeserializer<List<SimpleGrantedAuthority>> {
+
+ @Override
+ public List<SimpleGrantedAuthority> deserialize(JsonParser p, DeserializationContext ctxt)
+ throws IOException, JsonProcessingException {
+ UserGrantedAuthority[] ugaArray = p.readValueAs(UserGrantedAuthority[].class);
+ List<SimpleGrantedAuthority> sgaList = Lists.newArrayList();
+ for (UserGrantedAuthority uga : ugaArray) {
+ sgaList.add(new SimpleGrantedAuthority(uga.getAuthority()));
+ }
+
+ return sgaList;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
index 64bac23..029efdc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
@@ -193,7 +193,7 @@ public class AclTableMigrationTool {
}
private DomainObjectInfo getDomainObjectInfoFromRs(Result result) {
- String type = String.valueOf(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY),
+ String type = new String(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY),
Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_TYPE_COLUMN)));
String id = new String(result.getRow());
DomainObjectInfo newInfo = new DomainObjectInfo();
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index 1d60a53..e2f5258 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -35,6 +35,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.Authentication;
+import org.springframework.security.core.authority.SimpleGrantedAuthority;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
@@ -72,18 +73,19 @@ public class ServiceTestBase extends LocalFileMetadataTestCase {
if (!userService.userExists("ADMIN")) {
userService.createUser(new ManagedUser("ADMIN", "KYLIN", false, Arrays.asList(//
- new UserGrantedAuthority(Constant.ROLE_ADMIN), new UserGrantedAuthority(Constant.ROLE_ANALYST),
- new UserGrantedAuthority(Constant.ROLE_MODELER))));
+ new SimpleGrantedAuthority(Constant.ROLE_ADMIN), new SimpleGrantedAuthority(Constant.ROLE_ANALYST),
+ new SimpleGrantedAuthority(Constant.ROLE_MODELER))));
}
if (!userService.userExists("MODELER")) {
userService.createUser(new ManagedUser("MODELER", "MODELER", false, Arrays.asList(//
- new UserGrantedAuthority(Constant.ROLE_ANALYST), new UserGrantedAuthority(Constant.ROLE_MODELER))));
+ new SimpleGrantedAuthority(Constant.ROLE_ANALYST),
+ new SimpleGrantedAuthority(Constant.ROLE_MODELER))));
}
if (!userService.userExists("ANALYST")) {
userService.createUser(new ManagedUser("ANALYST", "ANALYST", false, Arrays.asList(//
- new UserGrantedAuthority(Constant.ROLE_ANALYST))));
+ new SimpleGrantedAuthority(Constant.ROLE_ANALYST))));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/tool-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml
index 0595bdd..df0725b 100644
--- a/tool-assembly/pom.xml
+++ b/tool-assembly/pom.xml
@@ -104,6 +104,7 @@
<include>org.apache.kylin:*</include>
<include>org.springframework.security:spring-security-core</include>
<include>org.springframework.security:spring-security-acl</include>
+ <include>org.springframework:spring-core</include>
</includes>
</artifactSet>
<relocations>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index cced5d2..06a7e5a 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -32,6 +32,10 @@
<version>2.1.0-SNAPSHOT</version>
</parent>
+ <properties>
+ <spring.framework.version>4.2.8.RELEASE</spring.framework.version>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
@@ -72,6 +76,14 @@
<scope>provided</scope>
</dependency>
+ <!--Spring-->
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ <version>${spring.framework.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
<!-- Env & Test -->
<dependency>
<groupId>org.apache.kylin</groupId>
[17/26] kylin git commit: minor, fix computed column and adhoc issues
Posted by li...@apache.org.
minor, fix computed column and adhoc issues
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cf1ba956
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cf1ba956
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cf1ba956
Branch: refs/heads/kylin-2.1.x
Commit: cf1ba9566c2b8ae8aa0f69dc8c27293bee156ab5
Parents: 5fb4f35
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Jun 30 15:50:09 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 30 16:00:37 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/MetadataManager.java | 32 +-------
.../kylin/metadata/model/DataModelDesc.java | 80 +++++++++-----------
.../metadata/model/ModelDimensionDesc.java | 12 ++-
.../metadata/model/tool/CalciteParser.java | 9 ++-
.../kylin/model/tool/CalciteParserTest.java | 26 +++++--
.../query/util/ConvertToComputedColumn.java | 36 +++------
.../apache/kylin/rest/service/AclService.java | 4 +-
.../apache/kylin/rest/service/ModelService.java | 2 +-
.../org/apache/kylin/rest/util/AdHocUtil.java | 58 +++++---------
.../apache/kylin/rest/util/AdHocUtilTest.java | 23 +++---
.../kylin/rest/service/ModelServiceTest.java | 4 +-
11 files changed, 116 insertions(+), 170 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 96ae294..6c4faa3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -25,7 +25,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -40,7 +39,6 @@ import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.ExternalFilterDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -120,26 +118,6 @@ public class MetadataManager {
// name => External Filter Desc
private CaseInsensitiveStringCache<ExternalFilterDesc> extFilterMap;
- public static class CCInfo {
- private ComputedColumnDesc computedColumnDesc;
- private Set<DataModelDesc> dataModelDescs;
-
- public CCInfo(ComputedColumnDesc computedColumnDesc, Set<DataModelDesc> dataModelDescs) {
- this.computedColumnDesc = computedColumnDesc;
- this.dataModelDescs = dataModelDescs;
- }
-
- public ComputedColumnDesc getComputedColumnDesc() {
- return computedColumnDesc;
- }
-
- public Set<DataModelDesc> getDataModelDescs() {
- return dataModelDescs;
- }
- }
-
- private Map<String, CCInfo> ccInfoMap = Maps.newHashMap();// this is to check any two models won't conflict computed columns
-
private MetadataManager(KylinConfig config) throws IOException {
init(config);
}
@@ -181,9 +159,6 @@ public class MetadataManager {
return Collections.unmodifiableMap(srcTableMap.getMap());
}
- public Map<String, CCInfo> getCcInfoMap() {
- return ccInfoMap;
- }
public Map<String, TableExtDesc> listAllTableExdMap() {
return srcTableExdMap.getMap();
@@ -373,7 +348,6 @@ public class MetadataManager {
@Override
public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
- ccInfoMap.clear();
for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) {
reloadDataModelDescAt(DataModelDesc.concatResourcePath(model));
}
@@ -546,7 +520,7 @@ public class MetadataManager {
return new ArrayList<>(dataModelDescMap.values());
}
- public List<DataModelDesc> getModels(String projectName) throws IOException {
+ public List<DataModelDesc> getModels(String projectName) {
ProjectInstance projectInstance = ProjectManager.getInstance(config).getProject(projectName);
ArrayList<DataModelDesc> ret = new ArrayList<>();
@@ -618,7 +592,7 @@ public class MetadataManager {
DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER);
if (!dataModelDesc.isDraft())
- dataModelDesc.init(config, this.getAllTablesMap(), this.ccInfoMap);
+ dataModelDesc.init(config, this.getAllTablesMap(), listDataModels());
dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc);
return dataModelDesc;
@@ -669,7 +643,7 @@ public class MetadataManager {
private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) throws IOException {
if (!dataModelDesc.isDraft())
- dataModelDesc.init(config, this.getAllTablesMap(), this.ccInfoMap);
+ dataModelDesc.init(config, this.getAllTablesMap(), listDataModels());
String path = dataModelDesc.getResourcePath();
getStore().putResource(path, dataModelDesc, MODELDESC_SERIALIZER);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 341f36e..bc35e2a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -18,8 +18,6 @@
package org.apache.kylin.metadata.model;
-import static org.apache.kylin.metadata.MetadataManager.CCInfo;
-
import java.io.Serializable;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -38,6 +36,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.JoinsTree.Chain;
@@ -49,6 +48,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
@@ -234,6 +234,7 @@ public class DataModelDesc extends RootPersistentEntity {
return factTableRefs.contains(t);
}
+ //TODO: different from isFactTable(TableRef t)
public boolean isFactTable(String fullTableName) {
for (TableRef t : factTableRefs) {
if (t.getTableIdentity().equals(fullTableName))
@@ -332,7 +333,7 @@ public class DataModelDesc extends RootPersistentEntity {
throw new IllegalArgumentException("Table not found by " + tableIdentity + " in model " + name);
}
- public void init(KylinConfig config, Map<String, TableDesc> originalTables, Map<String, CCInfo> ccInfoMap) {
+ public void init(KylinConfig config, Map<String, TableDesc> originalTables, List<DataModelDesc> dataModelDescs) {
//tweak the tables according to Computed Columns defined in model
Map<String, TableDesc> tables = Maps.newHashMap();
for (Map.Entry<String, TableDesc> entry : originalTables.entrySet()) {
@@ -351,12 +352,12 @@ public class DataModelDesc extends RootPersistentEntity {
initJoinsTree();
initDimensionsAndMetrics();
initPartitionDesc();
- initComputedColumns(ccInfoMap);
+ initComputedColumns(dataModelDescs);
initFilterCondition();
boolean reinit = validate();
if (reinit) { // model slightly changed by validate() and must init() again
- init(config, tables, ccInfoMap);
+ init(config, tables, dataModelDescs);
}
}
@@ -471,38 +472,43 @@ public class DataModelDesc extends RootPersistentEntity {
this.partitionDesc.init(this);
}
- private void initComputedColumns(Map<String, CCInfo> ccInfoMap) {
- if (ccInfoMap == null) {
- logger.error("cc info map is null");
+ private void initComputedColumns(List<DataModelDesc> allDataModelDescs) {
+ Preconditions.checkNotNull(allDataModelDescs);
+
+ List<Pair<ComputedColumnDesc, DataModelDesc>> existingCCs = Lists.newArrayList();
+
+ for (DataModelDesc dataModelDesc : allDataModelDescs) {
+ if (!StringUtils.equals(dataModelDesc.getName(), this.getName())) {
+ for (ComputedColumnDesc cc : dataModelDesc.getComputedColumnDescs()) {
+ existingCCs.add(Pair.newPair(cc, dataModelDesc));
+ }
+ }
}
- Set<String> ccSet = Sets.newHashSet();//make sure cc name does not duplicate within this model
+ for (ComputedColumnDesc newCC : this.computedColumnDescs) {
- for (ComputedColumnDesc thisCCDesc : this.computedColumnDescs) {
- thisCCDesc.init();
- String thisCCName = thisCCDesc.getFullName();
+ newCC.init();
+ final String newCCName = newCC.getFullName();
+ final String newCCColumnName = newCC.getColumnName();
- if (ccSet.contains(thisCCName)) {
- throw new IllegalArgumentException(String.format(
- "More than one computed column named %s exist in model %s", thisCCName, this.getName()));
- } else {
- ccSet.add(thisCCName);
- }
+ for (Pair<ComputedColumnDesc, DataModelDesc> pair : existingCCs) {
+ DataModelDesc dataModelDesc = pair.getSecond();
+ ComputedColumnDesc cc = pair.getFirst();
- CCInfo other = ccInfoMap.get(thisCCName);
+ if (StringUtils.equals(cc.getFullName(), newCCName) && !(cc.equals(newCC))) {
+ throw new IllegalArgumentException(
+ String.format("Computed column named %s is defined differently in model %s", newCCName,
+ dataModelDesc.getName()));
+ }
- if (other == null) {
- checkSameCCDefinition(ccInfoMap, thisCCDesc, thisCCName);
- ccInfoMap.put(thisCCName, new CCInfo(thisCCDesc, Sets.<DataModelDesc> newHashSet(this)));
- } else if (other.getDataModelDescs().size() == 1 && other.getDataModelDescs().contains(this)) {
- ccInfoMap.put(thisCCName, new CCInfo(thisCCDesc, Sets.<DataModelDesc> newHashSet(this)));
- } else if (other.getComputedColumnDesc().equals(thisCCDesc)) {
- other.getDataModelDescs().add(this);
- } else {
- throw new IllegalStateException(String.format(
- "Computed column named %s is already defined in other models: %s. Please change another name, or try to keep consistent definition", //
- thisCCName, other.getDataModelDescs()));
+ if (isTwoCCDefinitionEquals(cc.getExpression(), newCC.getExpression())
+ && !StringUtils.equals(cc.getColumnName(), newCCColumnName)) {
+ throw new IllegalArgumentException(String.format(
+ "Duplicate expression of %s with computed column %s in model %s, keep same computed column name could suppress this",
+ newCCColumnName, cc.getColumnName(), dataModelDesc.getName()));
+ }
}
+ existingCCs.add(Pair.newPair(newCC, this));
}
}
@@ -548,20 +554,6 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
- private void checkSameCCDefinition(Map<String, CCInfo> ccInfoMap, ComputedColumnDesc thisCCDesc,
- String thisCCName) {
- //check whether two computer columns's definition is the same.
- for (CCInfo sysCCInfo : ccInfoMap.values()) {
- String definition0 = thisCCDesc.getExpression();
- String definition1 = sysCCInfo.getComputedColumnDesc().getExpression();
- if (isTwoCCDefinitionEquals(definition0, definition1)) {
- throw new IllegalStateException(String.format(
- "Computed column %s's definition: %s is already defined in other models: %s. Please change another definition, or try to keep consistent definition",
- thisCCName, definition0, sysCCInfo.getDataModelDescs()));
- }
- }
- }
-
private boolean isTwoCCDefinitionEquals(String definition0, String definition1) {
definition0 = definition0.replaceAll("\\s*", "");
definition1 = definition1.replaceAll("\\s*", "");
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
index c0ddbad..283c39c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
@@ -31,7 +31,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class ModelDimensionDesc implements Serializable {
private static final long serialVersionUID = 1L;
-
+
@JsonProperty("table")
private String table;
@JsonProperty("columns")
@@ -58,12 +58,18 @@ public class ModelDimensionDesc implements Serializable {
if (columns != null) {
StringUtil.toUpperCaseArray(columns, columns);
}
-
+
if (model != null) {
table = model.findTable(table).getAlias();
if (columns != null) {
for (int i = 0; i < columns.length; i++) {
- columns[i] = model.findColumn(table, columns[i]).getName();
+ TblColRef column = model.findColumn(table, columns[i]);
+
+ if (column.getColumnDesc().isComputedColumnn() && !model.isFactTable(column.getTableRef())) {
+ throw new RuntimeException("Computed Column on lookup table is not allowed");
+ }
+
+ columns[i] = column.getName();
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
index 47a90bf..9b80969 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
@@ -22,7 +22,9 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
+import java.util.Set;
+import com.google.common.collect.Lists;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlLiteral;
@@ -37,7 +39,7 @@ import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.commons.lang3.tuple.Pair;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
public class CalciteParser {
public static SqlNode parse(String sql) throws SqlParseException {
@@ -151,19 +153,20 @@ public class CalciteParser {
String sql = prefix + expr + suffix;
SqlNode sqlNode = getOnlySelectNode(sql);
- final List<SqlIdentifier> sqlIdentifiers = Lists.newArrayList();
+ final Set<SqlIdentifier> s = Sets.newHashSet();
SqlVisitor sqlVisitor = new SqlBasicVisitor() {
@Override
public Object visit(SqlIdentifier id) {
if (id.names.size() > 1) {
throw new IllegalArgumentException("SqlIdentifier " + id + " contains DB/Table name");
}
- sqlIdentifiers.add(id);
+ s.add(id);
return null;
}
};
sqlNode.accept(sqlVisitor);
+ List<SqlIdentifier> sqlIdentifiers = Lists.newArrayList(s);
Collections.sort(sqlIdentifiers, new Comparator<SqlIdentifier>() {
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
index acf7b5a..81c1c97 100644
--- a/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
@@ -18,12 +18,13 @@
package org.apache.kylin.model.tool;
+import static org.junit.Assert.assertEquals;
+
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -38,13 +39,13 @@ public class CalciteParserTest {
@Test
public void testNoTableNameExists() throws SqlParseException {
String expr1 = "a + b";
- Assert.assertEquals("x.a + x.b", CalciteParser.insertAliasInExpr(expr1, "x"));
+ assertEquals("x.a + x.b", CalciteParser.insertAliasInExpr(expr1, "x"));
String expr2 = "a + year(b)";
- Assert.assertEquals("x.a + year(x.b)", CalciteParser.insertAliasInExpr(expr2, "x"));
+ assertEquals("x.a + year(x.b)", CalciteParser.insertAliasInExpr(expr2, "x"));
String expr3 = "a + hiveudf(b)";
- Assert.assertEquals("x.a + hiveudf(x.b)", CalciteParser.insertAliasInExpr(expr3, "x"));
+ assertEquals("x.a + hiveudf(x.b)", CalciteParser.insertAliasInExpr(expr3, "x"));
}
@Test
@@ -75,6 +76,17 @@ public class CalciteParserTest {
}
@Test
+ public void testCasewhen() {
+ String expr = "(CASE LSTG_FORMAT_NAME WHEN 'Auction' THEN 'x' WHEN 'y' THEN '222' ELSE 'z' END)";
+ String alias = "TEST_KYLIN_FACT";
+ String s = CalciteParser.insertAliasInExpr(expr, alias);
+ System.out.println(s);
+ assertEquals(
+ "(CASE TEST_KYLIN_FACT.LSTG_FORMAT_NAME WHEN 'Auction' THEN 'x' WHEN 'y' THEN '222' ELSE 'z' END)",
+ s);
+ }
+
+ @Test
public void testPos() throws SqlParseException {
String[] sqls = new String[] { "select \n a \n + \n b \n from t", //
"select\na\n+\nb\nfrom t", //
@@ -104,8 +116,8 @@ public class CalciteParserTest {
SqlNode sn2 = CalciteParser.getOnlySelectNode(sql2);
SqlNode sn3 = CalciteParser.getOnlySelectNode(sql3);
- Assert.assertEquals(true, CalciteParser.isNodeEqual(sn0, sn1));
- Assert.assertEquals(true, CalciteParser.isNodeEqual(sn0, sn2));
- Assert.assertEquals(false, CalciteParser.isNodeEqual(sn0, sn3));
+ assertEquals(true, CalciteParser.isNodeEqual(sn0, sn1));
+ assertEquals(true, CalciteParser.isNodeEqual(sn0, sn2));
+ assertEquals(false, CalciteParser.isNodeEqual(sn0, sn3));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
index 7f188c1..a9ff360 100644
--- a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++ b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -24,8 +24,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import javax.annotation.Nullable;
-
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlDynamicParam;
@@ -40,18 +38,15 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Functions;
-import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
@@ -65,7 +60,7 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
ImmutableSortedMap<String, String> computedColumns = getSortedComputedColumnWithProject(project);
String s = replaceComputedColumn(sql, computedColumns);
if (!StringUtils.equals(sql, s)) {
- logger.debug("change sql to " + s);
+ logger.debug("sql changed");
}
return s;
}
@@ -162,28 +157,15 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
}
private ImmutableSortedMap<String, String> getSortedComputedColumnWithProject(String project) {
- MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- Map<String, MetadataManager.CCInfo> ccInfoMap = metadataManager.getCcInfoMap();
- final ProjectInstance projectInstance = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(project);
-
- Iterable<MetadataManager.CCInfo> projectCCInfo = Iterables.filter(ccInfoMap.values(),
- new Predicate<MetadataManager.CCInfo>() {
- @Override
- public boolean apply(@Nullable MetadataManager.CCInfo ccInfo) {
- return Iterables.any(ccInfo.getDataModelDescs(), new Predicate<DataModelDesc>() {
- @Override
- public boolean apply(@Nullable DataModelDesc model) {
- return projectInstance.containsModel(model.getName());
- }
- });
- }
- });
Map<String, String> computedColumns = new HashMap<>();
- for (MetadataManager.CCInfo ccInfo : projectCCInfo) {
- computedColumns.put(ccInfo.getComputedColumnDesc().getColumnName(),
- ccInfo.getComputedColumnDesc().getExpression());
+
+ MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ List<DataModelDesc> dataModelDescs = metadataManager.getModels(project);
+ for (DataModelDesc dataModelDesc : dataModelDescs) {
+ for (ComputedColumnDesc computedColumnDesc : dataModelDesc.getComputedColumnDescs()) {
+ computedColumns.put(computedColumnDesc.getColumnName(), computedColumnDesc.getExpression());
+ }
}
return getMapSortedByValue(computedColumns);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index 02b8e9f..79ed4aa 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -281,9 +281,7 @@ public class AclService implements MutableAclService {
} else {
aceInfos.addAll(allAceInfos.values());
}
- } else {
- logger.warn("Get null AllAceInfos from AclRecord");
- }
+ }
List<AccessControlEntry> newAces = new ArrayList<AccessControlEntry>();
for (int i = 0; i < aceInfos.size(); i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index fa54eaa..89aee26 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -222,7 +222,7 @@ public class ModelService extends BasicService {
List<CubeInstance> cubes = cubeService.listAllCubes(null, null, modelName, true);
if (cubes != null && cubes.size() != 0) {
dataModelDesc.init(getConfig(), getMetadataManager().getAllTablesMap(),
- getMetadataManager().getCcInfoMap());
+ getMetadataManager().listDataModels());
List<String> dimCols = new ArrayList<String>();
List<String> dimAndMCols = new ArrayList<String>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
index 9694a21..0eff508 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
@@ -21,23 +21,18 @@ package org.apache.kylin.rest.util;
import java.sql.SQLException;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import javax.annotation.Nullable;
-
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.MetadataManager.CCInfo;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.query.routing.NoRealizationFoundException;
import org.apache.kylin.source.adhocquery.IAdHocConverter;
@@ -46,8 +41,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
public class AdHocUtil {
@@ -72,9 +65,13 @@ public class AdHocUtil {
logger.debug("Ad-hoc query runner {}", runner);
String expandCC = restoreComputedColumnToExpr(sql, project);
+ if (!StringUtils.equals(expandCC, sql)) {
+ logger.info("computed column in sql is expanded to: " + expandCC);
+ }
String adhocSql = converter.convert(expandCC);
- if (!adhocSql.equals(sql)) {
- logger.info("before delegating to adhoc, the query is converted to {} ", adhocSql);
+ if (!adhocSql.equals(expandCC)) {
+ logger.info("the query is converted to {} according to kylin.query.ad-hoc.converter-class-name",
+ adhocSql);
}
runner.executeQuery(adhocSql, results, columnMetas);
@@ -98,37 +95,22 @@ public class AdHocUtil {
private final static Pattern endWithAsPattern = Pattern.compile("\\s+as\\s+$", Pattern.CASE_INSENSITIVE);
public static String restoreComputedColumnToExpr(String beforeSql, String project) {
- MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- Map<String, CCInfo> ccInfoMap = metadataManager.getCcInfoMap();
- final ProjectInstance projectInstance = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(project);
-
- Iterable<CCInfo> projectCCInfo = Iterables.filter(ccInfoMap.values(), new Predicate<CCInfo>() {
- @Override
- public boolean apply(@Nullable CCInfo ccInfo) {
- return Iterables.any(ccInfo.getDataModelDescs(), new Predicate<DataModelDesc>() {
- @Override
- public boolean apply(@Nullable DataModelDesc model) {
- return projectInstance.containsModel(model.getName());
- }
- });
- }
- });
+ final MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ List<DataModelDesc> dataModelDescs = metadataManager.getModels(project);
String afterSql = beforeSql;
- for (CCInfo ccInfo : projectCCInfo) {
- afterSql = restoreComputedColumnToExpr(afterSql, ccInfo);
+ for (DataModelDesc dataModelDesc : dataModelDescs) {
+ for (ComputedColumnDesc computedColumnDesc : dataModelDesc.getComputedColumnDescs()) {
+ afterSql = restoreComputedColumnToExpr(afterSql, computedColumnDesc);
+ }
}
- if (!StringUtils.equals(beforeSql, afterSql)) {
- logger.info("computed column in sql is expanded before sending to adhoc engine: " + afterSql);
- }
return afterSql;
}
- static String restoreComputedColumnToExpr(String sql, CCInfo ccInfo) {
+ static String restoreComputedColumnToExpr(String sql, ComputedColumnDesc computedColumnDesc) {
- String ccName = ccInfo.getComputedColumnDesc().getColumnName();
+ String ccName = computedColumnDesc.getColumnName();
List<Triple<Integer, Integer, String>> replacements = Lists.newArrayList();
Matcher matcher = identifierInSqlPattern.matcher(sql);
@@ -146,14 +128,14 @@ public class AdHocUtil {
String quotedTableAlias = StringUtils.strip(matcher.group(2), ".");
String tableAlias = StringUtils.strip(quotedTableAlias, "\"");
replacements.add(Triple.of(matcher.start(1), matcher.end(1),
- replaceIdentifierInExpr(ccInfo.getComputedColumnDesc().getExpression(), tableAlias, true)));
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, true)));
} else { //only column
if (endWithAsPattern.matcher(sql.substring(0, matcher.start(1))).find()) {
//select DEAL_AMOUNT as "deal_amount" case
continue;
}
replacements.add(Triple.of(matcher.start(1), matcher.end(1),
- replaceIdentifierInExpr(ccInfo.getComputedColumnDesc().getExpression(), null, true)));
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, true)));
}
} else if (matcher.group(4) != null) { //without quote case: table.column or simply column
String columnName = matcher.group(6);
@@ -164,8 +146,8 @@ public class AdHocUtil {
if (matcher.group(5) != null) { //table name exist
String tableAlias = StringUtils.strip(matcher.group(5), ".");
- replacements.add(Triple.of(matcher.start(4), matcher.end(4), replaceIdentifierInExpr(
- ccInfo.getComputedColumnDesc().getExpression(), tableAlias, false)));
+ replacements.add(Triple.of(matcher.start(4), matcher.end(4),
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, false)));
} else { //only column
if (endWithAsPattern.matcher(sql.substring(0, matcher.start(4))).find()) {
@@ -173,7 +155,7 @@ public class AdHocUtil {
continue;
}
replacements.add(Triple.of(matcher.start(4), matcher.end(4),
- replaceIdentifierInExpr(ccInfo.getComputedColumnDesc().getExpression(), null, false)));
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, false)));
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
index 724844c..05b135a 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.kylin.rest.util;
-import static org.apache.kylin.metadata.MetadataManager.CCInfo;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -62,36 +61,34 @@ public class AdHocUtilTest {
when(computedColumnDesc.getColumnName()).thenReturn("DEAL_AMOUNT");
when(computedColumnDesc.getExpression()).thenReturn("price * number");
- CCInfo ccInfo = mock(CCInfo.class);
- when(ccInfo.getComputedColumnDesc()).thenReturn(computedColumnDesc);
-
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
- Assert.assertEquals(
- "select (price * number) from DB.TABLE group by date order by (price * number)",
- ret);
+ "select DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", computedColumnDesc);
+ Assert.assertEquals("select (price * number) from DB.TABLE group by date order by (price * number)", ret);
}
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select DEAL_AMOUNT as DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
+ "select DEAL_AMOUNT as DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT",
+ computedColumnDesc);
Assert.assertEquals(
"select (price * number) as DEAL_AMOUNT from DB.TABLE group by date order by (price * number)",
ret);
}
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select \"DEAL_AMOUNT\" AS deal_amount from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
+ "select \"DEAL_AMOUNT\" AS deal_amount from DB.TABLE group by date order by DEAL_AMOUNT",
+ computedColumnDesc);
Assert.assertEquals(
"select (price * number) AS deal_amount from DB.TABLE group by date order by (price * number)",
ret);
}
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select x.DEAL_AMOUNT AS deal_amount from DB.TABLE x group by date order by x.DEAL_AMOUNT", ccInfo);
+ "select x.DEAL_AMOUNT AS deal_amount from DB.TABLE x group by date order by x.DEAL_AMOUNT",
+ computedColumnDesc);
Assert.assertEquals(
- "select (x.price * x.number) AS deal_amount from DB.TABLE x group by date order by (x.price * x.number)",
- ret);
+ "select (x.price * x.number) AS deal_amount from DB.TABLE x group by date order by (x.price * x.number)",
+ ret);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index b78b18e..b8633fb 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -80,8 +80,8 @@ public class ModelServiceTest extends ServiceTestBase {
@Test
public void testFailureModelUpdateDueToComputedColumnConflict() throws IOException, JobException, NoSuchFieldException, IllegalAccessException {
- expectedEx.expect(IllegalStateException.class);
- expectedEx.expectMessage("Computed column named DEFAULT.TEST_KYLIN_FACT.DEAL_AMOUNT is already defined in other models: [DataModelDesc [name=ci_left_join_model], DataModelDesc [name=ci_inner_join_model]]. Please change another name, or try to keep consistent definition");
+ expectedEx.expect(IllegalArgumentException.class);
+ expectedEx.expectMessage("Computed column named DEFAULT.TEST_KYLIN_FACT.DEAL_AMOUNT is defined differently in model ci_inner_join_model");
List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default", true);
Assert.assertTrue(dataModelDescs.size() == 1);
[04/26] kylin git commit: fix ci
Posted by li...@apache.org.
fix ci
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/86032971
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/86032971
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/86032971
Branch: refs/heads/kylin-2.1.x
Commit: 86032971f766a893a28fce9017227999f5defe2d
Parents: 56a0fa5
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 29 11:46:04 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 13:51:45 2017 +0800
----------------------------------------------------------------------
.../query/sql_computedcolumn/query03.sql | 21 ++++++++++++++++++++
.../sql_computedcolumn/query03.sql.compare | 21 ++++++++++++++++++++
.../org/apache/kylin/rest/util/AdHocUtil.java | 19 ++++--------------
3 files changed, 46 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/86032971/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql
new file mode 100644
index 0000000..ff8a570
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql
@@ -0,0 +1,21 @@
+SELECT
+
+count(*) as cnt, sum(price) as sum_price, sum(DEAL_AMOUNT) as deal_amount, SELLER_COUNTRY.NAME, DEAL_YEAR as deal_year
+
+FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
+INNER JOIN TEST_ORDER as TEST_ORDER
+ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
+ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
+ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
+ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
+ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
+
+
+where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and "SELLER_ACCOUNT"."COUNTRY_ABBR" in ('T', 'R') and BUYER_ACCOUNT.COUNTRY_ABBR in ('T', 'R')
+group by SELLER_COUNTRY.NAME, DEAL_YEAR
http://git-wip-us.apache.org/repos/asf/kylin/blob/86032971/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare
new file mode 100644
index 0000000..160375c
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare
@@ -0,0 +1,21 @@
+SELECT
+
+count(*) as cnt, sum(price) as sum_price, sum(TEST_KYLIN_FACT.PRICE * TEST_KYLIN_FACT.ITEM_COUNT) as deal_amount, SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT) as deal_year
+
+FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
+INNER JOIN TEST_ORDER as TEST_ORDER
+ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
+ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
+ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
+ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
+ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
+
+
+where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SUBSTR("SELLER_ACCOUNT"."ACCOUNT_COUNTRY",1,1) in ('T', 'R') and SUBSTR(BUYER_ACCOUNT.ACCOUNT_COUNTRY,1,1) in ('T', 'R')
+group by SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT)
http://git-wip-us.apache.org/repos/asf/kylin/blob/86032971/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
index 9eecdc1..9694a21 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
@@ -35,6 +35,7 @@ import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.MetadataManager.CCInfo;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
@@ -185,23 +186,11 @@ public class AdHocUtil {
return sql;
}
- // identifier in expr must be DB.TABLE.COLUMN, all TABLE in expr should be guaranteed to be same
static String replaceIdentifierInExpr(String expr, String tableAlias, boolean quoted) {
- List<Triple<Integer, Integer, String>> replacements = Lists.newArrayList();
- Matcher matcher = identifierInExprPattern.matcher(expr);
- while (matcher.find()) {
-
- String t = tableAlias == null ? StringUtils.strip(matcher.group(3), ".") : tableAlias;
- String c = matcher.group(4);
-
- String replacement = quoted ? "\"" + t.toUpperCase() + "\".\"" + c.toUpperCase() + "\"" : t + "." + c;
- replacements.add(Triple.of(matcher.start(1), matcher.end(1), replacement));
+ if (tableAlias == null) {
+ return expr;
}
- Collections.reverse(replacements);
- for (Triple<Integer, Integer, String> triple : replacements) {
- expr = expr.substring(0, triple.getLeft()) + triple.getRight() + expr.substring(triple.getMiddle());
- }
- return expr;
+ return CalciteParser.insertAliasInExpr(expr, tableAlias);
}
}
[14/26] kylin git commit: KYLIN-2696 Check SQL injection in filter
condition
Posted by li...@apache.org.
KYLIN-2696 Check SQL injection in filter condition
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2546e00c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2546e00c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2546e00c
Branch: refs/heads/kylin-2.1.x
Commit: 2546e00cf2f80b0f8abba9ec3cfe9a0fbce5cb4f
Parents: e7f9dab
Author: Xiaqing <45...@qq.com>
Authored: Fri Jun 30 09:42:07 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 09:42:07 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 43 ++++++++++++++++++++
1 file changed, 43 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2546e00c/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 91802f7..341f36e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -352,6 +352,7 @@ public class DataModelDesc extends RootPersistentEntity {
initDimensionsAndMetrics();
initPartitionDesc();
initComputedColumns(ccInfoMap);
+ initFilterCondition();
boolean reinit = validate();
if (reinit) { // model slightly changed by validate() and must init() again
@@ -505,6 +506,48 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
+ //Check if the filter condition is illegal.
+ private void initFilterCondition() {
+ if (null == this.filterCondition) {
+ return;
+ }
+ int quotationType = 0;
+ int len = this.filterCondition.length();
+ for (int i = 0; i < len; i++) {
+ //If a ';' which is not within a string is found, throw exception.
+ if (';' == this.filterCondition.charAt(i) && 0 == quotationType) {
+ throw new IllegalStateException(
+ "Filter Condition is Illegal. Please check it and make sure it's an appropriate expression for WHERE clause");
+ }
+ if ('\'' == this.filterCondition.charAt(i)) {
+ if (quotationType > 0) {
+ if (1 == quotationType) {
+ quotationType = 0;
+ continue;
+ }
+ } else {
+ if (0 == quotationType) {
+ quotationType = 1;
+ continue;
+ }
+ }
+ }
+ if ('"' == this.filterCondition.charAt(i)) {
+ if (quotationType > 0) {
+ if (2 == quotationType) {
+ quotationType = 0;
+ continue;
+ }
+ } else {
+ if (0 == quotationType) {
+ quotationType = 2;
+ continue;
+ }
+ }
+ }
+ }
+ }
+
private void checkSameCCDefinition(Map<String, CCInfo> ccInfoMap, ComputedColumnDesc thisCCDesc,
String thisCCName) {
//check whether two computer columns's definition is the same.
[16/26] kylin git commit: minor, fix project rename bug (#1475)
Posted by li...@apache.org.
minor, fix project rename bug (#1475)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5fb4f350
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5fb4f350
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5fb4f350
Branch: refs/heads/kylin-2.1.x
Commit: 5fb4f3500d12e3ad160b2235667de6131907af57
Parents: f52eb0c
Author: Roger Shi <ro...@gmail.com>
Authored: Fri Jun 30 14:31:24 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 14:31:24 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/service/ProjectService.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5fb4f350/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index c6bd6cf..cd60128 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -121,6 +121,7 @@ public class ProjectService extends BasicService {
// rebind draft and project
for (Draft draft : getDraftManager().list(currentProject.getName())) {
draft.setProject(newProjectName);
+ getDraftManager().save(draft);
}
logger.debug("Project rename.");
[09/26] kylin git commit: #476 enable model read permission when
grant cube permissions
Posted by li...@apache.org.
#476 enable model read permission when grant cube permissions
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ca0d321
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ca0d321
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ca0d321
Branch: refs/heads/kylin-2.1.x
Commit: 8ca0d3217453929d0ac0c3078d9e78e769f852b0
Parents: 7f79083
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 17:23:13 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 18:05:01 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/controller2/AccessControllerV2.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ca0d321/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
index 1ca8652..bd7d897 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
@@ -21,12 +21,15 @@ package org.apache.kylin.rest.controller2;
import java.io.IOException;
import org.apache.kylin.common.persistence.AclEntity;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.rest.controller.BasicController;
import org.apache.kylin.rest.request.AccessRequest;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.ResponseCode;
+import org.apache.kylin.rest.security.AclEntityType;
import org.apache.kylin.rest.security.AclPermissionFactory;
import org.apache.kylin.rest.service.AccessService;
+import org.apache.kylin.rest.service.CubeService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.acls.model.Acl;
@@ -51,6 +54,10 @@ public class AccessControllerV2 extends BasicController {
@Qualifier("accessService")
private AccessService accessService;
+ @Autowired
+ @Qualifier("cubeMgmtService")
+ private CubeService cubeService;
+
/**
* Get access entry list of a domain object
*
@@ -100,6 +107,15 @@ public class AccessControllerV2 extends BasicController {
Permission permission = AclPermissionFactory.getPermission(accessRequest.getPermission());
Acl acl = accessService.grant(ae, permission, sid);
+ if (AclEntityType.CUBE_INSTANCE.equals(type)) {
+ CubeInstance instance = cubeService.getCubeManager().getCubeByUuid(uuid);
+ if (instance != null) {
+ AclEntity model = accessService.getAclEntity(AclEntityType.DATA_MODEL_DESC, instance.getModel().getUuid());
+ // FIXME: should not always grant
+ accessService.grant(model, permission, sid);
+ }
+ }
+
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, accessService.generateAceResponses(acl), "");
}
[02/26] kylin git commit: update pom
Posted by li...@apache.org.
update pom
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56a0fa5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56a0fa5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56a0fa5f
Branch: refs/heads/kylin-2.1.x
Commit: 56a0fa5f1d1bebde0297612ee9a4af0354ce672e
Parents: d61f867
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jun 28 21:30:52 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 13:51:45 2017 +0800
----------------------------------------------------------------------
core-metadata/pom.xml | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/56a0fa5f/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index d45a838..9e06ecf 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -70,9 +70,6 @@
<artifactId>avatica-core</artifactId>
</exclusion>
</exclusions>
- <!--set to provided on purpose, so that dependant on metadata module will not depend on calcite unnecessarily-->
- <!--https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html-->
- <scope>provided</scope>
</dependency>
<!-- Compiled -->
[19/26] kylin git commit: minor, refine drop job situation
Posted by li...@apache.org.
minor, refine drop job situation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03702f1e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03702f1e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03702f1e
Branch: refs/heads/kylin-2.1.x
Commit: 03702f1ecd3a79228468bb7ec6ae25ad369fd65c
Parents: 6228f88
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Jun 30 19:19:35 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 19:21:45 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/controller2/JobControllerV2.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/03702f1e/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
index a86082c..5a18e66 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
@@ -282,8 +282,7 @@ public class JobControllerV2 extends BasicController {
JobInstance jobInstance = jobService.getJobInstance(jobId);
JobStatusEnum status = jobInstance.getStatus();
- if (status == JobStatusEnum.NEW || status == JobStatusEnum.PENDING || status == JobStatusEnum.RUNNING
- || status == JobStatusEnum.STOPPED) {
+ if (status != JobStatusEnum.FINISHED && status != JobStatusEnum.DISCARDED) {
throw new BadRequestException(
"Cannot drop running job " + jobInstance.getName() + ", please discard it first.");
}
[05/26] kylin git commit: Minor change, format code
Posted by li...@apache.org.
Minor change, format code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8acbc62
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8acbc62
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8acbc62
Branch: refs/heads/kylin-2.1.x
Commit: c8acbc620b2e52fd9d2ba3fc6fc478a6418b0f97
Parents: 2f7aa45
Author: auphyroc99 <45...@qq.com>
Authored: Thu Jun 29 14:48:38 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 14:50:21 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/jdbc/DriverTest.java | 20 +++++++++++++-------
1 file changed, 13 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8acbc62/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
index 4fd9d33..70978d7 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
@@ -178,7 +178,8 @@ public class DriverTest {
info.put("password", "KYLIN");
Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
- PreparedStatement state = conn.prepareStatement("select cal_dt, count(*) from test_kylin_fact where seller_id=? group by cal_dt");
+ PreparedStatement state = conn
+ .prepareStatement("select cal_dt, count(*) from test_kylin_fact where seller_id=? group by cal_dt");
state.setLong(1, 10000001);
ResultSet resultSet = state.executeQuery();
@@ -189,14 +190,14 @@ public class DriverTest {
state.close();
conn.close();
}
-
+
@Test
- public void testSSLFromURL() throws SQLException{
+ public void testSSLFromURL() throws SQLException {
Driver driver = new DummyDriver();
Connection conn = driver.connect("jdbc:kylin:ssl=True;//test_url/test_db", null);
- assertEquals("test_url", ((KylinConnection)conn).getBaseUrl());
- assertEquals("test_db", ((KylinConnection)conn).getProject());
- assertTrue(Boolean.parseBoolean( (String)((KylinConnection)conn).getConnectionProperties().get("ssl")));
+ assertEquals("test_url", ((KylinConnection) conn).getBaseUrl());
+ assertEquals("test_db", ((KylinConnection) conn).getProject());
+ assertTrue(Boolean.parseBoolean((String) ((KylinConnection) conn).getConnectionProperties().get("ssl")));
}
private void printResultSet(ResultSet rs) throws SQLException {
@@ -221,7 +222,12 @@ public class DriverTest {
System.out.println("Metadata:");
for (int i = 0; i < metadata.getColumnCount(); i++) {
- String metaStr = metadata.getCatalogName(i + 1) + " " + metadata.getColumnClassName(i + 1) + " " + metadata.getColumnDisplaySize(i + 1) + " " + metadata.getColumnLabel(i + 1) + " " + metadata.getColumnName(i + 1) + " " + metadata.getColumnType(i + 1) + " " + metadata.getColumnTypeName(i + 1) + " " + metadata.getPrecision(i + 1) + " " + metadata.getScale(i + 1) + " " + metadata.getSchemaName(i + 1) + " " + metadata.getTableName(i + 1);
+ String metaStr = metadata.getCatalogName(i + 1) + " " + metadata.getColumnClassName(i + 1) + " "
+ + metadata.getColumnDisplaySize(i + 1) + " " + metadata.getColumnLabel(i + 1) + " "
+ + metadata.getColumnName(i + 1) + " " + metadata.getColumnType(i + 1) + " "
+ + metadata.getColumnTypeName(i + 1) + " " + metadata.getPrecision(i + 1) + " "
+ + metadata.getScale(i + 1) + " " + metadata.getSchemaName(i + 1) + " "
+ + metadata.getTableName(i + 1);
System.out.println(metaStr);
}
}
[12/26] kylin git commit: minor, fix agg combination bug
Posted by li...@apache.org.
minor, fix agg combination bug
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eca7f6b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eca7f6b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eca7f6b
Branch: refs/heads/kylin-2.1.x
Commit: 5eca7f6b59454df4681deb17f361d4708a2affca
Parents: 40054dd
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 23:07:16 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 23:11:00 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/model/AggregationGroup.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eca7f6b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 064d657..0533ea1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -320,7 +320,7 @@ public class AggregationGroup implements Serializable {
normalDims.removeAll(jointDims);
combination = combination * (1L << normalDims.size());
- if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid()) {
+ if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid() && !mandatoryDims.isEmpty()) {
combination += 1;
}
combination -= 1; // not include cuboid 0
[25/26] kylin git commit: minor, fix acl migrate tool
Posted by li...@apache.org.
minor, fix acl migrate tool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f4382df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f4382df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f4382df
Branch: refs/heads/kylin-2.1.x
Commit: 4f4382df90ddd04a963dda36bcc1093c4b5d77b4
Parents: 7002dd8
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Jun 30 23:40:42 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 23:45:52 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/service/AclTableMigrationTool.java | 4 ++--
.../src/main/java/org/apache/kylin/rest/service/CubeService.java | 2 --
2 files changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f4382df/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
index 029efdc..bb07c22 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
@@ -224,8 +224,8 @@ public class AclTableMigrationTool {
Map<String, AceInfo> allAceInfoMap = new HashMap<>();
NavigableMap<byte[], byte[]> familyMap = result.getFamilyMap(Bytes.toBytes(AclConstant.ACL_ACES_FAMILY));
for (Map.Entry<byte[], byte[]> entry : familyMap.entrySet()) {
- String sid = String.valueOf(entry.getKey());
- AceInfo aceInfo = aceSerializer.deserialize(familyMap.get(entry.getValue()));
+ String sid = new String(entry.getKey());
+ AceInfo aceInfo = aceSerializer.deserialize(entry.getValue());
if (null != aceInfo) {
allAceInfoMap.put(sid, aceInfo);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f4382df/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index b2e44f6..aa42cb0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -659,8 +659,6 @@ public class CubeService extends BasicService {
getDraftManager().save(draft);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
- + " or hasPermission(#draft, 'ADMINISTRATION') or hasPermission(#draft, 'MANAGEMENT')")
public void deleteDraft(Draft draft) throws IOException {
getDraftManager().delete(draft.getUuid());
}
[20/26] kylin git commit: Use calcite's keywords during subquery
convert.
Posted by li...@apache.org.
Use calcite's keywords during subquery convert.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc232d11
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc232d11
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc232d11
Branch: refs/heads/kylin-2.1.x
Commit: dc232d114c5ad8c50ebed60cf238a0f5b21802a8
Parents: 03702f1
Author: nichunen <ch...@kyligence.io>
Authored: Fri Jun 30 19:43:53 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 30 19:57:28 2017 +0800
----------------------------------------------------------------------
.../source/adhocquery/HiveAdhocConverter.java | 102 ++++++++++++++-----
1 file changed, 79 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dc232d11/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
index 89b2f7a..e8c06ff 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
@@ -45,29 +45,85 @@ public class HiveAdhocConverter implements IAdHocConverter {
Pattern.CASE_INSENSITIVE);
private static final Pattern TIMESTAMPADD_PATTERN = Pattern.compile("timestampadd\\s*\\(\\s*(.*?)\\s*,",
Pattern.CASE_INSENSITIVE);
- private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("ABSOLUTE", "ACTION", "ADD", "ALL",
- "ALLOCATE", "ALTER", "AND", "ANY", "ARE", "ASC", "ASSERTION", "AT", "AUTHORIZATION", "AVG", "BEGIN",
- "BETWEEN", "BIT", "BIT_LENGTH", "BOTH", "BY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG", "CHAR",
- "CHARACTER", "CHARACTER_LENGTH", "CHAR_LENGTH", "CHECK", "CLOSE", "COALESCE", "COLLATE", "COLLATION",
- "COLUMN", "COMMIT", "CONNECT", "CONNECTION", "CONSTRAINT", "CONSTRAINTS", "CONTINUE", "CONVERT",
- "CORRESPONDING", "COUNT", "CREATE", "CROSS", "CURRENT", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP",
- "CURRENT_USER", "CURSOR", "DATE", "DAY", "DEALLOCATE", "DEC", "DECIMAL", "DECLARE", "DEFAULT", "DEFERRABLE",
- "DEFERRED", "DELETE", "DESC", "DESCRIBE", "DESCRIPTOR", "DIAGNOSTICS", "DISCONNECT", "DISTINCT", "DOMAIN",
- "DOUBLE", "DROP", "ELSE", "END", "END-EXEC", "ESCAPE", "EXCEPT", "EXCEPTION", "EXEC", "EXECUTE", "EXISTS",
- "EXTERNAL", "EXTRACT", "FALSE", "FETCH", "FIRST", "FLOAT", "FOR", "FOREIGN", "FOUND", "FROM", "FULL", "GET",
- "GLOBAL", "GO", "GOTO", "GRANT", "GROUP", "HAVING", "HOUR", "IDENTITY", "IMMEDIATE", "IN", "INDICATOR",
- "INITIALLY", "INNER", "INADD", "INSENSITIVE", "INSERT", "INT", "INTEGER", "INTERSECT", "INTERVAL", "INTO",
- "IS", "ISOLATION", "JOIN", "KEY", "LANGUAGE", "LAST", "LEADING", "LEFT", "LEVEL", "LIKE", "LOCAL", "LOWER",
- "MATCH", "MAX", "MIN", "MINUTE", "MODULE", "MONTH", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NEXT", "NO",
- "NOT", "NULL", "NULLIF", "NUMERIC", "OCTET_LENGTH", "OF", "ON", "ONLY", "OPEN", "OPTION", "OR", "ORDER",
- "OUTER", "OUTADD", "OVERLAPS", "PAD", "PARTIAL", "POSITION", "PRECISION", "PREPARE", "PRESERVE", "PRIMARY",
- "PRIOR", "PRIVILEGES", "PROCEDURE", "PUBLIC", "READ", "REAL", "REFERENCES", "RELATIVE", "RESTRICT",
- "REVOKE", "RIGHT", "ROLLBACK", "ROWS", "SCHEMA", "SCROLL", "SECOND", "SECTION", "SELECT", "SESSION",
- "SESSION_USER", "SET", "SIZE", "SMALLINT", "SOME", "SPACE", "SQL", "SQLCODE", "SQLERROR", "SQLSTATE",
- "SUBSTRING", "SUM", "SYSTEM_USER", "TABLE", "TEMPORARY", "THEN", "TIME", "TIMESTAMP", "TIMEZONE_HOUR",
- "TIMEZONE_MINUTE", "TO", "TRAILING", "TRANSACTION", "TRANSLATE", "TRANSLATION", "TRIM", "TRUE", "UNION",
- "UNIQUE", "UNKNOWN", "UPDATE", "UPPER", "USAGE", "USER", "USING", "VALUE", "VALUES", "VARCHAR", "VARYING",
- "VIEW", "WHEN", "WHENEVER", "WHERE", "WITH", "WORK", "WRITE", "YEAR", "ZONE");
+ private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("A", "ABS", "ABSOLUTE", "ACTION",
+ "ADA", "ADD", "ADMIN", "AFTER", "ALL", "ALLOCATE", "ALLOW", "ALTER", "ALWAYS", "AND", "ANY", "APPLY", "ARE",
+ "ARRAY", "ARRAY_MAX_CARDINALITY", "ASC", "ASENSITIVE", "ASSERTION", "ASSIGNMENT", "ASYMMETRIC", "AT",
+ "ATOMIC", "ATTRIBUTE", "ATTRIBUTES", "AUTHORIZATION", "AVG", "BEFORE", "BEGIN", "BEGIN_FRAME",
+ "BEGIN_PARTITION", "BERNOULLI", "BETWEEN", "BIGINT", "BINARY", "BIT", "BLOB", "BOOLEAN", "BOTH", "BREADTH",
+ "BY", "C", "CALL", "CALLED", "CARDINALITY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG",
+ "CATALOG_NAME", "CEIL", "CEILING", "CENTURY", "CHAIN", "CHAR", "CHARACTER", "CHARACTERISTICS", "CHARACTERS",
+ "CHARACTER_LENGTH", "CHARACTER_SET_CATALOG", "CHARACTER_SET_NAME", "CHARACTER_SET_SCHEMA", "CHAR_LENGTH",
+ "CHECK", "CLASSIFIER", "CLASS_ORIGIN", "CLOB", "CLOSE", "COALESCE", "COBOL", "COLLATE", "COLLATION",
+ "COLLATION_CATALOG", "COLLATION_NAME", "COLLATION_SCHEMA", "COLLECT", "COLUMN", "COLUMN_NAME",
+ "COMMAND_FUNCTION", "COMMAND_FUNCTION_CODE", "COMMIT", "COMMITTED", "CONDITION", "CONDITION_NUMBER",
+ "CONNECT", "CONNECTION", "CONNECTION_NAME", "CONSTRAINT", "CONSTRAINTS", "CONSTRAINT_CATALOG",
+ "CONSTRAINT_NAME", "CONSTRAINT_SCHEMA", "CONSTRUCTOR", "CONTAINS", "CONTINUE", "CONVERT", "CORR",
+ "CORRESPONDING", "COUNT", "COVAR_POP", "COVAR_SAMP", "CREATE", "CROSS", "CUBE", "CUME_DIST", "CURRENT",
+ "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_DEFAULT_TRANSFORM_GROUP", "CURRENT_PATH", "CURRENT_ROLE",
+ "CURRENT_ROW", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_TRANSFORM_GROUP_FOR_TYPE",
+ "CURRENT_USER", "CURSOR", "CURSOR_NAME", "CYCLE", "DATA", "DATABASE", "DATE", "DATETIME_INTERVAL_CODE",
+ "DATETIME_INTERVAL_PRECISION", "DAY", "DEALLOCATE", "DEC", "DECADE", "DECIMAL", "DECLARE", "DEFAULT",
+ "DEFAULTS", "DEFERRABLE", "DEFERRED", "DEFINE", "DEFINED", "DEFINER", "DEGREE", "DELETE", "DENSE_RANK",
+ "DEPTH", "DEREF", "DERIVED", "DESC", "DESCRIBE", "DESCRIPTION", "DESCRIPTOR", "DETERMINISTIC",
+ "DIAGNOSTICS", "DISALLOW", "DISCONNECT", "DISPATCH", "DISTINCT", "DOMAIN", "DOUBLE", "DOW", "DOY", "DROP",
+ "DYNAMIC", "DYNAMIC_FUNCTION", "DYNAMIC_FUNCTION_CODE", "EACH", "ELEMENT", "ELSE", "EMPTY", "END",
+ "END-EXEC", "END_FRAME", "END_PARTITION", "EPOCH", "EQUALS", "ESCAPE", "EVERY", "EXCEPT", "EXCEPTION",
+ "EXCLUDE", "EXCLUDING", "EXEC", "EXECUTE", "EXISTS", "EXP", "EXPLAIN", "EXTEND", "EXTERNAL", "EXTRACT",
+ "FALSE", "FETCH", "FILTER", "FINAL", "FIRST", "FIRST_VALUE", "FLOAT", "FLOOR", "FOLLOWING", "FOR",
+ "FOREIGN", "FORTRAN", "FOUND", "FRAC_SECOND", "FRAME_ROW", "FREE", "FROM", "FULL", "FUNCTION", "FUSION",
+ "G", "GENERAL", "GENERATED", "GET", "GLOBAL", "GO", "GOTO", "GRANT", "GRANTED", "GROUP", "GROUPING",
+ "GROUPS", "HAVING", "HIERARCHY", "HOLD", "HOUR", "IDENTITY", "IMMEDIATE", "IMMEDIATELY", "IMPLEMENTATION",
+ "IMPORT", "IN", "INCLUDING", "INCREMENT", "INDICATOR", "INITIAL", "INITIALLY", "INNER", "INOUT", "INPUT",
+ "INSENSITIVE", "INSERT", "INSTANCE", "INSTANTIABLE", "INT", "INTEGER", "INTERSECT", "INTERSECTION",
+ "INTERVAL", "INTO", "INVOKER", "IS", "ISOLATION", "JAVA", "JOIN", "JSON", "K", "KEY", "KEY_MEMBER",
+ "KEY_TYPE", "LABEL", "LAG", "LANGUAGE", "LARGE", "LAST", "LAST_VALUE", "LATERAL", "LEAD", "LEADING", "LEFT",
+ "LENGTH", "LEVEL", "LIBRARY", "LIKE", "LIKE_REGEX", "LIMIT", "LN", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP",
+ "LOCATOR", "LOWER", "M", "MAP", "MATCH", "MATCHED", "MATCHES", "MATCH_NUMBER", "MATCH_RECOGNIZE", "MAX",
+ "MAXVALUE", "MEASURES", "MEMBER", "MERGE", "MESSAGE_LENGTH", "MESSAGE_OCTET_LENGTH", "MESSAGE_TEXT",
+ "METHOD", "MICROSECOND", "MILLENNIUM", "MIN", "MINUS", "MINUTE", "MINVALUE", "MOD", "MODIFIES", "MODULE",
+ "MONTH", "MORE", "MULTISET", "MUMPS", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NCLOB", "NESTING",
+ "NEW", "NEXT", "NO", "NONE", "NORMALIZE", "NORMALIZED", "NOT", "NTH_VALUE", "NTILE", "NULL", "NULLABLE",
+ "NULLIF", "NULLS", "NUMBER", "NUMERIC", "OBJECT", "OCCURRENCES_REGEX", "OCTETS", "OCTET_LENGTH", "OF",
+ "OFFSET", "OLD", "OMIT", "ON", "ONE", "ONLY", "OPEN", "OPTION", "OPTIONS", "OR", "ORDER", "ORDERING",
+ "ORDINALITY", "OTHERS", "OUT", "OUTER", "OUTPUT", "OVER", "OVERLAPS", "OVERLAY", "OVERRIDING", "PAD",
+ "PARAMETER", "PARAMETER_MODE", "PARAMETER_NAME", "PARAMETER_ORDINAL_POSITION", "PARAMETER_SPECIFIC_CATALOG",
+ "PARAMETER_SPECIFIC_NAME", "PARAMETER_SPECIFIC_SCHEMA", "PARTIAL", "PARTITION", "PASCAL", "PASSTHROUGH",
+ "PAST", "PATH", "PATTERN", "PER", "PERCENT", "PERCENTILE_CONT", "PERCENTILE_DISC", "PERCENT_RANK", "PERIOD",
+ "PERMUTE", "PLACING", "PLAN", "PLI", "PORTION", "POSITION", "POSITION_REGEX", "POWER", "PRECEDES",
+ "PRECEDING", "PRECISION", "PREPARE", "PRESERVE", "PREV", "PRIMARY", "PRIOR", "PRIVILEGES", "PROCEDURE",
+ "PUBLIC", "QUARTER", "RANGE", "RANK", "READ", "READS", "REAL", "RECURSIVE", "REF", "REFERENCES",
+ "REFERENCING", "REGR_AVGX", "REGR_AVGY", "REGR_COUNT", "REGR_INTERCEPT", "REGR_R2", "REGR_SLOPE",
+ "REGR_SXX", "REGR_SXY", "REGR_SYY", "RELATIVE", "RELEASE", "REPEATABLE", "REPLACE", "RESET", "RESTART",
+ "RESTRICT", "RESULT", "RETURN", "RETURNED_CARDINALITY", "RETURNED_LENGTH", "RETURNED_OCTET_LENGTH",
+ "RETURNED_SQLSTATE", "RETURNS", "REVOKE", "RIGHT", "ROLE", "ROLLBACK", "ROLLUP", "ROUTINE",
+ "ROUTINE_CATALOG", "ROUTINE_NAME", "ROUTINE_SCHEMA", "ROW", "ROWS", "ROW_COUNT", "ROW_NUMBER", "RUNNING",
+ "SAVEPOINT", "SCALE", "SCHEMA", "SCHEMA_NAME", "SCOPE", "SCOPE_CATALOGS", "SCOPE_NAME", "SCOPE_SCHEMA",
+ "SCROLL", "SEARCH", "SECOND", "SECTION", "SECURITY", "SEEK", "SELECT", "SELF", "SENSITIVE", "SEQUENCE",
+ "SERIALIZABLE", "SERVER", "SERVER_NAME", "SESSION", "SESSION_USER", "SET", "SETS", "SHOW", "SIMILAR",
+ "SIMPLE", "SIZE", "SKIP", "SMALLINT", "SOME", "SOURCE", "SPACE", "SPECIFIC", "SPECIFICTYPE",
+ "SPECIFIC_NAME", "SQL", "SQLEXCEPTION", "SQLSTATE", "SQLWARNING", "SQL_BIGINT", "SQL_BINARY", "SQL_BIT",
+ "SQL_BLOB", "SQL_BOOLEAN", "SQL_CHAR", "SQL_CLOB", "SQL_DATE", "SQL_DECIMAL", "SQL_DOUBLE", "SQL_FLOAT",
+ "SQL_INTEGER", "SQL_INTERVAL_DAY", "SQL_INTERVAL_DAY_TO_HOUR", "SQL_INTERVAL_DAY_TO_MINUTE",
+ "SQL_INTERVAL_DAY_TO_SECOND", "SQL_INTERVAL_HOUR", "SQL_INTERVAL_HOUR_TO_MINUTE",
+ "SQL_INTERVAL_HOUR_TO_SECOND", "SQL_INTERVAL_MINUTE", "SQL_INTERVAL_MINUTE_TO_SECOND", "SQL_INTERVAL_MONTH",
+ "SQL_INTERVAL_SECOND", "SQL_INTERVAL_YEAR", "SQL_INTERVAL_YEAR_TO_MONTH", "SQL_LONGVARBINARY",
+ "SQL_LONGVARCHAR", "SQL_LONGVARNCHAR", "SQL_NCHAR", "SQL_NCLOB", "SQL_NUMERIC", "SQL_NVARCHAR", "SQL_REAL",
+ "SQL_SMALLINT", "SQL_TIME", "SQL_TIMESTAMP", "SQL_TINYINT", "SQL_TSI_DAY", "SQL_TSI_FRAC_SECOND",
+ "SQL_TSI_HOUR", "SQL_TSI_MICROSECOND", "SQL_TSI_MINUTE", "SQL_TSI_MONTH", "SQL_TSI_QUARTER",
+ "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SQL_VARBINARY", "SQL_VARCHAR", "SQRT", "START", "STATE",
+ "STATEMENT", "STATIC", "STDDEV_POP", "STDDEV_SAMP", "STREAM", "STRUCTURE", "STYLE", "SUBCLASS_ORIGIN",
+ "SUBMULTISET", "SUBSET", "SUBSTITUTE", "SUBSTRING", "SUBSTRING_REGEX", "SUCCEEDS", "SUM", "SYMMETRIC",
+ "SYSTEM", "SYSTEM_TIME", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TABLE_NAME", "TEMPORARY", "THEN", "TIES",
+ "TIME", "TIMESTAMP", "TIMESTAMPADD", "TIMESTAMPDIFF", "TIMEZONE_HOUR", "TIMEZONE_MINUTE", "TINYINT", "TO",
+ "TOP_LEVEL_COUNT", "TRAILING", "TRANSACTION", "TRANSACTIONS_ACTIVE", "TRANSACTIONS_COMMITTED",
+ "TRANSACTIONS_ROLLED_BACK", "TRANSFORM", "TRANSFORMS", "TRANSLATE", "TRANSLATE_REGEX", "TRANSLATION",
+ "TREAT", "TRIGGER", "TRIGGER_CATALOG", "TRIGGER_NAME", "TRIGGER_SCHEMA", "TRIM", "TRIM_ARRAY", "TRUE",
+ "TRUNCATE", "TYPE", "UESCAPE", "UNBOUNDED", "UNCOMMITTED", "UNDER", "UNION", "UNIQUE", "UNKNOWN", "UNNAMED",
+ "UNNEST", "UPDATE", "UPPER", "UPSERT", "USAGE", "USER", "USER_DEFINED_TYPE_CATALOG",
+ "USER_DEFINED_TYPE_CODE", "USER_DEFINED_TYPE_NAME", "USER_DEFINED_TYPE_SCHEMA", "USING", "VALUE", "VALUES",
+ "VALUE_OF", "VARBINARY", "VARCHAR", "VARYING", "VAR_POP", "VAR_SAMP", "VERSION", "VERSIONING", "VIEW",
+ "WEEK", "WHEN", "WHENEVER", "WHERE", "WIDTH_BUCKET", "WINDOW", "WITH", "WITHIN", "WITHOUT", "WORK",
+ "WRAPPER", "WRITE", "XML", "YEAR", "ZONE");
public static String replaceString(String originString, String fromString, String toString) {
return originString.replace(fromString, toString);
[06/26] kylin git commit: minor, prevent to drop job if it is stopped
Posted by li...@apache.org.
minor, prevent to drop job if it is stopped
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8e1a7d51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8e1a7d51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8e1a7d51
Branch: refs/heads/kylin-2.1.x
Commit: 8e1a7d514519bedfbadccd3ba252ca240b56137f
Parents: c8acbc6
Author: Cheng Wang <ch...@kyligence.io>
Authored: Thu Jun 29 15:52:09 2017 +0800
Committer: 成 <ch...@kyligence.io>
Committed: Thu Jun 29 16:08:04 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller2/JobControllerV2.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e1a7d51/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
index f9db6d8..a86082c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
@@ -282,8 +282,10 @@ public class JobControllerV2 extends BasicController {
JobInstance jobInstance = jobService.getJobInstance(jobId);
JobStatusEnum status = jobInstance.getStatus();
- if (status == JobStatusEnum.NEW || status == JobStatusEnum.PENDING || status == JobStatusEnum.RUNNING) {
- throw new BadRequestException("Cannot drop running job " + jobInstance.getName());
+ if (status == JobStatusEnum.NEW || status == JobStatusEnum.PENDING || status == JobStatusEnum.RUNNING
+ || status == JobStatusEnum.STOPPED) {
+ throw new BadRequestException(
+ "Cannot drop running job " + jobInstance.getName() + ", please discard it first.");
}
jobService.dropJob(jobInstance);
[07/26] kylin git commit: #1294 refine acl
Posted by li...@apache.org.
#1294 refine acl
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7f790837
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7f790837
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7f790837
Branch: refs/heads/kylin-2.1.x
Commit: 7f79083722b634df8f4f814c6a59e0472dbcbe74
Parents: c490e6c
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 16:35:05 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 16:36:57 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/controller2/AccessControllerV2.java | 2 +-
.../src/main/java/org/apache/kylin/rest/service/AccessService.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f790837/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
index a61e321..1ca8652 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
@@ -80,7 +80,7 @@ public class AccessControllerV2 extends BasicController {
public EnvelopeResponse listAccessEntitiesV2(@PathVariable String type, @PathVariable String uuid) {
AclEntity ae = accessService.getAclEntity(type, uuid);
Acl acl = accessService.getAcl(ae);
- return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, accessService.generateListAceResponses(acl), "");
+ return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, accessService.generateAllAceResponses(acl), "");
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f790837/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 3a6d723..affa508 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -322,7 +322,7 @@ public class AccessService {
}
}
- public Object generateListAceResponses(Acl acl) {
+ public Object generateAllAceResponses(Acl acl) {
List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
while (acl != null) {
[23/26] kylin git commit: minor, rename adhoc to pushdown in KYLIN
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 8d18901..f3402ef 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -94,7 +94,7 @@ import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.util.AclUtil;
-import org.apache.kylin.rest.util.AdHocUtil;
+import org.apache.kylin.rest.util.PushDownUtil;
import org.apache.kylin.rest.util.TableauInterceptor;
import org.apache.kylin.storage.hybrid.HybridInstance;
import org.slf4j.Logger;
@@ -220,7 +220,8 @@ public class QueryService extends BasicService {
return null;
}
List<Query> queries = new ArrayList<Query>();
- QueryRecord record = queryStore.getResource(getQueryKeyById(creator), QueryRecord.class, QueryRecordSerializer.getInstance());
+ QueryRecord record = queryStore.getResource(getQueryKeyById(creator), QueryRecord.class,
+ QueryRecordSerializer.getInstance());
if (record != null) {
for (Query query : record.getQueries()) {
queries.add(query);
@@ -235,7 +236,7 @@ public class QueryService extends BasicService {
final Set<Long> cuboidIds = new HashSet<Long>();
float duration = response.getDuration() / (float) 1000;
boolean storageCacheUsed = response.isStorageCacheUsed();
- boolean isAdHoc = response.isAdHoc();
+ boolean isPushDown = response.isPushDown();
if (!response.isHitExceptionCache() && null != OLAPContext.getThreadLocalContexts()) {
for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) {
@@ -276,7 +277,7 @@ public class QueryService extends BasicService {
stringBuilder.append("Is Partial Result: ").append(response.isPartial()).append(newLine);
stringBuilder.append("Hit Exception Cache: ").append(response.isHitExceptionCache()).append(newLine);
stringBuilder.append("Storage cache used: ").append(storageCacheUsed).append(newLine);
- stringBuilder.append("Is Ad-hoc Query: ").append(isAdHoc).append(newLine);
+ stringBuilder.append("Is Query Push-Down: ").append(isPushDown).append(newLine);
stringBuilder.append("Message: ").append(response.getExceptionMessage()).append(newLine);
stringBuilder.append("==========================[QUERY]===============================").append(newLine);
@@ -292,14 +293,16 @@ public class QueryService extends BasicService {
return;
}
} catch (AccessDeniedException e) {
- logger.warn("Current user {} has no READ permission on current project {}, please ask Administrator for permission granting.");
+ logger.warn(
+ "Current user {} has no READ permission on current project {}, please ask Administrator for permission granting.");
//just continue
}
String realizationsStr = sqlResponse.getCube();//CUBE[name=abc],HYBRID[name=xyz]
if (StringUtils.isEmpty(realizationsStr)) {
- throw new AccessDeniedException("Ad-hoc query requires having READ permission on project, please ask Administrator to grant you permissions");
+ throw new AccessDeniedException(
+ "Query pushdown requires having READ permission on project, please ask Administrator to grant you permissions");
}
String[] splits = StringUtils.split(realizationsStr, ",");
@@ -344,7 +347,8 @@ public class QueryService extends BasicService {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
String serverMode = kylinConfig.getServerMode();
- if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase()))) {
+ if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase())
+ || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase()))) {
throw new BadRequestException(String.format(msg.getQUERY_NOT_ALLOWED(), serverMode));
}
if (StringUtils.isBlank(sqlRequest.getProject())) {
@@ -370,7 +374,8 @@ public class QueryService extends BasicService {
long startTime = System.currentTimeMillis();
SQLResponse sqlResponse = null;
- boolean queryCacheEnabled = checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in KylinConfig") && //
+ boolean queryCacheEnabled = checkCondition(kylinConfig.isQueryCacheEnabled(),
+ "query cache disabled in KylinConfig") && //
checkCondition(!BackdoorToggles.getDisableCache(), "query cache disabled in BackdoorToggles");
if (queryCacheEnabled) {
@@ -386,13 +391,22 @@ public class QueryService extends BasicService {
long scanBytesThreshold = kylinConfig.getQueryScanBytesCacheThreshold();
sqlResponse.setDuration(System.currentTimeMillis() - startTime);
logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", //
- String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()));
+ String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()),
+ String.valueOf(sqlResponse.getTotalScanCount()));
if (checkCondition(queryCacheEnabled, "query cache is disabled") //
&& checkCondition(!sqlResponse.getIsException(), "query has exception") //
- && checkCondition(sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scanCountThreshold || sqlResponse.getTotalScanBytes() > scanBytesThreshold, //
- "query is too lightweight with duration: {} (threshold {}), scan count: {} (threshold {}), scan bytes: {} (threshold {})", sqlResponse.getDuration(), durationThreshold, sqlResponse.getTotalScanCount(), scanCountThreshold, sqlResponse.getTotalScanBytes(), scanBytesThreshold)
- && checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(), "query response is too large: {} ({})", sqlResponse.getResults().size(), kylinConfig.getLargeQueryThreshold())) {
- cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest.getCacheKey(), sqlResponse));
+ && checkCondition(
+ sqlResponse.getDuration() > durationThreshold
+ || sqlResponse.getTotalScanCount() > scanCountThreshold
+ || sqlResponse.getTotalScanBytes() > scanBytesThreshold, //
+ "query is too lightweight with duration: {} (threshold {}), scan count: {} (threshold {}), scan bytes: {} (threshold {})",
+ sqlResponse.getDuration(), durationThreshold, sqlResponse.getTotalScanCount(),
+ scanCountThreshold, sqlResponse.getTotalScanBytes(), scanBytesThreshold)
+ && checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(),
+ "query response is too large: {} ({})", sqlResponse.getResults().size(),
+ kylinConfig.getLargeQueryThreshold())) {
+ cacheManager.getCache(SUCCESS_QUERY_CACHE)
+ .put(new Element(sqlRequest.getCacheKey(), sqlResponse));
}
} else {
@@ -411,7 +425,8 @@ public class QueryService extends BasicService {
sqlResponse.setTotalScanCount(queryContext.getScannedRows());
sqlResponse.setTotalScanBytes(queryContext.getScannedBytes());
- if (queryCacheEnabled && e.getCause() != null && ExceptionUtils.getRootCause(e) instanceof ResourceLimitExceededException) {
+ if (queryCacheEnabled && e.getCause() != null
+ && ExceptionUtils.getRootCause(e) instanceof ResourceLimitExceededException) {
Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
exceptionCache.put(new Element(sqlRequest.getCacheKey(), sqlResponse));
}
@@ -459,7 +474,8 @@ public class QueryService extends BasicService {
private SQLResponse queryWithSqlMassage(SQLRequest sqlRequest) throws Exception {
String userInfo = SecurityContextHolder.getContext().getAuthentication().getName();
- final Collection<? extends GrantedAuthority> grantedAuthorities = SecurityContextHolder.getContext().getAuthentication().getAuthorities();
+ final Collection<? extends GrantedAuthority> grantedAuthorities = SecurityContextHolder.getContext()
+ .getAuthentication().getAuthorities();
for (GrantedAuthority grantedAuthority : grantedAuthorities) {
userInfo += ",";
userInfo += grantedAuthority.getAuthority();
@@ -514,9 +530,12 @@ public class QueryService extends BasicService {
String schemaName = JDBCTableMeta.getString(2);
// Not every JDBC data provider offers full 10 columns, e.g., PostgreSQL has only 5
- TableMeta tblMeta = new TableMeta(catalogName == null ? Constant.FakeCatalogName : catalogName, schemaName == null ? Constant.FakeSchemaName : schemaName, JDBCTableMeta.getString(3), JDBCTableMeta.getString(4), JDBCTableMeta.getString(5), null, null, null, null, null);
+ TableMeta tblMeta = new TableMeta(catalogName == null ? Constant.FakeCatalogName : catalogName,
+ schemaName == null ? Constant.FakeSchemaName : schemaName, JDBCTableMeta.getString(3),
+ JDBCTableMeta.getString(4), JDBCTableMeta.getString(5), null, null, null, null, null);
- if (!cubedOnly || getProjectManager().isExposedTable(project, schemaName + "." + tblMeta.getTABLE_NAME())) {
+ if (!cubedOnly
+ || getProjectManager().isExposedTable(project, schemaName + "." + tblMeta.getTABLE_NAME())) {
tableMetas.add(tblMeta);
tableMap.put(tblMeta.getTABLE_SCHEM() + "#" + tblMeta.getTABLE_NAME(), tblMeta);
}
@@ -529,9 +548,18 @@ public class QueryService extends BasicService {
String schemaName = columnMeta.getString(2);
// kylin(optiq) is not strictly following JDBC specification
- ColumnMeta colmnMeta = new ColumnMeta(catalogName == null ? Constant.FakeCatalogName : catalogName, schemaName == null ? Constant.FakeSchemaName : schemaName, columnMeta.getString(3), columnMeta.getString(4), columnMeta.getInt(5), columnMeta.getString(6), columnMeta.getInt(7), getInt(columnMeta.getString(8)), columnMeta.getInt(9), columnMeta.getInt(10), columnMeta.getInt(11), columnMeta.getString(12), columnMeta.getString(13), getInt(columnMeta.getString(14)), getInt(columnMeta.getString(15)), columnMeta.getInt(16), columnMeta.getInt(17), columnMeta.getString(18), columnMeta.getString(19), columnMeta.getString(20), columnMeta.getString(21), getShort(columnMeta.getString(22)), columnMeta.getString(23));
-
- if (!cubedOnly || getProjectManager().isExposedColumn(project, schemaName + "." + colmnMeta.getTABLE_NAME(), colmnMeta.getCOLUMN_NAME())) {
+ ColumnMeta colmnMeta = new ColumnMeta(catalogName == null ? Constant.FakeCatalogName : catalogName,
+ schemaName == null ? Constant.FakeSchemaName : schemaName, columnMeta.getString(3),
+ columnMeta.getString(4), columnMeta.getInt(5), columnMeta.getString(6), columnMeta.getInt(7),
+ getInt(columnMeta.getString(8)), columnMeta.getInt(9), columnMeta.getInt(10),
+ columnMeta.getInt(11), columnMeta.getString(12), columnMeta.getString(13),
+ getInt(columnMeta.getString(14)), getInt(columnMeta.getString(15)), columnMeta.getInt(16),
+ columnMeta.getInt(17), columnMeta.getString(18), columnMeta.getString(19),
+ columnMeta.getString(20), columnMeta.getString(21), getShort(columnMeta.getString(22)),
+ columnMeta.getString(23));
+
+ if (!cubedOnly || getProjectManager().isExposedColumn(project,
+ schemaName + "." + colmnMeta.getTABLE_NAME(), colmnMeta.getCOLUMN_NAME())) {
tableMap.get(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME()).addColumn(colmnMeta);
}
}
@@ -551,7 +579,8 @@ public class QueryService extends BasicService {
}
@SuppressWarnings("checkstyle:methodlength")
- protected List<TableMetaWithType> getMetadataV2(CubeManager cubeMgr, String project, boolean cubedOnly) throws SQLException, IOException {
+ protected List<TableMetaWithType> getMetadataV2(CubeManager cubeMgr, String project, boolean cubedOnly)
+ throws SQLException, IOException {
//Message msg = MsgPicker.getMsg();
Connection conn = null;
@@ -578,9 +607,13 @@ public class QueryService extends BasicService {
String schemaName = JDBCTableMeta.getString(2);
// Not every JDBC data provider offers full 10 columns, e.g., PostgreSQL has only 5
- TableMetaWithType tblMeta = new TableMetaWithType(catalogName == null ? Constant.FakeCatalogName : catalogName, schemaName == null ? Constant.FakeSchemaName : schemaName, JDBCTableMeta.getString(3), JDBCTableMeta.getString(4), JDBCTableMeta.getString(5), null, null, null, null, null);
+ TableMetaWithType tblMeta = new TableMetaWithType(
+ catalogName == null ? Constant.FakeCatalogName : catalogName,
+ schemaName == null ? Constant.FakeSchemaName : schemaName, JDBCTableMeta.getString(3),
+ JDBCTableMeta.getString(4), JDBCTableMeta.getString(5), null, null, null, null, null);
- if (!cubedOnly || getProjectManager().isExposedTable(project, schemaName + "." + tblMeta.getTABLE_NAME())) {
+ if (!cubedOnly
+ || getProjectManager().isExposedTable(project, schemaName + "." + tblMeta.getTABLE_NAME())) {
tableMetas.add(tblMeta);
tableMap.put(tblMeta.getTABLE_SCHEM() + "#" + tblMeta.getTABLE_NAME(), tblMeta);
}
@@ -593,11 +626,22 @@ public class QueryService extends BasicService {
String schemaName = columnMeta.getString(2);
// kylin(optiq) is not strictly following JDBC specification
- ColumnMetaWithType colmnMeta = new ColumnMetaWithType(catalogName == null ? Constant.FakeCatalogName : catalogName, schemaName == null ? Constant.FakeSchemaName : schemaName, columnMeta.getString(3), columnMeta.getString(4), columnMeta.getInt(5), columnMeta.getString(6), columnMeta.getInt(7), getInt(columnMeta.getString(8)), columnMeta.getInt(9), columnMeta.getInt(10), columnMeta.getInt(11), columnMeta.getString(12), columnMeta.getString(13), getInt(columnMeta.getString(14)), getInt(columnMeta.getString(15)), columnMeta.getInt(16), columnMeta.getInt(17), columnMeta.getString(18), columnMeta.getString(19), columnMeta.getString(20), columnMeta.getString(21), getShort(columnMeta.getString(22)), columnMeta.getString(23));
-
- if (!cubedOnly || getProjectManager().isExposedColumn(project, schemaName + "." + colmnMeta.getTABLE_NAME(), colmnMeta.getCOLUMN_NAME())) {
+ ColumnMetaWithType colmnMeta = new ColumnMetaWithType(
+ catalogName == null ? Constant.FakeCatalogName : catalogName,
+ schemaName == null ? Constant.FakeSchemaName : schemaName, columnMeta.getString(3),
+ columnMeta.getString(4), columnMeta.getInt(5), columnMeta.getString(6), columnMeta.getInt(7),
+ getInt(columnMeta.getString(8)), columnMeta.getInt(9), columnMeta.getInt(10),
+ columnMeta.getInt(11), columnMeta.getString(12), columnMeta.getString(13),
+ getInt(columnMeta.getString(14)), getInt(columnMeta.getString(15)), columnMeta.getInt(16),
+ columnMeta.getInt(17), columnMeta.getString(18), columnMeta.getString(19),
+ columnMeta.getString(20), columnMeta.getString(21), getShort(columnMeta.getString(22)),
+ columnMeta.getString(23));
+
+ if (!cubedOnly || getProjectManager().isExposedColumn(project,
+ schemaName + "." + colmnMeta.getTABLE_NAME(), colmnMeta.getCOLUMN_NAME())) {
tableMap.get(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME()).addColumn(colmnMeta);
- columnMap.put(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME() + "#" + colmnMeta.getCOLUMN_NAME(), colmnMeta);
+ columnMap.put(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME() + "#"
+ + colmnMeta.getCOLUMN_NAME(), colmnMeta);
}
}
@@ -638,7 +682,8 @@ public class QueryService extends BasicService {
for (JoinTableDesc joinTableDesc : dataModelDesc.getJoinTables()) {
JoinDesc joinDesc = joinTableDesc.getJoin();
for (String pk : joinDesc.getPrimaryKey()) {
- String columnIdentity = (dataModelDesc.findTable(pk.substring(0, pk.indexOf("."))).getTableIdentity() + pk.substring(pk.indexOf("."))).replace('.', '#');
+ String columnIdentity = (dataModelDesc.findTable(pk.substring(0, pk.indexOf(".")))
+ .getTableIdentity() + pk.substring(pk.indexOf("."))).replace('.', '#');
if (columnMap.containsKey(columnIdentity)) {
columnMap.get(columnIdentity).getTYPE().add(ColumnMetaWithType.columnTypeEnum.PK);
} else {
@@ -647,7 +692,8 @@ public class QueryService extends BasicService {
}
for (String fk : joinDesc.getForeignKey()) {
- String columnIdentity = (dataModelDesc.findTable(fk.substring(0, fk.indexOf("."))).getTableIdentity() + fk.substring(fk.indexOf("."))).replace('.', '#');
+ String columnIdentity = (dataModelDesc.findTable(fk.substring(0, fk.indexOf(".")))
+ .getTableIdentity() + fk.substring(fk.indexOf("."))).replace('.', '#');
if (columnMap.containsKey(columnIdentity)) {
columnMap.get(columnIdentity).getTYPE().add(ColumnMetaWithType.columnTypeEnum.FK);
} else {
@@ -660,7 +706,8 @@ public class QueryService extends BasicService {
List<ModelDimensionDesc> dimensions = dataModelDesc.getDimensions();
for (ModelDimensionDesc dimension : dimensions) {
for (String column : dimension.getColumns()) {
- String columnIdentity = (dataModelDesc.findTable(dimension.getTable()).getTableIdentity() + "." + column).replace('.', '#');
+ String columnIdentity = (dataModelDesc.findTable(dimension.getTable()).getTableIdentity() + "."
+ + column).replace('.', '#');
if (columnMap.containsKey(columnIdentity)) {
columnMap.get(columnIdentity).getTYPE().add(ColumnMetaWithType.columnTypeEnum.DIMENSION);
} else {
@@ -672,7 +719,8 @@ public class QueryService extends BasicService {
String[] measures = dataModelDesc.getMetrics();
for (String measure : measures) {
- String columnIdentity = (dataModelDesc.findTable(measure.substring(0, measure.indexOf("."))).getTableIdentity() + measure.substring(measure.indexOf("."))).replace('.', '#');
+ String columnIdentity = (dataModelDesc.findTable(measure.substring(0, measure.indexOf(".")))
+ .getTableIdentity() + measure.substring(measure.indexOf("."))).replace('.', '#');
if (columnMap.containsKey(columnIdentity)) {
columnMap.get(columnIdentity).getTYPE().add(ColumnMetaWithType.columnTypeEnum.MEASURE);
} else {
@@ -703,7 +751,7 @@ public class QueryService extends BasicService {
Connection conn = null;
Statement stat = null;
ResultSet resultSet = null;
- Boolean isAdHoc = false;
+ Boolean isPushDown = false;
List<List<String>> results = Lists.newArrayList();
List<SelectedColumnMeta> columnMetas = Lists.newArrayList();
@@ -713,7 +761,7 @@ public class QueryService extends BasicService {
// special case for prepare query.
if (BackdoorToggles.getPrepareOnly()) {
- return getPrepareOnlySqlResponse(correctedSql, conn, isAdHoc, results, columnMetas);
+ return getPrepareOnlySqlResponse(correctedSql, conn, isPushDown, results, columnMetas);
}
stat = conn.createStatement();
@@ -725,7 +773,13 @@ public class QueryService extends BasicService {
// Fill in selected column meta
for (int i = 1; i <= columnCount; ++i) {
- columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), metaData.isSearchable(i), metaData.isCurrency(i), metaData.isNullable(i), metaData.isSigned(i), metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i), metaData.getSchemaName(i), metaData.getCatalogName(i), metaData.getTableName(i), metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), metaData.getColumnTypeName(i), metaData.isReadOnly(i), metaData.isWritable(i), metaData.isDefinitelyWritable(i)));
+ columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i),
+ metaData.isSearchable(i), metaData.isCurrency(i), metaData.isNullable(i), metaData.isSigned(i),
+ metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i),
+ metaData.getSchemaName(i), metaData.getCatalogName(i), metaData.getTableName(i),
+ metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i),
+ metaData.getColumnTypeName(i), metaData.isReadOnly(i), metaData.isWritable(i),
+ metaData.isDefinitelyWritable(i)));
}
// fill in results
@@ -738,17 +792,17 @@ public class QueryService extends BasicService {
results.add(oneRow);
}
} catch (SQLException sqlException) {
- isAdHoc = AdHocUtil.doAdHocQuery(sqlRequest.getProject(), correctedSql, results, columnMetas, sqlException);
+ isPushDown = PushDownUtil.doPushDownQuery(sqlRequest.getProject(), correctedSql, results, columnMetas,
+ sqlException);
} finally {
close(resultSet, stat, conn);
}
- return getSqlResponse(isAdHoc, results, columnMetas);
+ return getSqlResponse(isPushDown, results, columnMetas);
}
- private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection conn,
- Boolean isAdHoc, List<List<String>> results, List<SelectedColumnMeta> columnMetas)
- throws SQLException {
+ private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection conn, Boolean isPushDown,
+ List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws SQLException {
CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(true);
@@ -772,7 +826,12 @@ public class QueryService extends BasicService {
String columnName = field.getKey();
BasicSqlType basicSqlType = (BasicSqlType) field.getValue();
- columnMetas.add(new SelectedColumnMeta(false, config.caseSensitive(), false, false, basicSqlType.isNullable() ? 1 : 0, true, basicSqlType.getPrecision(), columnName, columnName, null, null, null, basicSqlType.getPrecision(), basicSqlType.getScale() < 0 ? 0 : basicSqlType.getScale(), basicSqlType.getSqlTypeName().getJdbcOrdinal(), basicSqlType.getSqlTypeName().getName(), true, false, false));
+ columnMetas.add(new SelectedColumnMeta(false, config.caseSensitive(), false, false,
+ basicSqlType.isNullable() ? 1 : 0, true, basicSqlType.getPrecision(), columnName,
+ columnName, null, null, null, basicSqlType.getPrecision(),
+ basicSqlType.getScale() < 0 ? 0 : basicSqlType.getScale(),
+ basicSqlType.getSqlTypeName().getJdbcOrdinal(), basicSqlType.getSqlTypeName().getName(),
+ true, false, false));
}
} else {
@@ -782,10 +841,11 @@ public class QueryService extends BasicService {
CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(false);
}
- return getSqlResponse(isAdHoc, results, columnMetas);
+ return getSqlResponse(isPushDown, results, columnMetas);
}
- private SQLResponse getSqlResponse(Boolean isAdHoc, List<List<String>> results, List<SelectedColumnMeta> columnMetas) {
+ private SQLResponse getSqlResponse(Boolean isPushDown, List<List<String>> results,
+ List<SelectedColumnMeta> columnMetas) {
boolean isPartialResult = false;
StringBuilder cubeSb = new StringBuilder();
@@ -804,7 +864,8 @@ public class QueryService extends BasicService {
}
logger.info(logSb.toString());
- SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, false, null, isPartialResult, isAdHoc);
+ SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, false, null, isPartialResult,
+ isPushDown);
response.setTotalScanCount(QueryContext.current().getScannedRows());
response.setTotalScanBytes(QueryContext.current().getScannedBytes());
return response;
@@ -815,7 +876,8 @@ public class QueryService extends BasicService {
* @param param
* @throws SQLException
*/
- private void setParam(PreparedStatement preparedState, int index, PrepareSqlRequest.StateParam param) throws SQLException {
+ private void setParam(PreparedStatement preparedState, int index, PrepareSqlRequest.StateParam param)
+ throws SQLException {
boolean isNull = (null == param.getValue());
Class<?> clazz;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
deleted file mode 100644
index 0eff508..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.util;
-
-import java.sql.SQLException;
-import java.util.Collections;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.apache.commons.lang3.tuple.Triple;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
-import org.apache.kylin.query.routing.NoRealizationFoundException;
-import org.apache.kylin.source.adhocquery.IAdHocConverter;
-import org.apache.kylin.source.adhocquery.IAdHocRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class AdHocUtil {
- private static final Logger logger = LoggerFactory.getLogger(AdHocUtil.class);
-
- public static boolean doAdHocQuery(String project, String sql, List<List<String>> results,
- List<SelectedColumnMeta> columnMetas, SQLException sqlException) throws Exception {
-
- boolean isExpectedCause = (ExceptionUtils.getRootCause(sqlException).getClass()
- .equals(NoRealizationFoundException.class));
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
-
- if (isExpectedCause && kylinConfig.isAdhocEnabled()) {
-
- logger.info("Query failed to utilize pre-calculation, routing to other engines", sqlException);
- IAdHocRunner runner = (IAdHocRunner) ClassUtil.newInstance(kylinConfig.getAdHocRunnerClassName());
- IAdHocConverter converter = (IAdHocConverter) ClassUtil
- .newInstance(kylinConfig.getAdHocConverterClassName());
-
- runner.init(kylinConfig);
-
- logger.debug("Ad-hoc query runner {}", runner);
-
- String expandCC = restoreComputedColumnToExpr(sql, project);
- if (!StringUtils.equals(expandCC, sql)) {
- logger.info("computed column in sql is expanded to: " + expandCC);
- }
- String adhocSql = converter.convert(expandCC);
- if (!adhocSql.equals(expandCC)) {
- logger.info("the query is converted to {} according to kylin.query.ad-hoc.converter-class-name",
- adhocSql);
- }
-
- runner.executeQuery(adhocSql, results, columnMetas);
-
- return true;
- } else {
- throw sqlException;
- }
- }
-
- private final static Pattern identifierInSqlPattern = Pattern.compile(
- //find pattern like "table"."column" or "column"
- "((?<![\\p{L}_0-9\\.\\\"])(\\\"[\\p{L}_0-9]+\\\"\\.)?(\\\"[\\p{L}_0-9]+\\\")(?![\\p{L}_0-9\\.\\\"]))" + "|"
- //find pattern like table.column or column
- + "((?<![\\p{L}_0-9\\.\\\"])([\\p{L}_0-9]+\\.)?([\\p{L}_0-9]+)(?![\\p{L}_0-9\\.\\\"]))");
-
- private final static Pattern identifierInExprPattern = Pattern.compile(
- // a.b.c
- "((?<![\\p{L}_0-9\\.\\\"])([\\p{L}_0-9]+\\.)([\\p{L}_0-9]+\\.)([\\p{L}_0-9]+)(?![\\p{L}_0-9\\.\\\"]))");
-
- private final static Pattern endWithAsPattern = Pattern.compile("\\s+as\\s+$", Pattern.CASE_INSENSITIVE);
-
- public static String restoreComputedColumnToExpr(String beforeSql, String project) {
- final MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- List<DataModelDesc> dataModelDescs = metadataManager.getModels(project);
-
- String afterSql = beforeSql;
- for (DataModelDesc dataModelDesc : dataModelDescs) {
- for (ComputedColumnDesc computedColumnDesc : dataModelDesc.getComputedColumnDescs()) {
- afterSql = restoreComputedColumnToExpr(afterSql, computedColumnDesc);
- }
- }
-
- return afterSql;
- }
-
- static String restoreComputedColumnToExpr(String sql, ComputedColumnDesc computedColumnDesc) {
-
- String ccName = computedColumnDesc.getColumnName();
- List<Triple<Integer, Integer, String>> replacements = Lists.newArrayList();
- Matcher matcher = identifierInSqlPattern.matcher(sql);
-
- while (matcher.find()) {
- if (matcher.group(1) != null) { //with quote case: "TABLE"."COLUMN"
-
- String quotedColumnName = matcher.group(3);
- Preconditions.checkNotNull(quotedColumnName);
- String columnName = StringUtils.strip(quotedColumnName, "\"");
- if (!columnName.equalsIgnoreCase(ccName)) {
- continue;
- }
-
- if (matcher.group(2) != null) { // table name exist
- String quotedTableAlias = StringUtils.strip(matcher.group(2), ".");
- String tableAlias = StringUtils.strip(quotedTableAlias, "\"");
- replacements.add(Triple.of(matcher.start(1), matcher.end(1),
- replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, true)));
- } else { //only column
- if (endWithAsPattern.matcher(sql.substring(0, matcher.start(1))).find()) {
- //select DEAL_AMOUNT as "deal_amount" case
- continue;
- }
- replacements.add(Triple.of(matcher.start(1), matcher.end(1),
- replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, true)));
- }
- } else if (matcher.group(4) != null) { //without quote case: table.column or simply column
- String columnName = matcher.group(6);
- Preconditions.checkNotNull(columnName);
- if (!columnName.equalsIgnoreCase(ccName)) {
- continue;
- }
-
- if (matcher.group(5) != null) { //table name exist
- String tableAlias = StringUtils.strip(matcher.group(5), ".");
- replacements.add(Triple.of(matcher.start(4), matcher.end(4),
- replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, false)));
-
- } else { //only column
- if (endWithAsPattern.matcher(sql.substring(0, matcher.start(4))).find()) {
- //select DEAL_AMOUNT as deal_amount case
- continue;
- }
- replacements.add(Triple.of(matcher.start(4), matcher.end(4),
- replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, false)));
- }
- }
- }
-
- Collections.reverse(replacements);
- for (Triple<Integer, Integer, String> triple : replacements) {
- sql = sql.substring(0, triple.getLeft()) + "(" + triple.getRight() + ")"
- + sql.substring(triple.getMiddle());
- }
- return sql;
- }
-
- static String replaceIdentifierInExpr(String expr, String tableAlias, boolean quoted) {
- if (tableAlias == null) {
- return expr;
- }
-
- return CalciteParser.insertAliasInExpr(expr, tableAlias);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
new file mode 100644
index 0000000..5d7f47a
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.util;
+
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
+import org.apache.kylin.query.routing.NoRealizationFoundException;
+import org.apache.kylin.source.adhocquery.IPushDownConverter;
+import org.apache.kylin.source.adhocquery.IPushDownRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class PushDownUtil {
+ private static final Logger logger = LoggerFactory.getLogger(PushDownUtil.class);
+
+ public static boolean doPushDownQuery(String project, String sql, List<List<String>> results,
+ List<SelectedColumnMeta> columnMetas, SQLException sqlException) throws Exception {
+
+ boolean isExpectedCause = (ExceptionUtils.getRootCause(sqlException).getClass()
+ .equals(NoRealizationFoundException.class));
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+
+ if (isExpectedCause && kylinConfig.isPushDownEnabled()) {
+
+ logger.info("Query failed to utilize pre-calculation, routing to other engines", sqlException);
+ IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(kylinConfig.getPushDownRunnerClassName());
+ IPushDownConverter converter = (IPushDownConverter) ClassUtil
+ .newInstance(kylinConfig.getPushDownConverterClassName());
+
+ runner.init(kylinConfig);
+
+ logger.debug("Query pushdown runner {}", runner);
+
+ String expandCC = restoreComputedColumnToExpr(sql, project);
+ if (!StringUtils.equals(expandCC, sql)) {
+ logger.info("computed column in sql is expanded to: " + expandCC);
+ }
+ String adhocSql = converter.convert(expandCC);
+ if (!adhocSql.equals(expandCC)) {
+ logger.info("the query is converted to {} according to kylin.query.pushdown.converter-class-name",
+ adhocSql);
+ }
+
+ runner.executeQuery(adhocSql, results, columnMetas);
+
+ return true;
+ } else {
+ throw sqlException;
+ }
+ }
+
+ private final static Pattern identifierInSqlPattern = Pattern.compile(
+ //find pattern like "table"."column" or "column"
+ "((?<![\\p{L}_0-9\\.\\\"])(\\\"[\\p{L}_0-9]+\\\"\\.)?(\\\"[\\p{L}_0-9]+\\\")(?![\\p{L}_0-9\\.\\\"]))" + "|"
+ //find pattern like table.column or column
+ + "((?<![\\p{L}_0-9\\.\\\"])([\\p{L}_0-9]+\\.)?([\\p{L}_0-9]+)(?![\\p{L}_0-9\\.\\\"]))");
+
+ private final static Pattern identifierInExprPattern = Pattern.compile(
+ // a.b.c
+ "((?<![\\p{L}_0-9\\.\\\"])([\\p{L}_0-9]+\\.)([\\p{L}_0-9]+\\.)([\\p{L}_0-9]+)(?![\\p{L}_0-9\\.\\\"]))");
+
+ private final static Pattern endWithAsPattern = Pattern.compile("\\s+as\\s+$", Pattern.CASE_INSENSITIVE);
+
+ public static String restoreComputedColumnToExpr(String beforeSql, String project) {
+ final MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ List<DataModelDesc> dataModelDescs = metadataManager.getModels(project);
+
+ String afterSql = beforeSql;
+ for (DataModelDesc dataModelDesc : dataModelDescs) {
+ for (ComputedColumnDesc computedColumnDesc : dataModelDesc.getComputedColumnDescs()) {
+ afterSql = restoreComputedColumnToExpr(afterSql, computedColumnDesc);
+ }
+ }
+ return afterSql;
+ }
+
+ static String restoreComputedColumnToExpr(String sql, ComputedColumnDesc computedColumnDesc) {
+
+ String ccName = computedColumnDesc.getColumnName();
+ List<Triple<Integer, Integer, String>> replacements = Lists.newArrayList();
+ Matcher matcher = identifierInSqlPattern.matcher(sql);
+
+ while (matcher.find()) {
+ if (matcher.group(1) != null) { //with quote case: "TABLE"."COLUMN"
+
+ String quotedColumnName = matcher.group(3);
+ Preconditions.checkNotNull(quotedColumnName);
+ String columnName = StringUtils.strip(quotedColumnName, "\"");
+ if (!columnName.equalsIgnoreCase(ccName)) {
+ continue;
+ }
+
+ if (matcher.group(2) != null) { // table name exist
+ String quotedTableAlias = StringUtils.strip(matcher.group(2), ".");
+ String tableAlias = StringUtils.strip(quotedTableAlias, "\"");
+ replacements.add(Triple.of(matcher.start(1), matcher.end(1),
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, true)));
+ } else { //only column
+ if (endWithAsPattern.matcher(sql.substring(0, matcher.start(1))).find()) {
+ //select DEAL_AMOUNT as "deal_amount" case
+ continue;
+ }
+ replacements.add(Triple.of(matcher.start(1), matcher.end(1),
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, true)));
+ }
+ } else if (matcher.group(4) != null) { //without quote case: table.column or simply column
+ String columnName = matcher.group(6);
+ Preconditions.checkNotNull(columnName);
+ if (!columnName.equalsIgnoreCase(ccName)) {
+ continue;
+ }
+
+ if (matcher.group(5) != null) { //table name exist
+ String tableAlias = StringUtils.strip(matcher.group(5), ".");
+ replacements.add(Triple.of(matcher.start(4), matcher.end(4),
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, false)));
+
+ } else { //only column
+ if (endWithAsPattern.matcher(sql.substring(0, matcher.start(4))).find()) {
+ //select DEAL_AMOUNT as deal_amount case
+ continue;
+ }
+ replacements.add(Triple.of(matcher.start(4), matcher.end(4),
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, false)));
+ }
+ }
+ }
+
+ Collections.reverse(replacements);
+ for (Triple<Integer, Integer, String> triple : replacements) {
+ sql = sql.substring(0, triple.getLeft()) + "(" + triple.getRight() + ")"
+ + sql.substring(triple.getMiddle());
+ }
+ return sql;
+ }
+
+ static String replaceIdentifierInExpr(String expr, String tableAlias, boolean quoted) {
+ if (tableAlias == null) {
+ return expr;
+ }
+
+ return CalciteParser.insertAliasInExpr(expr, tableAlias);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
deleted file mode 100644
index 05b135a..0000000
--- a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.rest.util;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class AdHocUtilTest {
-
- @Test
- public void testReplaceIdentifierInExpr() {
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("x * y", null, false);
- Assert.assertEquals("x * y", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("x_3 * y_3", "b_2", false);
- Assert.assertEquals("b_2.x_3 * b_2.y_3", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("substr(x,1,3)>y", "c", true);
- Assert.assertEquals("substr(c.x,1,3)>c.y", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", "c", true);
- Assert.assertEquals("strcmp(substr(c.x,1,3),c.y)", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, true);
- Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, false);
- Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
- }
- }
-
- @Test
- public void testRestoreComputedColumnToExpr() {
-
- ComputedColumnDesc computedColumnDesc = mock(ComputedColumnDesc.class);
- when(computedColumnDesc.getColumnName()).thenReturn("DEAL_AMOUNT");
- when(computedColumnDesc.getExpression()).thenReturn("price * number");
-
- {
- String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", computedColumnDesc);
- Assert.assertEquals("select (price * number) from DB.TABLE group by date order by (price * number)", ret);
- }
- {
- String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select DEAL_AMOUNT as DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT",
- computedColumnDesc);
- Assert.assertEquals(
- "select (price * number) as DEAL_AMOUNT from DB.TABLE group by date order by (price * number)",
- ret);
- }
- {
- String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select \"DEAL_AMOUNT\" AS deal_amount from DB.TABLE group by date order by DEAL_AMOUNT",
- computedColumnDesc);
- Assert.assertEquals(
- "select (price * number) AS deal_amount from DB.TABLE group by date order by (price * number)",
- ret);
- }
- {
- String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select x.DEAL_AMOUNT AS deal_amount from DB.TABLE x group by date order by x.DEAL_AMOUNT",
- computedColumnDesc);
- Assert.assertEquals(
- "select (x.price * x.number) AS deal_amount from DB.TABLE x group by date order by (x.price * x.number)",
- ret);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/test/java/org/apache/kylin/rest/util/PushDownUtilTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/PushDownUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/PushDownUtilTest.java
new file mode 100644
index 0000000..5302a70
--- /dev/null
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/PushDownUtilTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.rest.util;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class PushDownUtilTest {
+
+ @Test
+ public void testReplaceIdentifierInExpr() {
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("x * y", null, false);
+ Assert.assertEquals("x * y", ret);
+ }
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("x_3 * y_3", "b_2", false);
+ Assert.assertEquals("b_2.x_3 * b_2.y_3", ret);
+ }
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("substr(x,1,3)>y", "c", true);
+ Assert.assertEquals("substr(c.x,1,3)>c.y", ret);
+ }
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", "c", true);
+ Assert.assertEquals("strcmp(substr(c.x,1,3),c.y)", ret);
+ }
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, true);
+ Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
+ }
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, false);
+ Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
+ }
+ }
+
+ @Test
+ public void testRestoreComputedColumnToExpr() {
+
+ ComputedColumnDesc computedColumnDesc = mock(ComputedColumnDesc.class);
+ when(computedColumnDesc.getColumnName()).thenReturn("DEAL_AMOUNT");
+ when(computedColumnDesc.getExpression()).thenReturn("price * number");
+
+ {
+ String ret = PushDownUtil.restoreComputedColumnToExpr(
+ "select DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", computedColumnDesc);
+ Assert.assertEquals("select (price * number) from DB.TABLE group by date order by (price * number)", ret);
+ }
+ {
+ String ret = PushDownUtil.restoreComputedColumnToExpr(
+ "select DEAL_AMOUNT as DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT",
+ computedColumnDesc);
+ Assert.assertEquals(
+ "select (price * number) as DEAL_AMOUNT from DB.TABLE group by date order by (price * number)",
+ ret);
+ }
+ {
+ String ret = PushDownUtil.restoreComputedColumnToExpr(
+ "select \"DEAL_AMOUNT\" AS deal_amount from DB.TABLE group by date order by DEAL_AMOUNT",
+ computedColumnDesc);
+ Assert.assertEquals(
+ "select (price * number) AS deal_amount from DB.TABLE group by date order by (price * number)",
+ ret);
+ }
+ {
+ String ret = PushDownUtil.restoreComputedColumnToExpr(
+ "select x.DEAL_AMOUNT AS deal_amount from DB.TABLE x group by date order by x.DEAL_AMOUNT",
+ computedColumnDesc);
+ Assert.assertEquals(
+ "select (x.price * x.number) AS deal_amount from DB.TABLE x group by date order by (x.price * x.number)",
+ ret);
+ }
+ }
+}
[21/26] kylin git commit: minor, roll back validate join key
Posted by li...@apache.org.
minor, roll back validate join key
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ae8ca7c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ae8ca7c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ae8ca7c
Branch: refs/heads/kylin-2.1.x
Commit: 3ae8ca7cd3796583787e0657edec31b8bdb30ce1
Parents: dc232d1
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Jun 30 20:26:31 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 20:29:11 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 31 --------------------
1 file changed, 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ae8ca7c/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index abee71b..bc35e2a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -617,38 +617,7 @@ public class DataModelDesc extends RootPersistentEntity {
+ fkCols[i].getDatatype());
}
}
-
- if (!isSpecialTestModel()) {
- for (TblColRef t : fkCols) {
- if (!isJoinKeyDimension(t.getIdentity())) {
- throw new IllegalStateException(
- "Foreigner key: " + t.getIdentity() + " should be a dimension.");
- }
- }
-
- for (TblColRef t : pkCols) {
- if (!isJoinKeyDimension(t.getIdentity())) {
- throw new IllegalStateException("Primary key: " + t.getIdentity() + " should be a dimension.");
- }
- }
- }
- }
- }
-
- private boolean isJoinKeyDimension(String key) {
- for (ModelDimensionDesc d : dimensions) {
- for (String col : d.getColumns()) {
- if (key.equals(d.getTable() + '.' + col))
- return true;
- }
}
- return false;
- }
-
- private boolean isSpecialTestModel() {
- if (config.isDevEnv() && (name.startsWith("test_kylin_") || name.startsWith("test_streaming")))
- return true;
- return false;
}
private void initJoinsTree() {
[26/26] kylin git commit: minor, rename adhoc to pushdown
Posted by li...@apache.org.
minor, rename adhoc to pushdown
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/82b578a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/82b578a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/82b578a5
Branch: refs/heads/kylin-2.1.x
Commit: 82b578a561de76e93dcd0b56473bbd229e1292b9
Parents: 4f4382d
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Jul 1 12:25:21 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Sat Jul 1 12:30:52 2017 +0800
----------------------------------------------------------------------
build/smoke-test/sql/sql1.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/82b578a5/build/smoke-test/sql/sql1.json
----------------------------------------------------------------------
diff --git a/build/smoke-test/sql/sql1.json b/build/smoke-test/sql/sql1.json
index 7b90d0f..a8d01b0 100644
--- a/build/smoke-test/sql/sql1.json
+++ b/build/smoke-test/sql/sql1.json
@@ -3,7 +3,7 @@
"partial": false,
"affectedRowCount": 0,
"isException": false,
- "adHoc": false,
+ "pushDown": false,
"results": [
[
"10000"