You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ka...@apache.org on 2017/07/07 08:38:03 UTC
[01/50] [abbrv] kylin git commit: KYLIN-2592 Fix serialization issue
for distinct count in Spark cubing 2 [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2606 d610a6dd5 -> 1f40225d1 (forced update)
KYLIN-2592 Fix serialization issue for distinct count in Spark cubing 2
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a9b0b6ed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a9b0b6ed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a9b0b6ed
Branch: refs/heads/KYLIN-2606
Commit: a9b0b6ed3de04dbc2395e44dd6f7241c466a3d8b
Parents: 2b07f29
Author: kangkaisen <ka...@live.com>
Authored: Mon Jun 26 22:08:57 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jun 27 19:17:59 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9b0b6ed/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index 6011ab2..106653f 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -96,6 +96,7 @@ public class KylinKryoRegistrator implements KryoRegistrator {
addClassQuitely(kyroClasses, "com.google.common.collect.EmptyImmutableList");
addClassQuitely(kyroClasses, "java.nio.HeapShortBuffer");
+ addClassQuitely(kyroClasses, "java.nio.HeapLongBuffer");
addClassQuitely(kyroClasses, "scala.collection.immutable.Map$EmptyMap$");
addClassQuitely(kyroClasses, "org.apache.spark.sql.catalyst.expressions.GenericInternalRow");
addClassQuitely(kyroClasses, "org.apache.spark.unsafe.types.UTF8String");
[39/50] [abbrv] kylin git commit: minor,
rename adhoc to pushdown in KYLIN
Posted by ka...@apache.org.
minor, rename adhoc to pushdown in KYLIN
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7002dd86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7002dd86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7002dd86
Branch: refs/heads/KYLIN-2606
Commit: 7002dd864a3f43a2e97a022d1d348773609f446d
Parents: d91f522
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Jun 30 21:12:54 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 21:18:10 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 28 +-
.../main/resources/kylin-defaults.properties | 18 +-
.../kylin/metadata/project/ProjectManager.java | 8 +-
.../source/adhocquery/HiveAdhocConverter.java | 297 -------------------
.../adhocquery/HivePushDownConverter.java | 297 +++++++++++++++++++
.../source/adhocquery/IAdHocConverter.java | 25 --
.../kylin/source/adhocquery/IAdHocRunner.java | 39 ---
.../source/adhocquery/IPushDownConverter.java | 25 ++
.../source/adhocquery/IPushDownRunner.java | 39 +++
.../adhocquery/HiveAdhocConverterTest.java | 78 -----
.../adhocquery/HivePushDownConverterTest.java | 78 +++++
.../test_case_data/sandbox/kylin.properties | 18 +-
.../org/apache/kylin/query/KylinTestBase.java | 4 +-
.../kylin/query/adhoc/AdHocRunnerJdbcImpl.java | 122 --------
.../query/adhoc/PushDownRunnerJdbcImpl.java | 122 ++++++++
.../apache/kylin/rest/response/SQLResponse.java | 17 +-
.../apache/kylin/rest/service/QueryService.java | 150 +++++++---
.../org/apache/kylin/rest/util/AdHocUtil.java | 178 -----------
.../apache/kylin/rest/util/PushDownUtil.java | 177 +++++++++++
.../apache/kylin/rest/util/AdHocUtilTest.java | 94 ------
.../kylin/rest/util/PushDownUtilTest.java | 94 ++++++
21 files changed, 986 insertions(+), 922 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4a21ded..179d61f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1022,45 +1022,45 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
}
- public boolean isAdhocEnabled() {
- return StringUtils.isNotEmpty(getAdHocRunnerClassName());
+ public boolean isPushDownEnabled() {
+ return StringUtils.isNotEmpty(getPushDownRunnerClassName());
}
- public String getAdHocRunnerClassName() {
- return getOptional("kylin.query.ad-hoc.runner-class-name", "");
+ public String getPushDownRunnerClassName() {
+ return getOptional("kylin.query.pushdown.runner-class-name", "");
}
- public String getAdHocConverterClassName() {
- return getOptional("kylin.query.ad-hoc.converter-class-name",
- "org.apache.kylin.source.adhocquery.HiveAdhocConverter");
+ public String getPushDownConverterClassName() {
+ return getOptional("kylin.query.pushdown.converter-class-name",
+ "org.apache.kylin.source.adhocquery.HivePushDownConverter");
}
public String getJdbcUrl() {
- return getOptional("kylin.query.ad-hoc.jdbc.url", "");
+ return getOptional("kylin.query.pushdown.jdbc.url", "");
}
public String getJdbcDriverClass() {
- return getOptional("kylin.query.ad-hoc.jdbc.driver", "");
+ return getOptional("kylin.query.pushdown.jdbc.driver", "");
}
public String getJdbcUsername() {
- return getOptional("kylin.query.ad-hoc.jdbc.username", "");
+ return getOptional("kylin.query.pushdown.jdbc.username", "");
}
public String getJdbcPassword() {
- return getOptional("kylin.query.ad-hoc.jdbc.password", "");
+ return getOptional("kylin.query.pushdown.jdbc.password", "");
}
public int getPoolMaxTotal() {
- return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-max-total", "8"));
+ return Integer.parseInt(this.getOptional("kylin.query.pushdown.jdbc.pool-max-total", "8"));
}
public int getPoolMaxIdle() {
- return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-max-idle", "8"));
+ return Integer.parseInt(this.getOptional("kylin.query.pushdown.jdbc.pool-max-idle", "8"));
}
public int getPoolMinIdle() {
- return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-min-idle", "0"));
+ return Integer.parseInt(this.getOptional("kylin.query.pushdown.jdbc.pool-min-idle", "0"));
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index beac63c..cb511e7 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -237,15 +237,15 @@ kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-histo
#kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current
-### AD-HOC QUERY ###
+### QUERY PUSH DOWN ###
-#kylin.query.ad-hoc.runner-class-name=org.apache.kylin.query.adhoc.AdHocRunnerJdbcImpl
+#kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl
-#kylin.query.ad-hoc.jdbc.url=jdbc:hive2://sandbox:10000/default
-#kylin.query.ad-hoc.jdbc.driver=org.apache.hive.jdbc.HiveDriver
-#kylin.query.ad-hoc.jdbc.username=hive
-#kylin.query.ad-hoc.jdbc.password=
+#kylin.query.pushdown.jdbc.url=jdbc:hive2://sandbox:10000/default
+#kylin.query.pushdown.jdbc.driver=org.apache.hive.jdbc.HiveDriver
+#kylin.query.pushdown.jdbc.username=hive
+#kylin.query.pushdown.jdbc.password=
-#kylin.query.ad-hoc.jdbc.pool-max-total=8
-#kylin.query.ad-hoc.jdbc.pool-max-idle=8
-#kylin.query.ad-hoc.jdbc.pool-min-idle=0
+#kylin.query.pushdown.jdbc.pool-max-total=8
+#kylin.query.pushdown.jdbc.pool-max-idle=8
+#kylin.query.pushdown.jdbc.pool-min-idle=0
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index ea03c3c..b801d8c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -437,7 +437,7 @@ public class ProjectManager {
}
public Collection<TableDesc> listExposedTables(String project) {
- return config.isAdhocEnabled() ? //
+ return config.isPushDownEnabled() ? //
this.listDefinedTables(project) : //
l2Cache.listExposedTables(norm(project));
}
@@ -445,7 +445,7 @@ public class ProjectManager {
public List<ColumnDesc> listExposedColumns(String project, TableDesc tableDesc) {
Set<ColumnDesc> exposedColumns = l2Cache.listExposedColumns(norm(project), tableDesc.getIdentity());
- if (config.isAdhocEnabled()) {
+ if (config.isPushDownEnabled()) {
// take care of computed columns
Set<ColumnDesc> dedup = Sets.newHashSet(tableDesc.getColumns());
dedup.addAll(exposedColumns);
@@ -456,13 +456,13 @@ public class ProjectManager {
}
public boolean isExposedTable(String project, String table) {
- return config.isAdhocEnabled() ? //
+ return config.isPushDownEnabled() ? //
l2Cache.isDefinedTable(norm(project), table) : //
l2Cache.isExposedTable(norm(project), table);
}
public boolean isExposedColumn(String project, String table, String col) {
- return config.isAdhocEnabled() ? //
+ return config.isPushDownEnabled() ? //
l2Cache.isDefinedColumn(norm(project), table, col) || l2Cache.isExposedColumn(norm(project), table, col)
: //
l2Cache.isExposedColumn(norm(project), table, col);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
deleted file mode 100644
index e8c06ff..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.source.adhocquery;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Stack;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.ImmutableSet;
-
-//TODO: Some workaround ways to make sql readable by hive parser, should replaced it with a more well-designed way
-public class HiveAdhocConverter implements IAdHocConverter {
-
- @SuppressWarnings("unused")
- private static final Logger logger = LoggerFactory.getLogger(HiveAdhocConverter.class);
-
- private static final Pattern EXTRACT_PATTERN = Pattern.compile("extract\\s*(\\()\\s*(.*?)\\s*from(\\s+)",
- Pattern.CASE_INSENSITIVE);
- private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\()\\s*select\\s",
- Pattern.CASE_INSENSITIVE);
- private static final Pattern ALIAS_PATTERN = Pattern.compile("\\s+([`'_a-z0-9A-Z]+)", Pattern.CASE_INSENSITIVE);
- private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\((.*?) (?i)AS\\s*(.*?)\\s*\\)",
- Pattern.CASE_INSENSITIVE);
- private static final Pattern CONCAT_PATTERN = Pattern.compile("(['_a-z0-9A-Z]+)\\|\\|(['_a-z0-9A-Z]+)",
- Pattern.CASE_INSENSITIVE);
- private static final Pattern TIMESTAMPADD_PATTERN = Pattern.compile("timestampadd\\s*\\(\\s*(.*?)\\s*,",
- Pattern.CASE_INSENSITIVE);
- private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("A", "ABS", "ABSOLUTE", "ACTION",
- "ADA", "ADD", "ADMIN", "AFTER", "ALL", "ALLOCATE", "ALLOW", "ALTER", "ALWAYS", "AND", "ANY", "APPLY", "ARE",
- "ARRAY", "ARRAY_MAX_CARDINALITY", "ASC", "ASENSITIVE", "ASSERTION", "ASSIGNMENT", "ASYMMETRIC", "AT",
- "ATOMIC", "ATTRIBUTE", "ATTRIBUTES", "AUTHORIZATION", "AVG", "BEFORE", "BEGIN", "BEGIN_FRAME",
- "BEGIN_PARTITION", "BERNOULLI", "BETWEEN", "BIGINT", "BINARY", "BIT", "BLOB", "BOOLEAN", "BOTH", "BREADTH",
- "BY", "C", "CALL", "CALLED", "CARDINALITY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG",
- "CATALOG_NAME", "CEIL", "CEILING", "CENTURY", "CHAIN", "CHAR", "CHARACTER", "CHARACTERISTICS", "CHARACTERS",
- "CHARACTER_LENGTH", "CHARACTER_SET_CATALOG", "CHARACTER_SET_NAME", "CHARACTER_SET_SCHEMA", "CHAR_LENGTH",
- "CHECK", "CLASSIFIER", "CLASS_ORIGIN", "CLOB", "CLOSE", "COALESCE", "COBOL", "COLLATE", "COLLATION",
- "COLLATION_CATALOG", "COLLATION_NAME", "COLLATION_SCHEMA", "COLLECT", "COLUMN", "COLUMN_NAME",
- "COMMAND_FUNCTION", "COMMAND_FUNCTION_CODE", "COMMIT", "COMMITTED", "CONDITION", "CONDITION_NUMBER",
- "CONNECT", "CONNECTION", "CONNECTION_NAME", "CONSTRAINT", "CONSTRAINTS", "CONSTRAINT_CATALOG",
- "CONSTRAINT_NAME", "CONSTRAINT_SCHEMA", "CONSTRUCTOR", "CONTAINS", "CONTINUE", "CONVERT", "CORR",
- "CORRESPONDING", "COUNT", "COVAR_POP", "COVAR_SAMP", "CREATE", "CROSS", "CUBE", "CUME_DIST", "CURRENT",
- "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_DEFAULT_TRANSFORM_GROUP", "CURRENT_PATH", "CURRENT_ROLE",
- "CURRENT_ROW", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_TRANSFORM_GROUP_FOR_TYPE",
- "CURRENT_USER", "CURSOR", "CURSOR_NAME", "CYCLE", "DATA", "DATABASE", "DATE", "DATETIME_INTERVAL_CODE",
- "DATETIME_INTERVAL_PRECISION", "DAY", "DEALLOCATE", "DEC", "DECADE", "DECIMAL", "DECLARE", "DEFAULT",
- "DEFAULTS", "DEFERRABLE", "DEFERRED", "DEFINE", "DEFINED", "DEFINER", "DEGREE", "DELETE", "DENSE_RANK",
- "DEPTH", "DEREF", "DERIVED", "DESC", "DESCRIBE", "DESCRIPTION", "DESCRIPTOR", "DETERMINISTIC",
- "DIAGNOSTICS", "DISALLOW", "DISCONNECT", "DISPATCH", "DISTINCT", "DOMAIN", "DOUBLE", "DOW", "DOY", "DROP",
- "DYNAMIC", "DYNAMIC_FUNCTION", "DYNAMIC_FUNCTION_CODE", "EACH", "ELEMENT", "ELSE", "EMPTY", "END",
- "END-EXEC", "END_FRAME", "END_PARTITION", "EPOCH", "EQUALS", "ESCAPE", "EVERY", "EXCEPT", "EXCEPTION",
- "EXCLUDE", "EXCLUDING", "EXEC", "EXECUTE", "EXISTS", "EXP", "EXPLAIN", "EXTEND", "EXTERNAL", "EXTRACT",
- "FALSE", "FETCH", "FILTER", "FINAL", "FIRST", "FIRST_VALUE", "FLOAT", "FLOOR", "FOLLOWING", "FOR",
- "FOREIGN", "FORTRAN", "FOUND", "FRAC_SECOND", "FRAME_ROW", "FREE", "FROM", "FULL", "FUNCTION", "FUSION",
- "G", "GENERAL", "GENERATED", "GET", "GLOBAL", "GO", "GOTO", "GRANT", "GRANTED", "GROUP", "GROUPING",
- "GROUPS", "HAVING", "HIERARCHY", "HOLD", "HOUR", "IDENTITY", "IMMEDIATE", "IMMEDIATELY", "IMPLEMENTATION",
- "IMPORT", "IN", "INCLUDING", "INCREMENT", "INDICATOR", "INITIAL", "INITIALLY", "INNER", "INOUT", "INPUT",
- "INSENSITIVE", "INSERT", "INSTANCE", "INSTANTIABLE", "INT", "INTEGER", "INTERSECT", "INTERSECTION",
- "INTERVAL", "INTO", "INVOKER", "IS", "ISOLATION", "JAVA", "JOIN", "JSON", "K", "KEY", "KEY_MEMBER",
- "KEY_TYPE", "LABEL", "LAG", "LANGUAGE", "LARGE", "LAST", "LAST_VALUE", "LATERAL", "LEAD", "LEADING", "LEFT",
- "LENGTH", "LEVEL", "LIBRARY", "LIKE", "LIKE_REGEX", "LIMIT", "LN", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP",
- "LOCATOR", "LOWER", "M", "MAP", "MATCH", "MATCHED", "MATCHES", "MATCH_NUMBER", "MATCH_RECOGNIZE", "MAX",
- "MAXVALUE", "MEASURES", "MEMBER", "MERGE", "MESSAGE_LENGTH", "MESSAGE_OCTET_LENGTH", "MESSAGE_TEXT",
- "METHOD", "MICROSECOND", "MILLENNIUM", "MIN", "MINUS", "MINUTE", "MINVALUE", "MOD", "MODIFIES", "MODULE",
- "MONTH", "MORE", "MULTISET", "MUMPS", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NCLOB", "NESTING",
- "NEW", "NEXT", "NO", "NONE", "NORMALIZE", "NORMALIZED", "NOT", "NTH_VALUE", "NTILE", "NULL", "NULLABLE",
- "NULLIF", "NULLS", "NUMBER", "NUMERIC", "OBJECT", "OCCURRENCES_REGEX", "OCTETS", "OCTET_LENGTH", "OF",
- "OFFSET", "OLD", "OMIT", "ON", "ONE", "ONLY", "OPEN", "OPTION", "OPTIONS", "OR", "ORDER", "ORDERING",
- "ORDINALITY", "OTHERS", "OUT", "OUTER", "OUTPUT", "OVER", "OVERLAPS", "OVERLAY", "OVERRIDING", "PAD",
- "PARAMETER", "PARAMETER_MODE", "PARAMETER_NAME", "PARAMETER_ORDINAL_POSITION", "PARAMETER_SPECIFIC_CATALOG",
- "PARAMETER_SPECIFIC_NAME", "PARAMETER_SPECIFIC_SCHEMA", "PARTIAL", "PARTITION", "PASCAL", "PASSTHROUGH",
- "PAST", "PATH", "PATTERN", "PER", "PERCENT", "PERCENTILE_CONT", "PERCENTILE_DISC", "PERCENT_RANK", "PERIOD",
- "PERMUTE", "PLACING", "PLAN", "PLI", "PORTION", "POSITION", "POSITION_REGEX", "POWER", "PRECEDES",
- "PRECEDING", "PRECISION", "PREPARE", "PRESERVE", "PREV", "PRIMARY", "PRIOR", "PRIVILEGES", "PROCEDURE",
- "PUBLIC", "QUARTER", "RANGE", "RANK", "READ", "READS", "REAL", "RECURSIVE", "REF", "REFERENCES",
- "REFERENCING", "REGR_AVGX", "REGR_AVGY", "REGR_COUNT", "REGR_INTERCEPT", "REGR_R2", "REGR_SLOPE",
- "REGR_SXX", "REGR_SXY", "REGR_SYY", "RELATIVE", "RELEASE", "REPEATABLE", "REPLACE", "RESET", "RESTART",
- "RESTRICT", "RESULT", "RETURN", "RETURNED_CARDINALITY", "RETURNED_LENGTH", "RETURNED_OCTET_LENGTH",
- "RETURNED_SQLSTATE", "RETURNS", "REVOKE", "RIGHT", "ROLE", "ROLLBACK", "ROLLUP", "ROUTINE",
- "ROUTINE_CATALOG", "ROUTINE_NAME", "ROUTINE_SCHEMA", "ROW", "ROWS", "ROW_COUNT", "ROW_NUMBER", "RUNNING",
- "SAVEPOINT", "SCALE", "SCHEMA", "SCHEMA_NAME", "SCOPE", "SCOPE_CATALOGS", "SCOPE_NAME", "SCOPE_SCHEMA",
- "SCROLL", "SEARCH", "SECOND", "SECTION", "SECURITY", "SEEK", "SELECT", "SELF", "SENSITIVE", "SEQUENCE",
- "SERIALIZABLE", "SERVER", "SERVER_NAME", "SESSION", "SESSION_USER", "SET", "SETS", "SHOW", "SIMILAR",
- "SIMPLE", "SIZE", "SKIP", "SMALLINT", "SOME", "SOURCE", "SPACE", "SPECIFIC", "SPECIFICTYPE",
- "SPECIFIC_NAME", "SQL", "SQLEXCEPTION", "SQLSTATE", "SQLWARNING", "SQL_BIGINT", "SQL_BINARY", "SQL_BIT",
- "SQL_BLOB", "SQL_BOOLEAN", "SQL_CHAR", "SQL_CLOB", "SQL_DATE", "SQL_DECIMAL", "SQL_DOUBLE", "SQL_FLOAT",
- "SQL_INTEGER", "SQL_INTERVAL_DAY", "SQL_INTERVAL_DAY_TO_HOUR", "SQL_INTERVAL_DAY_TO_MINUTE",
- "SQL_INTERVAL_DAY_TO_SECOND", "SQL_INTERVAL_HOUR", "SQL_INTERVAL_HOUR_TO_MINUTE",
- "SQL_INTERVAL_HOUR_TO_SECOND", "SQL_INTERVAL_MINUTE", "SQL_INTERVAL_MINUTE_TO_SECOND", "SQL_INTERVAL_MONTH",
- "SQL_INTERVAL_SECOND", "SQL_INTERVAL_YEAR", "SQL_INTERVAL_YEAR_TO_MONTH", "SQL_LONGVARBINARY",
- "SQL_LONGVARCHAR", "SQL_LONGVARNCHAR", "SQL_NCHAR", "SQL_NCLOB", "SQL_NUMERIC", "SQL_NVARCHAR", "SQL_REAL",
- "SQL_SMALLINT", "SQL_TIME", "SQL_TIMESTAMP", "SQL_TINYINT", "SQL_TSI_DAY", "SQL_TSI_FRAC_SECOND",
- "SQL_TSI_HOUR", "SQL_TSI_MICROSECOND", "SQL_TSI_MINUTE", "SQL_TSI_MONTH", "SQL_TSI_QUARTER",
- "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SQL_VARBINARY", "SQL_VARCHAR", "SQRT", "START", "STATE",
- "STATEMENT", "STATIC", "STDDEV_POP", "STDDEV_SAMP", "STREAM", "STRUCTURE", "STYLE", "SUBCLASS_ORIGIN",
- "SUBMULTISET", "SUBSET", "SUBSTITUTE", "SUBSTRING", "SUBSTRING_REGEX", "SUCCEEDS", "SUM", "SYMMETRIC",
- "SYSTEM", "SYSTEM_TIME", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TABLE_NAME", "TEMPORARY", "THEN", "TIES",
- "TIME", "TIMESTAMP", "TIMESTAMPADD", "TIMESTAMPDIFF", "TIMEZONE_HOUR", "TIMEZONE_MINUTE", "TINYINT", "TO",
- "TOP_LEVEL_COUNT", "TRAILING", "TRANSACTION", "TRANSACTIONS_ACTIVE", "TRANSACTIONS_COMMITTED",
- "TRANSACTIONS_ROLLED_BACK", "TRANSFORM", "TRANSFORMS", "TRANSLATE", "TRANSLATE_REGEX", "TRANSLATION",
- "TREAT", "TRIGGER", "TRIGGER_CATALOG", "TRIGGER_NAME", "TRIGGER_SCHEMA", "TRIM", "TRIM_ARRAY", "TRUE",
- "TRUNCATE", "TYPE", "UESCAPE", "UNBOUNDED", "UNCOMMITTED", "UNDER", "UNION", "UNIQUE", "UNKNOWN", "UNNAMED",
- "UNNEST", "UPDATE", "UPPER", "UPSERT", "USAGE", "USER", "USER_DEFINED_TYPE_CATALOG",
- "USER_DEFINED_TYPE_CODE", "USER_DEFINED_TYPE_NAME", "USER_DEFINED_TYPE_SCHEMA", "USING", "VALUE", "VALUES",
- "VALUE_OF", "VARBINARY", "VARCHAR", "VARYING", "VAR_POP", "VAR_SAMP", "VERSION", "VERSIONING", "VIEW",
- "WEEK", "WHEN", "WHENEVER", "WHERE", "WIDTH_BUCKET", "WINDOW", "WITH", "WITHIN", "WITHOUT", "WORK",
- "WRAPPER", "WRITE", "XML", "YEAR", "ZONE");
-
- public static String replaceString(String originString, String fromString, String toString) {
- return originString.replace(fromString, toString);
- }
-
- public static String extractReplace(String originString) {
- Matcher extractMatcher = EXTRACT_PATTERN.matcher(originString);
- String replacedString = originString;
- Map<Integer, Integer> parenthesesPairs = null;
-
- while (extractMatcher.find()) {
- if (parenthesesPairs == null) {
- parenthesesPairs = findParenthesesPairs(originString);
- }
-
- String functionStr = extractMatcher.group(2);
- int startIdx = extractMatcher.end(3);
- int endIdx = parenthesesPairs.get(extractMatcher.start(1));
- String extractInner = originString.substring(startIdx, endIdx);
- int originStart = extractMatcher.start(0);
- int originEnd = endIdx + 1;
-
- replacedString = replaceString(replacedString, originString.substring(originStart, originEnd),
- functionStr + "(" + extractInner + ")");
- }
-
- return replacedString;
- }
-
- public static String castReplace(String originString) {
- Matcher castMatcher = CAST_PATTERN.matcher(originString);
- String replacedString = originString;
-
- while (castMatcher.find()) {
- String castStr = castMatcher.group();
- String type = castMatcher.group(2);
- String supportedType = "";
- switch (type.toUpperCase()) {
- case "INTEGER":
- supportedType = "int";
- break;
- case "SHORT":
- supportedType = "smallint";
- break;
- case "LONG":
- supportedType = "bigint";
- break;
- default:
- supportedType = type;
- }
-
- if (!supportedType.equals(type)) {
- String replacedCastStr = castStr.replace(type, supportedType);
- replacedString = replaceString(replacedString, castStr, replacedCastStr);
- }
- }
-
- return replacedString;
- }
-
- public static String subqueryReplace(String originString) {
- Matcher subqueryMatcher = FROM_PATTERN.matcher(originString);
- String replacedString = originString;
- Map<Integer, Integer> parenthesesPairs = null;
-
- while (subqueryMatcher.find()) {
- if (parenthesesPairs == null) {
- parenthesesPairs = findParenthesesPairs(originString);
- }
-
- int startIdx = subqueryMatcher.start(1);
- int endIdx = parenthesesPairs.get(startIdx) + 1;
-
- Matcher aliasMatcher = ALIAS_PATTERN.matcher(originString.substring(endIdx));
- if (aliasMatcher.find()) {
- String aliasCandidate = aliasMatcher.group(1);
-
- if (aliasCandidate != null && !sqlKeyWordsExceptAS.contains(aliasCandidate.toUpperCase())) {
- continue;
- }
-
- replacedString = replaceString(replacedString, originString.substring(startIdx, endIdx),
- originString.substring(startIdx, endIdx) + " as alias");
- }
- }
-
- return replacedString;
- }
-
- public static String timestampaddReplace(String originString) {
- Matcher timestampaddMatcher = TIMESTAMPADD_PATTERN.matcher(originString);
- String replacedString = originString;
-
- while (timestampaddMatcher.find()) {
- String interval = timestampaddMatcher.group(1);
- String timestampaddStr = replaceString(timestampaddMatcher.group(), interval, "'" + interval + "'");
- replacedString = replaceString(replacedString, timestampaddMatcher.group(), timestampaddStr);
- }
-
- return replacedString;
- }
-
- public static String concatReplace(String originString) {
- Matcher concatMatcher = CONCAT_PATTERN.matcher(originString);
- String replacedString = originString;
-
- while (concatMatcher.find()) {
- String leftString = concatMatcher.group(1);
- String rightString = concatMatcher.group(2);
- replacedString = replaceString(replacedString, leftString + "||" + rightString,
- "concat(" + leftString + "," + rightString + ")");
- }
-
- return replacedString;
- }
-
- public static String doConvert(String originStr) {
- // Step1.Replace " with `
- String convertedSql = replaceString(originStr, "\"", "`");
-
- // Step2.Replace extract functions
- convertedSql = extractReplace(convertedSql);
-
- // Step3.Replace cast type string
- convertedSql = castReplace(convertedSql);
-
- // Step4.Replace sub query
- convertedSql = subqueryReplace(convertedSql);
-
- // Step5.Replace char_length with length
- convertedSql = replaceString(convertedSql, "char_length", "length");
-
- // Step6.Replace "||" with concat
- convertedSql = concatReplace(convertedSql);
-
- // Step7.Add quote for interval in timestampadd
- convertedSql = timestampaddReplace(convertedSql);
-
- return convertedSql;
- }
-
- private static Map<Integer, Integer> findParenthesesPairs(String sql) {
- Map<Integer, Integer> result = new HashMap<>();
- if (sql.length() > 1) {
- Stack<Integer> lStack = new Stack<>();
- boolean inStrVal = false;
- for (int i = 0; i < sql.length(); i++) {
- switch (sql.charAt(i)) {
- case '(':
- if (!inStrVal) {
- lStack.push(i);
- }
- break;
- case ')':
- if (!inStrVal && !lStack.empty()) {
- result.put(lStack.pop(), i);
- }
- break;
- default:
- break;
- }
- }
- }
- return result;
- }
-
- @Override
- public String convert(String originSql) {
- return doConvert(originSql);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
new file mode 100644
index 0000000..8d89294
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.source.adhocquery;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Stack;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ImmutableSet;
+
+//TODO: Some workaround ways to make sql readable by hive parser, should replaced it with a more well-designed way
+public class HivePushDownConverter implements IPushDownConverter {
+
+ @SuppressWarnings("unused")
+ private static final Logger logger = LoggerFactory.getLogger(HivePushDownConverter.class);
+
+ private static final Pattern EXTRACT_PATTERN = Pattern.compile("extract\\s*(\\()\\s*(.*?)\\s*from(\\s+)",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\()\\s*select\\s",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern ALIAS_PATTERN = Pattern.compile("\\s+([`'_a-z0-9A-Z]+)", Pattern.CASE_INSENSITIVE);
+ private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\((.*?) (?i)AS\\s*(.*?)\\s*\\)",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern CONCAT_PATTERN = Pattern.compile("(['_a-z0-9A-Z]+)\\|\\|(['_a-z0-9A-Z]+)",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern TIMESTAMPADD_PATTERN = Pattern.compile("timestampadd\\s*\\(\\s*(.*?)\\s*,",
+ Pattern.CASE_INSENSITIVE);
+ private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("A", "ABS", "ABSOLUTE", "ACTION",
+ "ADA", "ADD", "ADMIN", "AFTER", "ALL", "ALLOCATE", "ALLOW", "ALTER", "ALWAYS", "AND", "ANY", "APPLY", "ARE",
+ "ARRAY", "ARRAY_MAX_CARDINALITY", "ASC", "ASENSITIVE", "ASSERTION", "ASSIGNMENT", "ASYMMETRIC", "AT",
+ "ATOMIC", "ATTRIBUTE", "ATTRIBUTES", "AUTHORIZATION", "AVG", "BEFORE", "BEGIN", "BEGIN_FRAME",
+ "BEGIN_PARTITION", "BERNOULLI", "BETWEEN", "BIGINT", "BINARY", "BIT", "BLOB", "BOOLEAN", "BOTH", "BREADTH",
+ "BY", "C", "CALL", "CALLED", "CARDINALITY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG",
+ "CATALOG_NAME", "CEIL", "CEILING", "CENTURY", "CHAIN", "CHAR", "CHARACTER", "CHARACTERISTICS", "CHARACTERS",
+ "CHARACTER_LENGTH", "CHARACTER_SET_CATALOG", "CHARACTER_SET_NAME", "CHARACTER_SET_SCHEMA", "CHAR_LENGTH",
+ "CHECK", "CLASSIFIER", "CLASS_ORIGIN", "CLOB", "CLOSE", "COALESCE", "COBOL", "COLLATE", "COLLATION",
+ "COLLATION_CATALOG", "COLLATION_NAME", "COLLATION_SCHEMA", "COLLECT", "COLUMN", "COLUMN_NAME",
+ "COMMAND_FUNCTION", "COMMAND_FUNCTION_CODE", "COMMIT", "COMMITTED", "CONDITION", "CONDITION_NUMBER",
+ "CONNECT", "CONNECTION", "CONNECTION_NAME", "CONSTRAINT", "CONSTRAINTS", "CONSTRAINT_CATALOG",
+ "CONSTRAINT_NAME", "CONSTRAINT_SCHEMA", "CONSTRUCTOR", "CONTAINS", "CONTINUE", "CONVERT", "CORR",
+ "CORRESPONDING", "COUNT", "COVAR_POP", "COVAR_SAMP", "CREATE", "CROSS", "CUBE", "CUME_DIST", "CURRENT",
+ "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_DEFAULT_TRANSFORM_GROUP", "CURRENT_PATH", "CURRENT_ROLE",
+ "CURRENT_ROW", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_TRANSFORM_GROUP_FOR_TYPE",
+ "CURRENT_USER", "CURSOR", "CURSOR_NAME", "CYCLE", "DATA", "DATABASE", "DATE", "DATETIME_INTERVAL_CODE",
+ "DATETIME_INTERVAL_PRECISION", "DAY", "DEALLOCATE", "DEC", "DECADE", "DECIMAL", "DECLARE", "DEFAULT",
+ "DEFAULTS", "DEFERRABLE", "DEFERRED", "DEFINE", "DEFINED", "DEFINER", "DEGREE", "DELETE", "DENSE_RANK",
+ "DEPTH", "DEREF", "DERIVED", "DESC", "DESCRIBE", "DESCRIPTION", "DESCRIPTOR", "DETERMINISTIC",
+ "DIAGNOSTICS", "DISALLOW", "DISCONNECT", "DISPATCH", "DISTINCT", "DOMAIN", "DOUBLE", "DOW", "DOY", "DROP",
+ "DYNAMIC", "DYNAMIC_FUNCTION", "DYNAMIC_FUNCTION_CODE", "EACH", "ELEMENT", "ELSE", "EMPTY", "END",
+ "END-EXEC", "END_FRAME", "END_PARTITION", "EPOCH", "EQUALS", "ESCAPE", "EVERY", "EXCEPT", "EXCEPTION",
+ "EXCLUDE", "EXCLUDING", "EXEC", "EXECUTE", "EXISTS", "EXP", "EXPLAIN", "EXTEND", "EXTERNAL", "EXTRACT",
+ "FALSE", "FETCH", "FILTER", "FINAL", "FIRST", "FIRST_VALUE", "FLOAT", "FLOOR", "FOLLOWING", "FOR",
+ "FOREIGN", "FORTRAN", "FOUND", "FRAC_SECOND", "FRAME_ROW", "FREE", "FROM", "FULL", "FUNCTION", "FUSION",
+ "G", "GENERAL", "GENERATED", "GET", "GLOBAL", "GO", "GOTO", "GRANT", "GRANTED", "GROUP", "GROUPING",
+ "GROUPS", "HAVING", "HIERARCHY", "HOLD", "HOUR", "IDENTITY", "IMMEDIATE", "IMMEDIATELY", "IMPLEMENTATION",
+ "IMPORT", "IN", "INCLUDING", "INCREMENT", "INDICATOR", "INITIAL", "INITIALLY", "INNER", "INOUT", "INPUT",
+ "INSENSITIVE", "INSERT", "INSTANCE", "INSTANTIABLE", "INT", "INTEGER", "INTERSECT", "INTERSECTION",
+ "INTERVAL", "INTO", "INVOKER", "IS", "ISOLATION", "JAVA", "JOIN", "JSON", "K", "KEY", "KEY_MEMBER",
+ "KEY_TYPE", "LABEL", "LAG", "LANGUAGE", "LARGE", "LAST", "LAST_VALUE", "LATERAL", "LEAD", "LEADING", "LEFT",
+ "LENGTH", "LEVEL", "LIBRARY", "LIKE", "LIKE_REGEX", "LIMIT", "LN", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP",
+ "LOCATOR", "LOWER", "M", "MAP", "MATCH", "MATCHED", "MATCHES", "MATCH_NUMBER", "MATCH_RECOGNIZE", "MAX",
+ "MAXVALUE", "MEASURES", "MEMBER", "MERGE", "MESSAGE_LENGTH", "MESSAGE_OCTET_LENGTH", "MESSAGE_TEXT",
+ "METHOD", "MICROSECOND", "MILLENNIUM", "MIN", "MINUS", "MINUTE", "MINVALUE", "MOD", "MODIFIES", "MODULE",
+ "MONTH", "MORE", "MULTISET", "MUMPS", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NCLOB", "NESTING",
+ "NEW", "NEXT", "NO", "NONE", "NORMALIZE", "NORMALIZED", "NOT", "NTH_VALUE", "NTILE", "NULL", "NULLABLE",
+ "NULLIF", "NULLS", "NUMBER", "NUMERIC", "OBJECT", "OCCURRENCES_REGEX", "OCTETS", "OCTET_LENGTH", "OF",
+ "OFFSET", "OLD", "OMIT", "ON", "ONE", "ONLY", "OPEN", "OPTION", "OPTIONS", "OR", "ORDER", "ORDERING",
+ "ORDINALITY", "OTHERS", "OUT", "OUTER", "OUTPUT", "OVER", "OVERLAPS", "OVERLAY", "OVERRIDING", "PAD",
+ "PARAMETER", "PARAMETER_MODE", "PARAMETER_NAME", "PARAMETER_ORDINAL_POSITION", "PARAMETER_SPECIFIC_CATALOG",
+ "PARAMETER_SPECIFIC_NAME", "PARAMETER_SPECIFIC_SCHEMA", "PARTIAL", "PARTITION", "PASCAL", "PASSTHROUGH",
+ "PAST", "PATH", "PATTERN", "PER", "PERCENT", "PERCENTILE_CONT", "PERCENTILE_DISC", "PERCENT_RANK", "PERIOD",
+ "PERMUTE", "PLACING", "PLAN", "PLI", "PORTION", "POSITION", "POSITION_REGEX", "POWER", "PRECEDES",
+ "PRECEDING", "PRECISION", "PREPARE", "PRESERVE", "PREV", "PRIMARY", "PRIOR", "PRIVILEGES", "PROCEDURE",
+ "PUBLIC", "QUARTER", "RANGE", "RANK", "READ", "READS", "REAL", "RECURSIVE", "REF", "REFERENCES",
+ "REFERENCING", "REGR_AVGX", "REGR_AVGY", "REGR_COUNT", "REGR_INTERCEPT", "REGR_R2", "REGR_SLOPE",
+ "REGR_SXX", "REGR_SXY", "REGR_SYY", "RELATIVE", "RELEASE", "REPEATABLE", "REPLACE", "RESET", "RESTART",
+ "RESTRICT", "RESULT", "RETURN", "RETURNED_CARDINALITY", "RETURNED_LENGTH", "RETURNED_OCTET_LENGTH",
+ "RETURNED_SQLSTATE", "RETURNS", "REVOKE", "RIGHT", "ROLE", "ROLLBACK", "ROLLUP", "ROUTINE",
+ "ROUTINE_CATALOG", "ROUTINE_NAME", "ROUTINE_SCHEMA", "ROW", "ROWS", "ROW_COUNT", "ROW_NUMBER", "RUNNING",
+ "SAVEPOINT", "SCALE", "SCHEMA", "SCHEMA_NAME", "SCOPE", "SCOPE_CATALOGS", "SCOPE_NAME", "SCOPE_SCHEMA",
+ "SCROLL", "SEARCH", "SECOND", "SECTION", "SECURITY", "SEEK", "SELECT", "SELF", "SENSITIVE", "SEQUENCE",
+ "SERIALIZABLE", "SERVER", "SERVER_NAME", "SESSION", "SESSION_USER", "SET", "SETS", "SHOW", "SIMILAR",
+ "SIMPLE", "SIZE", "SKIP", "SMALLINT", "SOME", "SOURCE", "SPACE", "SPECIFIC", "SPECIFICTYPE",
+ "SPECIFIC_NAME", "SQL", "SQLEXCEPTION", "SQLSTATE", "SQLWARNING", "SQL_BIGINT", "SQL_BINARY", "SQL_BIT",
+ "SQL_BLOB", "SQL_BOOLEAN", "SQL_CHAR", "SQL_CLOB", "SQL_DATE", "SQL_DECIMAL", "SQL_DOUBLE", "SQL_FLOAT",
+ "SQL_INTEGER", "SQL_INTERVAL_DAY", "SQL_INTERVAL_DAY_TO_HOUR", "SQL_INTERVAL_DAY_TO_MINUTE",
+ "SQL_INTERVAL_DAY_TO_SECOND", "SQL_INTERVAL_HOUR", "SQL_INTERVAL_HOUR_TO_MINUTE",
+ "SQL_INTERVAL_HOUR_TO_SECOND", "SQL_INTERVAL_MINUTE", "SQL_INTERVAL_MINUTE_TO_SECOND", "SQL_INTERVAL_MONTH",
+ "SQL_INTERVAL_SECOND", "SQL_INTERVAL_YEAR", "SQL_INTERVAL_YEAR_TO_MONTH", "SQL_LONGVARBINARY",
+ "SQL_LONGVARCHAR", "SQL_LONGVARNCHAR", "SQL_NCHAR", "SQL_NCLOB", "SQL_NUMERIC", "SQL_NVARCHAR", "SQL_REAL",
+ "SQL_SMALLINT", "SQL_TIME", "SQL_TIMESTAMP", "SQL_TINYINT", "SQL_TSI_DAY", "SQL_TSI_FRAC_SECOND",
+ "SQL_TSI_HOUR", "SQL_TSI_MICROSECOND", "SQL_TSI_MINUTE", "SQL_TSI_MONTH", "SQL_TSI_QUARTER",
+ "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SQL_VARBINARY", "SQL_VARCHAR", "SQRT", "START", "STATE",
+ "STATEMENT", "STATIC", "STDDEV_POP", "STDDEV_SAMP", "STREAM", "STRUCTURE", "STYLE", "SUBCLASS_ORIGIN",
+ "SUBMULTISET", "SUBSET", "SUBSTITUTE", "SUBSTRING", "SUBSTRING_REGEX", "SUCCEEDS", "SUM", "SYMMETRIC",
+ "SYSTEM", "SYSTEM_TIME", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TABLE_NAME", "TEMPORARY", "THEN", "TIES",
+ "TIME", "TIMESTAMP", "TIMESTAMPADD", "TIMESTAMPDIFF", "TIMEZONE_HOUR", "TIMEZONE_MINUTE", "TINYINT", "TO",
+ "TOP_LEVEL_COUNT", "TRAILING", "TRANSACTION", "TRANSACTIONS_ACTIVE", "TRANSACTIONS_COMMITTED",
+ "TRANSACTIONS_ROLLED_BACK", "TRANSFORM", "TRANSFORMS", "TRANSLATE", "TRANSLATE_REGEX", "TRANSLATION",
+ "TREAT", "TRIGGER", "TRIGGER_CATALOG", "TRIGGER_NAME", "TRIGGER_SCHEMA", "TRIM", "TRIM_ARRAY", "TRUE",
+ "TRUNCATE", "TYPE", "UESCAPE", "UNBOUNDED", "UNCOMMITTED", "UNDER", "UNION", "UNIQUE", "UNKNOWN", "UNNAMED",
+ "UNNEST", "UPDATE", "UPPER", "UPSERT", "USAGE", "USER", "USER_DEFINED_TYPE_CATALOG",
+ "USER_DEFINED_TYPE_CODE", "USER_DEFINED_TYPE_NAME", "USER_DEFINED_TYPE_SCHEMA", "USING", "VALUE", "VALUES",
+ "VALUE_OF", "VARBINARY", "VARCHAR", "VARYING", "VAR_POP", "VAR_SAMP", "VERSION", "VERSIONING", "VIEW",
+ "WEEK", "WHEN", "WHENEVER", "WHERE", "WIDTH_BUCKET", "WINDOW", "WITH", "WITHIN", "WITHOUT", "WORK",
+ "WRAPPER", "WRITE", "XML", "YEAR", "ZONE");
+
+ public static String replaceString(String originString, String fromString, String toString) {
+ return originString.replace(fromString, toString);
+ }
+
+ public static String extractReplace(String originString) {
+ Matcher extractMatcher = EXTRACT_PATTERN.matcher(originString);
+ String replacedString = originString;
+ Map<Integer, Integer> parenthesesPairs = null;
+
+ while (extractMatcher.find()) {
+ if (parenthesesPairs == null) {
+ parenthesesPairs = findParenthesesPairs(originString);
+ }
+
+ String functionStr = extractMatcher.group(2);
+ int startIdx = extractMatcher.end(3);
+ int endIdx = parenthesesPairs.get(extractMatcher.start(1));
+ String extractInner = originString.substring(startIdx, endIdx);
+ int originStart = extractMatcher.start(0);
+ int originEnd = endIdx + 1;
+
+ replacedString = replaceString(replacedString, originString.substring(originStart, originEnd),
+ functionStr + "(" + extractInner + ")");
+ }
+
+ return replacedString;
+ }
+
+ public static String castReplace(String originString) {
+ Matcher castMatcher = CAST_PATTERN.matcher(originString);
+ String replacedString = originString;
+
+ while (castMatcher.find()) {
+ String castStr = castMatcher.group();
+ String type = castMatcher.group(2);
+ String supportedType = "";
+ switch (type.toUpperCase()) {
+ case "INTEGER":
+ supportedType = "int";
+ break;
+ case "SHORT":
+ supportedType = "smallint";
+ break;
+ case "LONG":
+ supportedType = "bigint";
+ break;
+ default:
+ supportedType = type;
+ }
+
+ if (!supportedType.equals(type)) {
+ String replacedCastStr = castStr.replace(type, supportedType);
+ replacedString = replaceString(replacedString, castStr, replacedCastStr);
+ }
+ }
+
+ return replacedString;
+ }
+
+ public static String subqueryReplace(String originString) {
+ Matcher subqueryMatcher = FROM_PATTERN.matcher(originString);
+ String replacedString = originString;
+ Map<Integer, Integer> parenthesesPairs = null;
+
+ while (subqueryMatcher.find()) {
+ if (parenthesesPairs == null) {
+ parenthesesPairs = findParenthesesPairs(originString);
+ }
+
+ int startIdx = subqueryMatcher.start(1);
+ int endIdx = parenthesesPairs.get(startIdx) + 1;
+
+ Matcher aliasMatcher = ALIAS_PATTERN.matcher(originString.substring(endIdx));
+ if (aliasMatcher.find()) {
+ String aliasCandidate = aliasMatcher.group(1);
+
+ if (aliasCandidate != null && !sqlKeyWordsExceptAS.contains(aliasCandidate.toUpperCase())) {
+ continue;
+ }
+
+ replacedString = replaceString(replacedString, originString.substring(startIdx, endIdx),
+ originString.substring(startIdx, endIdx) + " as alias");
+ }
+ }
+
+ return replacedString;
+ }
+
+ public static String timestampaddReplace(String originString) {
+ Matcher timestampaddMatcher = TIMESTAMPADD_PATTERN.matcher(originString);
+ String replacedString = originString;
+
+ while (timestampaddMatcher.find()) {
+ String interval = timestampaddMatcher.group(1);
+ String timestampaddStr = replaceString(timestampaddMatcher.group(), interval, "'" + interval + "'");
+ replacedString = replaceString(replacedString, timestampaddMatcher.group(), timestampaddStr);
+ }
+
+ return replacedString;
+ }
+
+ public static String concatReplace(String originString) {
+ Matcher concatMatcher = CONCAT_PATTERN.matcher(originString);
+ String replacedString = originString;
+
+ while (concatMatcher.find()) {
+ String leftString = concatMatcher.group(1);
+ String rightString = concatMatcher.group(2);
+ replacedString = replaceString(replacedString, leftString + "||" + rightString,
+ "concat(" + leftString + "," + rightString + ")");
+ }
+
+ return replacedString;
+ }
+
+ public static String doConvert(String originStr) {
+ // Step1.Replace " with `
+ String convertedSql = replaceString(originStr, "\"", "`");
+
+ // Step2.Replace extract functions
+ convertedSql = extractReplace(convertedSql);
+
+ // Step3.Replace cast type string
+ convertedSql = castReplace(convertedSql);
+
+ // Step4.Replace sub query
+ convertedSql = subqueryReplace(convertedSql);
+
+ // Step5.Replace char_length with length
+ convertedSql = replaceString(convertedSql, "char_length", "length");
+
+ // Step6.Replace "||" with concat
+ convertedSql = concatReplace(convertedSql);
+
+ // Step7.Add quote for interval in timestampadd
+ convertedSql = timestampaddReplace(convertedSql);
+
+ return convertedSql;
+ }
+
+ private static Map<Integer, Integer> findParenthesesPairs(String sql) {
+ Map<Integer, Integer> result = new HashMap<>();
+ if (sql.length() > 1) {
+ Stack<Integer> lStack = new Stack<>();
+ boolean inStrVal = false;
+ for (int i = 0; i < sql.length(); i++) {
+ switch (sql.charAt(i)) {
+ case '(':
+ if (!inStrVal) {
+ lStack.push(i);
+ }
+ break;
+ case ')':
+ if (!inStrVal && !lStack.empty()) {
+ result.put(lStack.pop(), i);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public String convert(String originSql) {
+ return doConvert(originSql);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java
deleted file mode 100644
index c4b87f8..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.source.adhocquery;
-
-/**
- * convert the query to satisfy the parser of adhoc query engine
- */
-public interface IAdHocConverter {
- String convert(String originSql);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java
deleted file mode 100644
index 369325c..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.source.adhocquery;
-
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
-
-public interface IAdHocRunner {
-
- void init(KylinConfig config);
-
- /**
- * Run an ad-hoc query in the source database in case Kylin cannot serve using cube.
- *
- * @param query the query statement
- * @param returnRows an empty list to collect returning rows
- * @param returnColumnMeta an empty list to collect metadata of returning columns
- * @throws Exception if running ad-hoc query fails
- */
- void executeQuery(String query, List<List<String>> returnRows, List<SelectedColumnMeta> returnColumnMeta) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java
new file mode 100644
index 0000000..c4e7515
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.source.adhocquery;
+
+/**
+ * convert the query to satisfy the parser of push down query engine
+ */
+public interface IPushDownConverter {
+ String convert(String originSql);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
new file mode 100644
index 0000000..c8d18aa
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.adhocquery;
+
+import java.util.List;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
+
+public interface IPushDownRunner {
+
+ void init(KylinConfig config);
+
+ /**
+ * Run an pushdown query in the source database in case Kylin cannot serve using cube.
+ *
+ * @param query the query statement
+ * @param returnRows an empty list to collect returning rows
+ * @param returnColumnMeta an empty list to collect metadata of returning columns
+ * @throws Exception if running pushdown query fails
+ */
+ void executeQuery(String query, List<List<String>> returnRows, List<SelectedColumnMeta> returnColumnMeta) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java
deleted file mode 100644
index cfb0f32..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.adhocquery;
-
-import org.junit.Test;
-
-import junit.framework.TestCase;
-
-public class HiveAdhocConverterTest extends TestCase {
- @Test
- public void testStringReplace() {
- String originString = "select count(*) as cnt from test_kylin_fact where char_length(lstg_format_name) < 10";
- String replacedString = HiveAdhocConverter.replaceString(originString, "char_length", "length");
- assertEquals("select count(*) as cnt from test_kylin_fact where length(lstg_format_name) < 10", replacedString);
- }
-
- @Test
- public void testExtractReplace() {
- String originString = "ignore EXTRACT(YEAR FROM KYLIN_CAL_DT.CAL_DT) ignore";
- String replacedString = HiveAdhocConverter.extractReplace(originString);
- assertEquals("ignore YEAR(KYLIN_CAL_DT.CAL_DT) ignore", replacedString);
- }
-
- @Test
- public void testCastReplace() {
- String originString = "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS INTEGER)) ignore";
- String replacedString = HiveAdhocConverter.castReplace(originString);
- assertEquals("ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS int)) ignore", replacedString);
- }
-
- @Test
- public void testSubqueryReplace1() {
- String originString = "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) group by seller_id,lstg_format_name";
- String replacedString = HiveAdhocConverter.subqueryReplace(originString);
- assertEquals(
- "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) as alias group by seller_id,lstg_format_name",
- replacedString);
- }
-
- @Test
- public void testSubqueryReplace2() {
- String originString = "select count(*) from ( select test_kylin_fact.lstg_format_name from test_kylin_fact where test_kylin_fact.lstg_format_name='FP-GTC' group by test_kylin_fact.lstg_format_name ) t ";
- String replacedString = HiveAdhocConverter.subqueryReplace(originString);
- assertEquals(originString, replacedString);
- }
-
- @Test
- public void testSubqueryReplace3() {
- String originString = "select fact.lstg_format_name from (select * from test_kylin_fact where cal_dt > date'2010-01-01' ) as fact group by fact.lstg_format_name order by CASE WHEN fact.lstg_format_name IS NULL THEN 'sdf' ELSE fact.lstg_format_name END ";
- String replacedString = HiveAdhocConverter.subqueryReplace(originString);
- assertEquals(originString, replacedString);
- }
-
- @Test
- public void testConcatReplace() {
- String originString = "select count(*) as cnt from test_kylin_fact where lstg_format_name||'a'='ABINa'";
- String replacedString = HiveAdhocConverter.concatReplace(originString);
- assertEquals("select count(*) as cnt from test_kylin_fact where concat(lstg_format_name,'a')='ABINa'",
- replacedString);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java
new file mode 100644
index 0000000..42c628a
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.adhocquery;
+
+import org.junit.Test;
+
+import junit.framework.TestCase;
+
+public class HivePushDownConverterTest extends TestCase {
+ @Test
+ public void testStringReplace() {
+ String originString = "select count(*) as cnt from test_kylin_fact where char_length(lstg_format_name) < 10";
+ String replacedString = HivePushDownConverter.replaceString(originString, "char_length", "length");
+ assertEquals("select count(*) as cnt from test_kylin_fact where length(lstg_format_name) < 10", replacedString);
+ }
+
+ @Test
+ public void testExtractReplace() {
+ String originString = "ignore EXTRACT(YEAR FROM KYLIN_CAL_DT.CAL_DT) ignore";
+ String replacedString = HivePushDownConverter.extractReplace(originString);
+ assertEquals("ignore YEAR(KYLIN_CAL_DT.CAL_DT) ignore", replacedString);
+ }
+
+ @Test
+ public void testCastReplace() {
+ String originString = "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS INTEGER)) ignore";
+ String replacedString = HivePushDownConverter.castReplace(originString);
+ assertEquals("ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS int)) ignore", replacedString);
+ }
+
+ @Test
+ public void testSubqueryReplace1() {
+ String originString = "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) group by seller_id,lstg_format_name";
+ String replacedString = HivePushDownConverter.subqueryReplace(originString);
+ assertEquals(
+ "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) as alias group by seller_id,lstg_format_name",
+ replacedString);
+ }
+
+ @Test
+ public void testSubqueryReplace2() {
+ String originString = "select count(*) from ( select test_kylin_fact.lstg_format_name from test_kylin_fact where test_kylin_fact.lstg_format_name='FP-GTC' group by test_kylin_fact.lstg_format_name ) t ";
+ String replacedString = HivePushDownConverter.subqueryReplace(originString);
+ assertEquals(originString, replacedString);
+ }
+
+ @Test
+ public void testSubqueryReplace3() {
+ String originString = "select fact.lstg_format_name from (select * from test_kylin_fact where cal_dt > date'2010-01-01' ) as fact group by fact.lstg_format_name order by CASE WHEN fact.lstg_format_name IS NULL THEN 'sdf' ELSE fact.lstg_format_name END ";
+ String replacedString = HivePushDownConverter.subqueryReplace(originString);
+ assertEquals(originString, replacedString);
+ }
+
+ @Test
+ public void testConcatReplace() {
+ String originString = "select count(*) as cnt from test_kylin_fact where lstg_format_name||'a'='ABINa'";
+ String replacedString = HivePushDownConverter.concatReplace(originString);
+ assertEquals("select count(*) as cnt from test_kylin_fact where concat(lstg_format_name,'a')='ABINa'",
+ replacedString);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 35386de..6a571df 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -186,14 +186,14 @@ kylin.engine.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current
kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current
-### AD-HOC QUERY ###
-#kylin.query.ad-hoc.runner-class-name=org.apache.kylin.query.adhoc.AdHocRunnerJdbcImpl
+### QUERY PUSH DOWN ###
+#kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl
-#kylin.query.ad-hoc.jdbc.url=jdbc:hive2://sandbox:10000/default
-#kylin.query.ad-hoc.jdbc.driver=org.apache.hive.jdbc.HiveDriver
-#kylin.query.ad-hoc.jdbc.username=hive
-#kylin.query.ad-hoc.jdbc.password=
+#kylin.query.pushdown.jdbc.url=jdbc:hive2://sandbox:10000/default
+#kylin.query.pushdown.jdbc.driver=org.apache.hive.jdbc.HiveDriver
+#kylin.query.pushdown.jdbc.username=hive
+#kylin.query.pushdown.jdbc.password=
-#kylin.query.ad-hoc.jdbc.pool-max-total=8
-#kylin.query.ad-hoc.jdbc.pool-max-idle=8
-#kylin.query.ad-hoc.jdbc.pool-min-idle=0
+#kylin.query.pushdown.jdbc.pool-max-total=8
+#kylin.query.pushdown.jdbc.pool-max-idle=8
+#kylin.query.pushdown.jdbc.pool-min-idle=0
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index af05237..3634100 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -51,7 +51,7 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
-import org.apache.kylin.rest.util.AdHocUtil;
+import org.apache.kylin.rest.util.PushDownUtil;
import org.dbunit.DatabaseUnitException;
import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
@@ -263,7 +263,7 @@ public class KylinTestBase {
} catch (SQLException sqlException) {
List<List<String>> results = Lists.newArrayList();
List<SelectedColumnMeta> columnMetas = Lists.newArrayList();
- AdHocUtil.doAdHocQuery(ProjectInstance.DEFAULT_PROJECT_NAME, sql, results, columnMetas, sqlException);
+ PushDownUtil.doPushDownQuery(ProjectInstance.DEFAULT_PROJECT_NAME, sql, results, columnMetas, sqlException);
return results.size();
} finally {
if (resultSet != null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java
deleted file mode 100644
index 852cebf..0000000
--- a/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.adhoc;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.pool.impl.GenericObjectPool;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
-import org.apache.kylin.source.adhocquery.IAdHocRunner;
-
-public class AdHocRunnerJdbcImpl implements IAdHocRunner {
-
- private static org.apache.kylin.query.adhoc.JdbcConnectionPool pool = null;
-
- @Override
- public void init(KylinConfig config) {
- if (pool == null) {
- pool = new JdbcConnectionPool();
- JdbcConnectionFactory factory = new JdbcConnectionFactory(config.getJdbcUrl(), config.getJdbcDriverClass(),
- config.getJdbcUsername(), config.getJdbcPassword());
- GenericObjectPool.Config poolConfig = new GenericObjectPool.Config();
- poolConfig.maxActive = config.getPoolMaxTotal();
- poolConfig.maxIdle = config.getPoolMaxIdle();
- poolConfig.minIdle = config.getPoolMinIdle();
-
- try {
- pool.createPool(factory, poolConfig);
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage(), e);
- }
- }
- }
-
- @Override
- public void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas)
- throws Exception {
- Statement statement = null;
- Connection connection = this.getConnection();
- ResultSet resultSet = null;
-
- try {
- statement = connection.createStatement();
- resultSet = statement.executeQuery(query);
- extractResults(resultSet, results);
- } catch (SQLException sqlException) {
- throw sqlException;
- }
-
- //extract column metadata
- ResultSetMetaData metaData = null;
- int columnCount = 0;
- try {
- metaData = resultSet.getMetaData();
- columnCount = metaData.getColumnCount();
-
- // fill in selected column meta
- for (int i = 1; i <= columnCount; ++i) {
- columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false,
- metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i),
- metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null,
- metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i),
- metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false));
- }
-
- } catch (SQLException sqlException) {
- throw sqlException;
- }
-
- closeConnection(connection);
- }
-
- private Connection getConnection() {
- return pool.getConnection();
- }
-
- private void closeConnection(Connection connection) {
- pool.returnConnection(connection);
- }
-
- static void extractResults(ResultSet resultSet, List<List<String>> results) throws SQLException {
- List<String> oneRow = new LinkedList<String>();
-
- try {
- while (resultSet.next()) {
- //logger.debug("resultSet value: " + resultSet.getString(1));
- for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) {
- oneRow.add((resultSet.getString(i + 1)));
- }
-
- results.add(new LinkedList<String>(oneRow));
- oneRow.clear();
- }
- } catch (SQLException sqlException) {
- throw sqlException;
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
new file mode 100644
index 0000000..8001880
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.adhoc;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.pool.impl.GenericObjectPool;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
+import org.apache.kylin.source.adhocquery.IPushDownRunner;
+
+public class PushDownRunnerJdbcImpl implements IPushDownRunner {
+
+ private static org.apache.kylin.query.adhoc.JdbcConnectionPool pool = null;
+
+ @Override
+ public void init(KylinConfig config) {
+ if (pool == null) {
+ pool = new JdbcConnectionPool();
+ JdbcConnectionFactory factory = new JdbcConnectionFactory(config.getJdbcUrl(), config.getJdbcDriverClass(),
+ config.getJdbcUsername(), config.getJdbcPassword());
+ GenericObjectPool.Config poolConfig = new GenericObjectPool.Config();
+ poolConfig.maxActive = config.getPoolMaxTotal();
+ poolConfig.maxIdle = config.getPoolMaxIdle();
+ poolConfig.minIdle = config.getPoolMinIdle();
+
+ try {
+ pool.createPool(factory, poolConfig);
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
+ }
+
+ @Override
+ public void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas)
+ throws Exception {
+ Statement statement = null;
+ Connection connection = this.getConnection();
+ ResultSet resultSet = null;
+
+ try {
+ statement = connection.createStatement();
+ resultSet = statement.executeQuery(query);
+ extractResults(resultSet, results);
+ } catch (SQLException sqlException) {
+ throw sqlException;
+ }
+
+ //extract column metadata
+ ResultSetMetaData metaData = null;
+ int columnCount = 0;
+ try {
+ metaData = resultSet.getMetaData();
+ columnCount = metaData.getColumnCount();
+
+ // fill in selected column meta
+ for (int i = 1; i <= columnCount; ++i) {
+ columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false,
+ metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i),
+ metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null,
+ metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i),
+ metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false));
+ }
+
+ } catch (SQLException sqlException) {
+ throw sqlException;
+ }
+
+ closeConnection(connection);
+ }
+
+ private Connection getConnection() {
+ return pool.getConnection();
+ }
+
+ private void closeConnection(Connection connection) {
+ pool.returnConnection(connection);
+ }
+
+ static void extractResults(ResultSet resultSet, List<List<String>> results) throws SQLException {
+ List<String> oneRow = new LinkedList<String>();
+
+ try {
+ while (resultSet.next()) {
+ //logger.debug("resultSet value: " + resultSet.getString(1));
+ for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) {
+ oneRow.add((resultSet.getString(i + 1)));
+ }
+
+ results.add(new LinkedList<String>(oneRow));
+ oneRow.clear();
+ }
+ } catch (SQLException sqlException) {
+ throw sqlException;
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index d841dee..79a2c05 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@ -61,12 +61,13 @@ public class SQLResponse implements Serializable {
protected boolean storageCacheUsed = false;
- protected boolean queryAdHoc = false;
+ protected boolean queryPushDown = false;
public SQLResponse() {
}
- public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, int affectedRowCount, boolean isException, String exceptionMessage) {
+ public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, int affectedRowCount,
+ boolean isException, String exceptionMessage) {
this.columnMetas = columnMetas;
this.results = results;
this.affectedRowCount = affectedRowCount;
@@ -74,7 +75,8 @@ public class SQLResponse implements Serializable {
this.exceptionMessage = exceptionMessage;
}
- public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube, int affectedRowCount, boolean isException, String exceptionMessage) {
+ public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube,
+ int affectedRowCount, boolean isException, String exceptionMessage) {
this.columnMetas = columnMetas;
this.results = results;
this.cube = cube;
@@ -83,7 +85,8 @@ public class SQLResponse implements Serializable {
this.exceptionMessage = exceptionMessage;
}
- public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube, int affectedRowCount, boolean isException, String exceptionMessage, boolean isPartial, boolean isAdhoc) {
+ public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube,
+ int affectedRowCount, boolean isException, String exceptionMessage, boolean isPartial, boolean isPushDown) {
this.columnMetas = columnMetas;
this.results = results;
this.cube = cube;
@@ -91,7 +94,7 @@ public class SQLResponse implements Serializable {
this.isException = isException;
this.exceptionMessage = exceptionMessage;
this.isPartial = isPartial;
- this.queryAdHoc = isAdhoc;
+ this.queryPushDown = isPushDown;
}
public List<SelectedColumnMeta> getColumnMetas() {
@@ -147,8 +150,8 @@ public class SQLResponse implements Serializable {
return isPartial;
}
- public boolean isAdHoc() {
- return queryAdHoc;
+ public boolean isPushDown() {
+ return queryPushDown;
}
public long getTotalScanCount() {
[02/50] [abbrv] kylin git commit: KYLIN-1668
Posted by ka...@apache.org.
KYLIN-1668
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f3cb39e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f3cb39e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f3cb39e
Branch: refs/heads/KYLIN-2606
Commit: 3f3cb39e1cfa56231d9a6ede5c8becd5a0c6c6af
Parents: a9b0b6e
Author: luguosheng <lu...@luguoshengdeMacBook-Pro.local>
Authored: Tue Jun 27 21:37:12 2017 +0800
Committer: chenzhx <34...@qq.com>
Committed: Tue Jun 27 21:46:30 2017 +0800
----------------------------------------------------------------------
webapp/app/partials/cubeDesigner/advanced_settings.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3f3cb39e/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index ffa7589..f0c3bcd 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -221,7 +221,7 @@
<td>
<!--Column Name -->
<input type="text" class="form-control" placeholder="Column Name.." ng-if="state.mode=='edit'"
- ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column)" ng-disabled="instance.status=='READY'"
+ ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column)" ng-disabled="true"
tooltip="rowkey column name.." tooltip-trigger="focus"
ng-model="rowkey_column.column" class="form-control">
[28/50] [abbrv] kylin git commit: KYLIN-2696 Check SQL injection in
filter condition
Posted by ka...@apache.org.
KYLIN-2696 Check SQL injection in filter condition
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2546e00c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2546e00c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2546e00c
Branch: refs/heads/KYLIN-2606
Commit: 2546e00cf2f80b0f8abba9ec3cfe9a0fbce5cb4f
Parents: e7f9dab
Author: Xiaqing <45...@qq.com>
Authored: Fri Jun 30 09:42:07 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 09:42:07 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 43 ++++++++++++++++++++
1 file changed, 43 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2546e00c/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 91802f7..341f36e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -352,6 +352,7 @@ public class DataModelDesc extends RootPersistentEntity {
initDimensionsAndMetrics();
initPartitionDesc();
initComputedColumns(ccInfoMap);
+ initFilterCondition();
boolean reinit = validate();
if (reinit) { // model slightly changed by validate() and must init() again
@@ -505,6 +506,48 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
+ //Check if the filter condition is illegal.
+ private void initFilterCondition() {
+ if (null == this.filterCondition) {
+ return;
+ }
+ int quotationType = 0;
+ int len = this.filterCondition.length();
+ for (int i = 0; i < len; i++) {
+ //If a ';' which is not within a string is found, throw exception.
+ if (';' == this.filterCondition.charAt(i) && 0 == quotationType) {
+ throw new IllegalStateException(
+ "Filter Condition is Illegal. Please check it and make sure it's an appropriate expression for WHERE clause");
+ }
+ if ('\'' == this.filterCondition.charAt(i)) {
+ if (quotationType > 0) {
+ if (1 == quotationType) {
+ quotationType = 0;
+ continue;
+ }
+ } else {
+ if (0 == quotationType) {
+ quotationType = 1;
+ continue;
+ }
+ }
+ }
+ if ('"' == this.filterCondition.charAt(i)) {
+ if (quotationType > 0) {
+ if (2 == quotationType) {
+ quotationType = 0;
+ continue;
+ }
+ } else {
+ if (0 == quotationType) {
+ quotationType = 2;
+ continue;
+ }
+ }
+ }
+ }
+ }
+
private void checkSameCCDefinition(Map<String, CCInfo> ccInfoMap, ComputedColumnDesc thisCCDesc,
String thisCCName) {
//check whether two computer columns's definition is the same.
[30/50] [abbrv] kylin git commit: minor,
fix project rename bug (#1475)
Posted by ka...@apache.org.
minor, fix project rename bug (#1475)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5fb4f350
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5fb4f350
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5fb4f350
Branch: refs/heads/KYLIN-2606
Commit: 5fb4f3500d12e3ad160b2235667de6131907af57
Parents: f52eb0c
Author: Roger Shi <ro...@gmail.com>
Authored: Fri Jun 30 14:31:24 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 14:31:24 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/service/ProjectService.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5fb4f350/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index c6bd6cf..cd60128 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -121,6 +121,7 @@ public class ProjectService extends BasicService {
// rebind draft and project
for (Draft draft : getDraftManager().list(currentProject.getName())) {
draft.setProject(newProjectName);
+ getDraftManager().save(draft);
}
logger.debug("Project rename.");
[35/50] [abbrv] kylin git commit: Use calcite's keywords during
subquery convert.
Posted by ka...@apache.org.
Use calcite's keywords during subquery convert.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc232d11
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc232d11
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc232d11
Branch: refs/heads/KYLIN-2606
Commit: dc232d114c5ad8c50ebed60cf238a0f5b21802a8
Parents: 03702f1
Author: nichunen <ch...@kyligence.io>
Authored: Fri Jun 30 19:43:53 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 30 19:57:28 2017 +0800
----------------------------------------------------------------------
.../source/adhocquery/HiveAdhocConverter.java | 102 ++++++++++++++-----
1 file changed, 79 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dc232d11/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
index 89b2f7a..e8c06ff 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
@@ -45,29 +45,85 @@ public class HiveAdhocConverter implements IAdHocConverter {
Pattern.CASE_INSENSITIVE);
private static final Pattern TIMESTAMPADD_PATTERN = Pattern.compile("timestampadd\\s*\\(\\s*(.*?)\\s*,",
Pattern.CASE_INSENSITIVE);
- private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("ABSOLUTE", "ACTION", "ADD", "ALL",
- "ALLOCATE", "ALTER", "AND", "ANY", "ARE", "ASC", "ASSERTION", "AT", "AUTHORIZATION", "AVG", "BEGIN",
- "BETWEEN", "BIT", "BIT_LENGTH", "BOTH", "BY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG", "CHAR",
- "CHARACTER", "CHARACTER_LENGTH", "CHAR_LENGTH", "CHECK", "CLOSE", "COALESCE", "COLLATE", "COLLATION",
- "COLUMN", "COMMIT", "CONNECT", "CONNECTION", "CONSTRAINT", "CONSTRAINTS", "CONTINUE", "CONVERT",
- "CORRESPONDING", "COUNT", "CREATE", "CROSS", "CURRENT", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP",
- "CURRENT_USER", "CURSOR", "DATE", "DAY", "DEALLOCATE", "DEC", "DECIMAL", "DECLARE", "DEFAULT", "DEFERRABLE",
- "DEFERRED", "DELETE", "DESC", "DESCRIBE", "DESCRIPTOR", "DIAGNOSTICS", "DISCONNECT", "DISTINCT", "DOMAIN",
- "DOUBLE", "DROP", "ELSE", "END", "END-EXEC", "ESCAPE", "EXCEPT", "EXCEPTION", "EXEC", "EXECUTE", "EXISTS",
- "EXTERNAL", "EXTRACT", "FALSE", "FETCH", "FIRST", "FLOAT", "FOR", "FOREIGN", "FOUND", "FROM", "FULL", "GET",
- "GLOBAL", "GO", "GOTO", "GRANT", "GROUP", "HAVING", "HOUR", "IDENTITY", "IMMEDIATE", "IN", "INDICATOR",
- "INITIALLY", "INNER", "INADD", "INSENSITIVE", "INSERT", "INT", "INTEGER", "INTERSECT", "INTERVAL", "INTO",
- "IS", "ISOLATION", "JOIN", "KEY", "LANGUAGE", "LAST", "LEADING", "LEFT", "LEVEL", "LIKE", "LOCAL", "LOWER",
- "MATCH", "MAX", "MIN", "MINUTE", "MODULE", "MONTH", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NEXT", "NO",
- "NOT", "NULL", "NULLIF", "NUMERIC", "OCTET_LENGTH", "OF", "ON", "ONLY", "OPEN", "OPTION", "OR", "ORDER",
- "OUTER", "OUTADD", "OVERLAPS", "PAD", "PARTIAL", "POSITION", "PRECISION", "PREPARE", "PRESERVE", "PRIMARY",
- "PRIOR", "PRIVILEGES", "PROCEDURE", "PUBLIC", "READ", "REAL", "REFERENCES", "RELATIVE", "RESTRICT",
- "REVOKE", "RIGHT", "ROLLBACK", "ROWS", "SCHEMA", "SCROLL", "SECOND", "SECTION", "SELECT", "SESSION",
- "SESSION_USER", "SET", "SIZE", "SMALLINT", "SOME", "SPACE", "SQL", "SQLCODE", "SQLERROR", "SQLSTATE",
- "SUBSTRING", "SUM", "SYSTEM_USER", "TABLE", "TEMPORARY", "THEN", "TIME", "TIMESTAMP", "TIMEZONE_HOUR",
- "TIMEZONE_MINUTE", "TO", "TRAILING", "TRANSACTION", "TRANSLATE", "TRANSLATION", "TRIM", "TRUE", "UNION",
- "UNIQUE", "UNKNOWN", "UPDATE", "UPPER", "USAGE", "USER", "USING", "VALUE", "VALUES", "VARCHAR", "VARYING",
- "VIEW", "WHEN", "WHENEVER", "WHERE", "WITH", "WORK", "WRITE", "YEAR", "ZONE");
+ private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("A", "ABS", "ABSOLUTE", "ACTION",
+ "ADA", "ADD", "ADMIN", "AFTER", "ALL", "ALLOCATE", "ALLOW", "ALTER", "ALWAYS", "AND", "ANY", "APPLY", "ARE",
+ "ARRAY", "ARRAY_MAX_CARDINALITY", "ASC", "ASENSITIVE", "ASSERTION", "ASSIGNMENT", "ASYMMETRIC", "AT",
+ "ATOMIC", "ATTRIBUTE", "ATTRIBUTES", "AUTHORIZATION", "AVG", "BEFORE", "BEGIN", "BEGIN_FRAME",
+ "BEGIN_PARTITION", "BERNOULLI", "BETWEEN", "BIGINT", "BINARY", "BIT", "BLOB", "BOOLEAN", "BOTH", "BREADTH",
+ "BY", "C", "CALL", "CALLED", "CARDINALITY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG",
+ "CATALOG_NAME", "CEIL", "CEILING", "CENTURY", "CHAIN", "CHAR", "CHARACTER", "CHARACTERISTICS", "CHARACTERS",
+ "CHARACTER_LENGTH", "CHARACTER_SET_CATALOG", "CHARACTER_SET_NAME", "CHARACTER_SET_SCHEMA", "CHAR_LENGTH",
+ "CHECK", "CLASSIFIER", "CLASS_ORIGIN", "CLOB", "CLOSE", "COALESCE", "COBOL", "COLLATE", "COLLATION",
+ "COLLATION_CATALOG", "COLLATION_NAME", "COLLATION_SCHEMA", "COLLECT", "COLUMN", "COLUMN_NAME",
+ "COMMAND_FUNCTION", "COMMAND_FUNCTION_CODE", "COMMIT", "COMMITTED", "CONDITION", "CONDITION_NUMBER",
+ "CONNECT", "CONNECTION", "CONNECTION_NAME", "CONSTRAINT", "CONSTRAINTS", "CONSTRAINT_CATALOG",
+ "CONSTRAINT_NAME", "CONSTRAINT_SCHEMA", "CONSTRUCTOR", "CONTAINS", "CONTINUE", "CONVERT", "CORR",
+ "CORRESPONDING", "COUNT", "COVAR_POP", "COVAR_SAMP", "CREATE", "CROSS", "CUBE", "CUME_DIST", "CURRENT",
+ "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_DEFAULT_TRANSFORM_GROUP", "CURRENT_PATH", "CURRENT_ROLE",
+ "CURRENT_ROW", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_TRANSFORM_GROUP_FOR_TYPE",
+ "CURRENT_USER", "CURSOR", "CURSOR_NAME", "CYCLE", "DATA", "DATABASE", "DATE", "DATETIME_INTERVAL_CODE",
+ "DATETIME_INTERVAL_PRECISION", "DAY", "DEALLOCATE", "DEC", "DECADE", "DECIMAL", "DECLARE", "DEFAULT",
+ "DEFAULTS", "DEFERRABLE", "DEFERRED", "DEFINE", "DEFINED", "DEFINER", "DEGREE", "DELETE", "DENSE_RANK",
+ "DEPTH", "DEREF", "DERIVED", "DESC", "DESCRIBE", "DESCRIPTION", "DESCRIPTOR", "DETERMINISTIC",
+ "DIAGNOSTICS", "DISALLOW", "DISCONNECT", "DISPATCH", "DISTINCT", "DOMAIN", "DOUBLE", "DOW", "DOY", "DROP",
+ "DYNAMIC", "DYNAMIC_FUNCTION", "DYNAMIC_FUNCTION_CODE", "EACH", "ELEMENT", "ELSE", "EMPTY", "END",
+ "END-EXEC", "END_FRAME", "END_PARTITION", "EPOCH", "EQUALS", "ESCAPE", "EVERY", "EXCEPT", "EXCEPTION",
+ "EXCLUDE", "EXCLUDING", "EXEC", "EXECUTE", "EXISTS", "EXP", "EXPLAIN", "EXTEND", "EXTERNAL", "EXTRACT",
+ "FALSE", "FETCH", "FILTER", "FINAL", "FIRST", "FIRST_VALUE", "FLOAT", "FLOOR", "FOLLOWING", "FOR",
+ "FOREIGN", "FORTRAN", "FOUND", "FRAC_SECOND", "FRAME_ROW", "FREE", "FROM", "FULL", "FUNCTION", "FUSION",
+ "G", "GENERAL", "GENERATED", "GET", "GLOBAL", "GO", "GOTO", "GRANT", "GRANTED", "GROUP", "GROUPING",
+ "GROUPS", "HAVING", "HIERARCHY", "HOLD", "HOUR", "IDENTITY", "IMMEDIATE", "IMMEDIATELY", "IMPLEMENTATION",
+ "IMPORT", "IN", "INCLUDING", "INCREMENT", "INDICATOR", "INITIAL", "INITIALLY", "INNER", "INOUT", "INPUT",
+ "INSENSITIVE", "INSERT", "INSTANCE", "INSTANTIABLE", "INT", "INTEGER", "INTERSECT", "INTERSECTION",
+ "INTERVAL", "INTO", "INVOKER", "IS", "ISOLATION", "JAVA", "JOIN", "JSON", "K", "KEY", "KEY_MEMBER",
+ "KEY_TYPE", "LABEL", "LAG", "LANGUAGE", "LARGE", "LAST", "LAST_VALUE", "LATERAL", "LEAD", "LEADING", "LEFT",
+ "LENGTH", "LEVEL", "LIBRARY", "LIKE", "LIKE_REGEX", "LIMIT", "LN", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP",
+ "LOCATOR", "LOWER", "M", "MAP", "MATCH", "MATCHED", "MATCHES", "MATCH_NUMBER", "MATCH_RECOGNIZE", "MAX",
+ "MAXVALUE", "MEASURES", "MEMBER", "MERGE", "MESSAGE_LENGTH", "MESSAGE_OCTET_LENGTH", "MESSAGE_TEXT",
+ "METHOD", "MICROSECOND", "MILLENNIUM", "MIN", "MINUS", "MINUTE", "MINVALUE", "MOD", "MODIFIES", "MODULE",
+ "MONTH", "MORE", "MULTISET", "MUMPS", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NCLOB", "NESTING",
+ "NEW", "NEXT", "NO", "NONE", "NORMALIZE", "NORMALIZED", "NOT", "NTH_VALUE", "NTILE", "NULL", "NULLABLE",
+ "NULLIF", "NULLS", "NUMBER", "NUMERIC", "OBJECT", "OCCURRENCES_REGEX", "OCTETS", "OCTET_LENGTH", "OF",
+ "OFFSET", "OLD", "OMIT", "ON", "ONE", "ONLY", "OPEN", "OPTION", "OPTIONS", "OR", "ORDER", "ORDERING",
+ "ORDINALITY", "OTHERS", "OUT", "OUTER", "OUTPUT", "OVER", "OVERLAPS", "OVERLAY", "OVERRIDING", "PAD",
+ "PARAMETER", "PARAMETER_MODE", "PARAMETER_NAME", "PARAMETER_ORDINAL_POSITION", "PARAMETER_SPECIFIC_CATALOG",
+ "PARAMETER_SPECIFIC_NAME", "PARAMETER_SPECIFIC_SCHEMA", "PARTIAL", "PARTITION", "PASCAL", "PASSTHROUGH",
+ "PAST", "PATH", "PATTERN", "PER", "PERCENT", "PERCENTILE_CONT", "PERCENTILE_DISC", "PERCENT_RANK", "PERIOD",
+ "PERMUTE", "PLACING", "PLAN", "PLI", "PORTION", "POSITION", "POSITION_REGEX", "POWER", "PRECEDES",
+ "PRECEDING", "PRECISION", "PREPARE", "PRESERVE", "PREV", "PRIMARY", "PRIOR", "PRIVILEGES", "PROCEDURE",
+ "PUBLIC", "QUARTER", "RANGE", "RANK", "READ", "READS", "REAL", "RECURSIVE", "REF", "REFERENCES",
+ "REFERENCING", "REGR_AVGX", "REGR_AVGY", "REGR_COUNT", "REGR_INTERCEPT", "REGR_R2", "REGR_SLOPE",
+ "REGR_SXX", "REGR_SXY", "REGR_SYY", "RELATIVE", "RELEASE", "REPEATABLE", "REPLACE", "RESET", "RESTART",
+ "RESTRICT", "RESULT", "RETURN", "RETURNED_CARDINALITY", "RETURNED_LENGTH", "RETURNED_OCTET_LENGTH",
+ "RETURNED_SQLSTATE", "RETURNS", "REVOKE", "RIGHT", "ROLE", "ROLLBACK", "ROLLUP", "ROUTINE",
+ "ROUTINE_CATALOG", "ROUTINE_NAME", "ROUTINE_SCHEMA", "ROW", "ROWS", "ROW_COUNT", "ROW_NUMBER", "RUNNING",
+ "SAVEPOINT", "SCALE", "SCHEMA", "SCHEMA_NAME", "SCOPE", "SCOPE_CATALOGS", "SCOPE_NAME", "SCOPE_SCHEMA",
+ "SCROLL", "SEARCH", "SECOND", "SECTION", "SECURITY", "SEEK", "SELECT", "SELF", "SENSITIVE", "SEQUENCE",
+ "SERIALIZABLE", "SERVER", "SERVER_NAME", "SESSION", "SESSION_USER", "SET", "SETS", "SHOW", "SIMILAR",
+ "SIMPLE", "SIZE", "SKIP", "SMALLINT", "SOME", "SOURCE", "SPACE", "SPECIFIC", "SPECIFICTYPE",
+ "SPECIFIC_NAME", "SQL", "SQLEXCEPTION", "SQLSTATE", "SQLWARNING", "SQL_BIGINT", "SQL_BINARY", "SQL_BIT",
+ "SQL_BLOB", "SQL_BOOLEAN", "SQL_CHAR", "SQL_CLOB", "SQL_DATE", "SQL_DECIMAL", "SQL_DOUBLE", "SQL_FLOAT",
+ "SQL_INTEGER", "SQL_INTERVAL_DAY", "SQL_INTERVAL_DAY_TO_HOUR", "SQL_INTERVAL_DAY_TO_MINUTE",
+ "SQL_INTERVAL_DAY_TO_SECOND", "SQL_INTERVAL_HOUR", "SQL_INTERVAL_HOUR_TO_MINUTE",
+ "SQL_INTERVAL_HOUR_TO_SECOND", "SQL_INTERVAL_MINUTE", "SQL_INTERVAL_MINUTE_TO_SECOND", "SQL_INTERVAL_MONTH",
+ "SQL_INTERVAL_SECOND", "SQL_INTERVAL_YEAR", "SQL_INTERVAL_YEAR_TO_MONTH", "SQL_LONGVARBINARY",
+ "SQL_LONGVARCHAR", "SQL_LONGVARNCHAR", "SQL_NCHAR", "SQL_NCLOB", "SQL_NUMERIC", "SQL_NVARCHAR", "SQL_REAL",
+ "SQL_SMALLINT", "SQL_TIME", "SQL_TIMESTAMP", "SQL_TINYINT", "SQL_TSI_DAY", "SQL_TSI_FRAC_SECOND",
+ "SQL_TSI_HOUR", "SQL_TSI_MICROSECOND", "SQL_TSI_MINUTE", "SQL_TSI_MONTH", "SQL_TSI_QUARTER",
+ "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SQL_VARBINARY", "SQL_VARCHAR", "SQRT", "START", "STATE",
+ "STATEMENT", "STATIC", "STDDEV_POP", "STDDEV_SAMP", "STREAM", "STRUCTURE", "STYLE", "SUBCLASS_ORIGIN",
+ "SUBMULTISET", "SUBSET", "SUBSTITUTE", "SUBSTRING", "SUBSTRING_REGEX", "SUCCEEDS", "SUM", "SYMMETRIC",
+ "SYSTEM", "SYSTEM_TIME", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TABLE_NAME", "TEMPORARY", "THEN", "TIES",
+ "TIME", "TIMESTAMP", "TIMESTAMPADD", "TIMESTAMPDIFF", "TIMEZONE_HOUR", "TIMEZONE_MINUTE", "TINYINT", "TO",
+ "TOP_LEVEL_COUNT", "TRAILING", "TRANSACTION", "TRANSACTIONS_ACTIVE", "TRANSACTIONS_COMMITTED",
+ "TRANSACTIONS_ROLLED_BACK", "TRANSFORM", "TRANSFORMS", "TRANSLATE", "TRANSLATE_REGEX", "TRANSLATION",
+ "TREAT", "TRIGGER", "TRIGGER_CATALOG", "TRIGGER_NAME", "TRIGGER_SCHEMA", "TRIM", "TRIM_ARRAY", "TRUE",
+ "TRUNCATE", "TYPE", "UESCAPE", "UNBOUNDED", "UNCOMMITTED", "UNDER", "UNION", "UNIQUE", "UNKNOWN", "UNNAMED",
+ "UNNEST", "UPDATE", "UPPER", "UPSERT", "USAGE", "USER", "USER_DEFINED_TYPE_CATALOG",
+ "USER_DEFINED_TYPE_CODE", "USER_DEFINED_TYPE_NAME", "USER_DEFINED_TYPE_SCHEMA", "USING", "VALUE", "VALUES",
+ "VALUE_OF", "VARBINARY", "VARCHAR", "VARYING", "VAR_POP", "VAR_SAMP", "VERSION", "VERSIONING", "VIEW",
+ "WEEK", "WHEN", "WHENEVER", "WHERE", "WIDTH_BUCKET", "WINDOW", "WITH", "WITHIN", "WITHOUT", "WORK",
+ "WRAPPER", "WRITE", "XML", "YEAR", "ZONE");
public static String replaceString(String originString, String fromString, String toString) {
return originString.replace(fromString, toString);
[46/50] [abbrv] kylin git commit: KYLIN-1895 Cube Name displays the
name of CubeDesc on UI
Posted by ka...@apache.org.
KYLIN-1895 Cube Name displays the name of CubeDesc on UI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/926a17ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/926a17ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/926a17ae
Branch: refs/heads/KYLIN-2606
Commit: 926a17ae3a4646eeda0cb19abcbb7fe182b174e4
Parents: b0cc68b
Author: chenzhx <34...@qq.com>
Authored: Fri Sep 9 16:26:51 2016 +0800
Committer: chenzhx <34...@qq.com>
Committed: Wed Jul 5 14:14:24 2017 +0800
----------------------------------------------------------------------
webapp/app/partials/cubeDesigner/info.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/926a17ae/webapp/app/partials/cubeDesigner/info.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/info.html b/webapp/app/partials/cubeDesigner/info.html
index b1e4d48..35e6f86 100644
--- a/webapp/app/partials/cubeDesigner/info.html
+++ b/webapp/app/partials/cubeDesigner/info.html
@@ -49,7 +49,7 @@
</div>
<!--Cube Name-->
- <div class="form-group middle-popover" ng-class="{'required':state.mode=='edit'}">
+ <div class="form-group middle-popover required" ng-if="state.mode=='edit'">
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
<b>Cube Name</b> <i class="fa fa-info-circle" kylinpopover placement="right" title="Cube Name" template="cubeNameTip.html"></i>
[11/50] [abbrv] kylin git commit: minor,
enhance computed column checks
Posted by ka...@apache.org.
minor, enhance computed column checks
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d61f867f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d61f867f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d61f867f
Branch: refs/heads/KYLIN-2606
Commit: d61f867ff9fa170e155310edeb03db6059433f2f
Parents: 69532cd
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jun 28 21:13:25 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 13:51:45 2017 +0800
----------------------------------------------------------------------
core-metadata/pom.xml | 16 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 15 +-
.../metadata/model/ComputedColumnDesc.java | 3 +
.../kylin/metadata/model/DataModelDesc.java | 30 ++-
.../apache/kylin/metadata/model/TblColRef.java | 2 +-
.../metadata/model/tool/CalciteParser.java | 203 +++++++++++++++++++
.../kylin/model/tool/CalciteParserTest.java | 111 ++++++++++
.../model_desc/ci_inner_join_model.json | 6 +-
.../model_desc/ci_left_join_model.json | 6 +-
.../query/util/ConvertToComputedColumn.java | 111 ++--------
.../query/util/ConvertToComputedColumnTest.java | 22 --
11 files changed, 379 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index f5ab12a..d45a838 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -17,7 +17,8 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -60,6 +61,19 @@
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>atopcalcite</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-core</artifactId>
+ </exclusion>
+ </exclusions>
+ <!--set to provided on purpose, so that dependant on metadata module will not depend on calcite unnecessarily-->
+ <!--https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html-->
+ <scope>provided</scope>
+ </dependency>
<!-- Compiled -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 5d15d56..3c84c96 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -19,9 +19,9 @@
package org.apache.kylin.metadata.model;
import java.io.Serializable;
-import java.util.regex.Pattern;
import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
@@ -82,7 +82,8 @@ public class ColumnDesc implements Serializable {
this.index = other.index;
}
- public ColumnDesc(String id, String name, String datatype, String comment, String dataGen, String index, String computedColumnExpr) {
+ public ColumnDesc(String id, String name, String datatype, String comment, String dataGen, String index,
+ String computedColumnExpr) {
this.id = id;
this.name = name;
this.datatype = datatype;
@@ -193,11 +194,12 @@ public class ColumnDesc implements Serializable {
return index;
}
- public String getComputedColumnExpr(String tableAlias, String tableIdentity) {
+ public String getComputedColumnExpr(String tableAlias) {
Preconditions.checkState(computedColumnExpr != null);
+ Preconditions.checkState(tableAlias != null);
- //http://stackoverflow.com/questions/5054995/how-to-replace-case-insensitive-literal-substrings-in-java
- return computedColumnExpr.replaceAll("(?i)" + Pattern.quote(tableIdentity), tableAlias);
+ String s = CalciteParser.insertAliasInExpr(computedColumnExpr, tableAlias);
+ return s;
}
public boolean isComputedColumnn() {
@@ -257,6 +259,7 @@ public class ColumnDesc implements Serializable {
@Override
public String toString() {
- return "ColumnDesc{" + "id='" + id + '\'' + ", name='" + name + '\'' + ", datatype='" + datatype + '\'' + ", comment='" + comment + '\'' + '}';
+ return "ColumnDesc{" + "id='" + id + '\'' + ", name='" + name + '\'' + ", datatype='" + datatype + '\''
+ + ", comment='" + comment + '\'' + '}';
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 013cb8c..540b5fc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class ComputedColumnDesc {
@@ -42,6 +43,8 @@ public class ComputedColumnDesc {
tableIdentity = tableIdentity.toUpperCase();
columnName = columnName.toUpperCase();
+
+ CalciteParser.ensureNoTableNameExists(expression);
}
public String getFullName() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 7de955e..91802f7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -489,17 +489,11 @@ public class DataModelDesc extends RootPersistentEntity {
}
CCInfo other = ccInfoMap.get(thisCCName);
- if (other == null || (other.getDataModelDescs().size() == 1 && other.getDataModelDescs().contains(this))) {
- //check whether two computer columns's definition is the same.
- for (CCInfo sysCCInfo : ccInfoMap.values()) {
- String definition0 = thisCCDesc.getExpression();
- String definition1 = sysCCInfo.getComputedColumnDesc().getExpression();
- if (isTwoCCDefinitionEquals(definition0, definition1)) {
- throw new IllegalStateException(String.format(
- "Computed column %s'definition: %s is already defined in other models: %s. Please change another definition, or try to keep consistent definition",
- thisCCName, definition0, sysCCInfo.getDataModelDescs()));
- }
- }
+
+ if (other == null) {
+ checkSameCCDefinition(ccInfoMap, thisCCDesc, thisCCName);
+ ccInfoMap.put(thisCCName, new CCInfo(thisCCDesc, Sets.<DataModelDesc> newHashSet(this)));
+ } else if (other.getDataModelDescs().size() == 1 && other.getDataModelDescs().contains(this)) {
ccInfoMap.put(thisCCName, new CCInfo(thisCCDesc, Sets.<DataModelDesc> newHashSet(this)));
} else if (other.getComputedColumnDesc().equals(thisCCDesc)) {
other.getDataModelDescs().add(this);
@@ -511,6 +505,20 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
+ private void checkSameCCDefinition(Map<String, CCInfo> ccInfoMap, ComputedColumnDesc thisCCDesc,
+ String thisCCName) {
+ //check whether two computer columns's definition is the same.
+ for (CCInfo sysCCInfo : ccInfoMap.values()) {
+ String definition0 = thisCCDesc.getExpression();
+ String definition1 = sysCCInfo.getComputedColumnDesc().getExpression();
+ if (isTwoCCDefinitionEquals(definition0, definition1)) {
+ throw new IllegalStateException(String.format(
+ "Computed column %s's definition: %s is already defined in other models: %s. Please change another definition, or try to keep consistent definition",
+ thisCCName, definition0, sysCCInfo.getDataModelDescs()));
+ }
+ }
+ }
+
private boolean isTwoCCDefinitionEquals(String definition0, String definition1) {
definition0 = definition0.replaceAll("\\s*", "");
definition1 = definition1.replaceAll("\\s*", "");
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 5a28e8b..5563856 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -158,7 +158,7 @@ public class TblColRef implements Serializable {
if (!column.isComputedColumnn()) {
return getIdentity();
} else {
- return column.getComputedColumnExpr(getTableAlias(), table.getTableIdentity());
+ return column.getComputedColumnExpr(getTableAlias());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
new file mode 100644
index 0000000..47a90bf
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model.tool;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.util.SqlBasicVisitor;
+import org.apache.calcite.sql.util.SqlVisitor;
+import org.apache.commons.lang3.tuple.Pair;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class CalciteParser {
+ public static SqlNode parse(String sql) throws SqlParseException {
+ SqlParser.ConfigBuilder parserBuilder = SqlParser.configBuilder();
+ SqlParser sqlParser = SqlParser.create(sql, parserBuilder.build());
+ return sqlParser.parseQuery();
+ }
+
+ public static SqlNode getOnlySelectNode(String sql) {
+ SqlNodeList selectList = null;
+ try {
+ selectList = ((SqlSelect) CalciteParser.parse(sql)).getSelectList();
+ } catch (SqlParseException e) {
+ throw new RuntimeException("Failed to parse expression \'" + sql
+ + "\', please make sure the expression is valid and no table name exists in the expression");
+ }
+
+ Preconditions.checkArgument(selectList.size() == 1,
+ "Expression is invalid because size of select list exceeds one");
+
+ return selectList.get(0);
+ }
+
+ public static SqlNode getExpNode(String expr) {
+ return getOnlySelectNode("select " + expr + " from t");
+
+ }
+
+ public static boolean isNodeEqual(SqlNode node0, SqlNode node1) {
+ if (node0 == null) {
+ return node1 == null;
+ } else if (node1 == null) {
+ return false;
+ }
+
+ if (!Objects.equals(node0.getClass().getSimpleName(), node1.getClass().getSimpleName())) {
+ return false;
+ }
+
+ if (node0 instanceof SqlCall) {
+ SqlCall thisNode = (SqlCall) node0;
+ SqlCall thatNode = (SqlCall) node1;
+ if (!thisNode.getOperator().getName().equalsIgnoreCase(thatNode.getOperator().getName())) {
+ return false;
+ }
+ return isNodeEqual(thisNode.getOperandList(), thatNode.getOperandList());
+ }
+ if (node0 instanceof SqlLiteral) {
+ SqlLiteral thisNode = (SqlLiteral) node0;
+ SqlLiteral thatNode = (SqlLiteral) node1;
+ return Objects.equals(thisNode.getValue(), thatNode.getValue());
+ }
+ if (node0 instanceof SqlNodeList) {
+ SqlNodeList thisNode = (SqlNodeList) node0;
+ SqlNodeList thatNode = (SqlNodeList) node1;
+ if (thisNode.getList().size() != thatNode.getList().size()) {
+ return false;
+ }
+ for (int i = 0; i < thisNode.getList().size(); i++) {
+ SqlNode thisChild = thisNode.getList().get(i);
+ final SqlNode thatChild = thatNode.getList().get(i);
+ if (!isNodeEqual(thisChild, thatChild)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ if (node0 instanceof SqlIdentifier) {
+ SqlIdentifier thisNode = (SqlIdentifier) node0;
+ SqlIdentifier thatNode = (SqlIdentifier) node1;
+ // compare ignore table alias.eg: expression like "a.b + a.c + a.d" ,alias a will be ignored when compared
+ String name0 = thisNode.names.get(thisNode.names.size() - 1).replace("\"", "");
+ String name1 = thatNode.names.get(thatNode.names.size() - 1).replace("\"", "");
+ return name0.equalsIgnoreCase(name1);
+ }
+
+ return false;
+ }
+
+ private static boolean isNodeEqual(List<SqlNode> operands0, List<SqlNode> operands1) {
+ if (operands0.size() != operands1.size()) {
+ return false;
+ }
+ for (int i = 0; i < operands0.size(); i++) {
+ if (!isNodeEqual(operands0.get(i), operands1.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static void ensureNoTableNameExists(String expr) {
+ SqlNode sqlNode = getExpNode(expr);
+
+ SqlVisitor sqlVisitor = new SqlBasicVisitor() {
+ @Override
+ public Object visit(SqlIdentifier id) {
+ if (id.names.size() > 1) {
+ throw new IllegalArgumentException("SqlIdentifier " + id + " contains DB/Table name");
+ }
+ return null;
+ }
+ };
+
+ sqlNode.accept(sqlVisitor);
+ }
+
+ public static String insertAliasInExpr(String expr, String alias) {
+ String prefix = "select ";
+ String suffix = " from t";
+ String sql = prefix + expr + suffix;
+ SqlNode sqlNode = getOnlySelectNode(sql);
+
+ final List<SqlIdentifier> sqlIdentifiers = Lists.newArrayList();
+ SqlVisitor sqlVisitor = new SqlBasicVisitor() {
+ @Override
+ public Object visit(SqlIdentifier id) {
+ if (id.names.size() > 1) {
+ throw new IllegalArgumentException("SqlIdentifier " + id + " contains DB/Table name");
+ }
+ sqlIdentifiers.add(id);
+ return null;
+ }
+ };
+
+ sqlNode.accept(sqlVisitor);
+
+ Collections.sort(sqlIdentifiers, new Comparator<SqlIdentifier>() {
+ @Override
+ public int compare(SqlIdentifier o1, SqlIdentifier o2) {
+ int linegap = o2.getParserPosition().getLineNum() - o1.getParserPosition().getLineNum();
+ if (linegap != 0)
+ return linegap;
+
+ return o2.getParserPosition().getColumnNum() - o1.getParserPosition().getColumnNum();
+ }
+ });
+
+ for (SqlIdentifier sqlIdentifier : sqlIdentifiers) {
+ Pair<Integer, Integer> replacePos = getReplacePos(sqlIdentifier, sql);
+ int start = replacePos.getLeft();
+ sql = sql.substring(0, start) + alias + "." + sql.substring(start);
+ }
+
+ return sql.substring(prefix.length(), sql.length() - suffix.length());
+ }
+
+ public static Pair<Integer, Integer> getReplacePos(SqlNode node, String... lines) {
+ SqlParserPos pos = node.getParserPosition();
+ int lineStart = pos.getLineNum();
+ int lineEnd = pos.getEndLineNum();
+ int columnStart = pos.getColumnNum() - 1;
+ int columnEnd = pos.getEndColumnNum();
+ //for the case that sql is multi lines
+ for (int i = 0; i < lineStart - 1; i++) {
+ columnStart += lines[i].length() + 1;
+ }
+ for (int i = 0; i < lineEnd - 1; i++) {
+ columnEnd += lines[i].length() + 1;
+ }
+ return Pair.of(columnStart, columnEnd);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
new file mode 100644
index 0000000..acf7b5a
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.model.tool;
+
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import com.google.common.base.Preconditions;
+
+public class CalciteParserTest {
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testNoTableNameExists() throws SqlParseException {
+ String expr1 = "a + b";
+ Assert.assertEquals("x.a + x.b", CalciteParser.insertAliasInExpr(expr1, "x"));
+
+ String expr2 = "a + year(b)";
+ Assert.assertEquals("x.a + year(x.b)", CalciteParser.insertAliasInExpr(expr2, "x"));
+
+ String expr3 = "a + hiveudf(b)";
+ Assert.assertEquals("x.a + hiveudf(x.b)", CalciteParser.insertAliasInExpr(expr3, "x"));
+ }
+
+ @Test
+ public void testTableNameExists1() throws SqlParseException {
+ String expr1 = "a + x.b";
+
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("SqlIdentifier X.B contains DB/Table name");
+ CalciteParser.insertAliasInExpr(expr1, "x");
+ }
+
+ @Test
+ public void testTableNameExists2() throws SqlParseException {
+ String expr1 = "a + year(x.b)";
+
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("SqlIdentifier X.B contains DB/Table name");
+ CalciteParser.insertAliasInExpr(expr1, "x");
+ }
+
+ @Test
+ public void testTableNameExists3() throws SqlParseException {
+ String expr1 = "a + hiveudf(x.b)";
+
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("SqlIdentifier X.B contains DB/Table name");
+ CalciteParser.insertAliasInExpr(expr1, "x");
+ }
+
+ @Test
+ public void testPos() throws SqlParseException {
+ String[] sqls = new String[] { "select \n a \n + \n b \n from t", //
+ "select\na\n+\nb\nfrom t", //
+ "select \r\n a \r\n + \r\n b \r\n from t", //
+ "select\r\na\r\n+\r\nb\r\nfrom t" };
+
+ for (String sql : sqls) {
+ SqlNode parse = ((SqlSelect) CalciteParser.parse(sql)).getSelectList().get(0);
+ String[] lines = sql.split("\n");
+ Pair<Integer, Integer> replacePos = CalciteParser.getReplacePos(parse, lines);
+ String substring = sql.substring(replacePos.getLeft(), replacePos.getRight());
+ Preconditions.checkArgument(substring.startsWith("a"));
+ Preconditions.checkArgument(substring.endsWith("b"));
+ }
+
+ }
+
+ @Test
+ public void testEqual() throws SqlParseException {
+ String sql0 = "select a.a + a.b + a.c from t as a";
+ String sql1 = "select (((a . a + a.b + a.c))) from t as a";
+ String sql2 = "select (a + b) + c from t";
+ String sql3 = "select a.a + (a.b + a.c) from t as a";
+
+ SqlNode sn0 = CalciteParser.getOnlySelectNode(sql0);
+ SqlNode sn1 = CalciteParser.getOnlySelectNode(sql1);
+ SqlNode sn2 = CalciteParser.getOnlySelectNode(sql2);
+ SqlNode sn3 = CalciteParser.getOnlySelectNode(sql3);
+
+ Assert.assertEquals(true, CalciteParser.isNodeEqual(sn0, sn1));
+ Assert.assertEquals(true, CalciteParser.isNodeEqual(sn0, sn2));
+ Assert.assertEquals(false, CalciteParser.isNodeEqual(sn0, sn3));
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
index d084ce2..5225cf3 100644
--- a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
+++ b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
@@ -125,21 +125,21 @@
{
"tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
"columnName": "DEAL_AMOUNT",
- "expression": "DEFAULT.TEST_KYLIN_FACT.PRICE * DEFAULT.TEST_KYLIN_FACT.ITEM_COUNT",
+ "expression": "PRICE * ITEM_COUNT",
"datatype": "decimal",
"comment": "deal amount of inner join model"
},
{
"tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
"columnName": "DEAL_YEAR",
- "expression": "year(DEFAULT.TEST_KYLIN_FACT.CAL_DT)",
+ "expression": "year(CAL_DT)",
"datatype": "integer",
"comment": "the year of the deal of the inner join model"
},
{
"tableIdentity": "DEFAULT.TEST_ACCOUNT",
"columnName": "COUNTRY_ABBR",
- "expression": "SUBSTR(DEFAULT.TEST_ACCOUNT.ACCOUNT_COUNTRY,0,1)",
+ "expression": "SUBSTR(ACCOUNT_COUNTRY,0,1)",
"datatype": "string",
"comment": "first char of country of the inner join model"
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
index 4a10899..0058425 100644
--- a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
+++ b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
@@ -125,21 +125,21 @@
{
"tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
"columnName": "DEAL_AMOUNT",
- "expression": "DEFAULT.TEST_KYLIN_FACT.PRICE * DEFAULT.TEST_KYLIN_FACT.ITEM_COUNT",
+ "expression": "PRICE * ITEM_COUNT",
"datatype": "decimal",
"comment": "deal amount of left join model"
},
{
"tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
"columnName": "DEAL_YEAR",
- "expression": "year(DEFAULT.TEST_KYLIN_FACT.CAL_DT)",
+ "expression": "year(CAL_DT)",
"datatype": "integer",
"comment": "the year of the deal of the left join model"
},
{
"tableIdentity": "DEFAULT.TEST_ACCOUNT",
"columnName": "COUNTRY_ABBR",
- "expression": "SUBSTR(DEFAULT.TEST_ACCOUNT.ACCOUNT_COUNTRY,0,1)",
+ "expression": "SUBSTR(ACCOUNT_COUNTRY,0,1)",
"datatype": "string",
"comment": "first char of country of the left join model"
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
index d8f1220..7f188c1 100644
--- a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++ b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -23,7 +23,6 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import javax.annotation.Nullable;
@@ -35,15 +34,14 @@ import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
-import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.parser.SqlParseException;
-import org.apache.calcite.sql.parser.SqlParser;
-import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.util.SqlVisitor;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
@@ -65,7 +63,11 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
return sql;
}
ImmutableSortedMap<String, String> computedColumns = getSortedComputedColumnWithProject(project);
- return replaceComputedColumn(sql, computedColumns);
+ String s = replaceComputedColumn(sql, computedColumns);
+ if (!StringUtils.equals(sql, s)) {
+ logger.debug("change sql to " + s);
+ }
+ return s;
}
static String replaceComputedColumn(String inputSql, ImmutableSortedMap<String, String> computedColumn) {
@@ -88,7 +90,7 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
logger.error("Convert to computedColumn Fail,parse sql fail ", e.getMessage());
}
for (SqlNode node : matchedNodes) {
- Pair<Integer, Integer> startEndPos = getReplacePos(lines, node);
+ Pair<Integer, Integer> startEndPos = CalciteParser.getReplacePos(node, lines);
int start = startEndPos.getLeft();
int end = startEndPos.getRight();
//add table alias like t1.column,if exists alias
@@ -104,32 +106,18 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
return result;
}
- private static Pair<Integer, Integer> getReplacePos(String[] lines, SqlNode node) {
- SqlParserPos pos = node.getParserPosition();
- int lineStart = pos.getLineNum();
- int columnStart = pos.getColumnNum() - 1;
- int columnEnd = pos.getEndColumnNum();
- //for the case that sql is multi lines
- for (int i = 0; i < lineStart - 1; i++) {
- int offset = lines[i].length();
- columnStart += offset + 1;
- columnEnd += offset + 1;
- }
- return Pair.of(columnStart, columnEnd);
- }
-
//Return matched node's position and its alias(if exists).If can not find matches, return an empty capacity list
private static List<SqlNode> getMatchedNodes(String inputSql, String ccExp) throws SqlParseException {
if (ccExp == null || ccExp.equals("")) {
return new ArrayList<>();
}
ArrayList<SqlNode> toBeReplacedNodes = new ArrayList<>();
- SqlNode ccNode = getCCExpNode(ccExp);
+ SqlNode ccNode = CalciteParser.getExpNode(ccExp);
List<SqlNode> inputNodes = getInputTreeNodes(inputSql);
// find whether user input sql's tree node equals computed columns's define expression
for (SqlNode inputNode : inputNodes) {
- if (isNodeEqual(inputNode, ccNode)) {
+ if (CalciteParser.isNodeEqual(inputNode, ccNode)) {
toBeReplacedNodes.add(inputNode);
}
}
@@ -138,85 +126,10 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
private static List<SqlNode> getInputTreeNodes(String inputSql) throws SqlParseException {
SqlTreeVisitor stv = new SqlTreeVisitor();
- parse(inputSql).accept(stv);
+ CalciteParser.parse(inputSql).accept(stv);
return stv.getSqlNodes();
}
- private static SqlNode getCCExpNode(String ccExp) throws SqlParseException {
- ccExp = "select " + ccExp + " from t";
- return ((SqlSelect) parse(ccExp)).getSelectList().get(0);
- }
-
- static SqlNode parse(String sql) throws SqlParseException {
- SqlParser.ConfigBuilder parserBuilder = SqlParser.configBuilder();
- SqlParser sqlParser = SqlParser.create(sql, parserBuilder.build());
- return sqlParser.parseQuery();
- }
-
- static boolean isNodeEqual(SqlNode node0, SqlNode node1) {
- if (node0 == null) {
- return node1 == null;
- } else if (node1 == null) {
- return false;
- }
-
- if (!Objects.equals(node0.getClass().getSimpleName(), node1.getClass().getSimpleName())) {
- return false;
- }
-
- if (node0 instanceof SqlCall) {
- SqlCall thisNode = (SqlCall) node0;
- SqlCall thatNode = (SqlCall) node1;
- if (!thisNode.getOperator().getName().equalsIgnoreCase(thatNode.getOperator().getName())) {
- return false;
- }
- return isNodeEqual(thisNode.getOperandList(), thatNode.getOperandList());
- }
- if (node0 instanceof SqlLiteral) {
- SqlLiteral thisNode = (SqlLiteral) node0;
- SqlLiteral thatNode = (SqlLiteral) node1;
- return Objects.equals(thisNode.getValue(), thatNode.getValue());
- }
- if (node0 instanceof SqlNodeList) {
- SqlNodeList thisNode = (SqlNodeList) node0;
- SqlNodeList thatNode = (SqlNodeList) node1;
- if (thisNode.getList().size() != thatNode.getList().size()) {
- return false;
- }
- for (int i = 0; i < thisNode.getList().size(); i++) {
- SqlNode thisChild = thisNode.getList().get(i);
- final SqlNode thatChild = thatNode.getList().get(i);
- if (!isNodeEqual(thisChild, thatChild)) {
- return false;
- }
- }
- return true;
- }
- if (node0 instanceof SqlIdentifier) {
- SqlIdentifier thisNode = (SqlIdentifier) node0;
- SqlIdentifier thatNode = (SqlIdentifier) node1;
- // compare ignore table alias.eg: expression like "a.b + a.c + a.d" ,alias a will be ignored when compared
- String name0 = thisNode.names.get(thisNode.names.size() - 1).replace("\"", "");
- String name1 = thatNode.names.get(thatNode.names.size() - 1).replace("\"", "");
- return name0.equalsIgnoreCase(name1);
- }
-
- logger.error("Convert to computed column fail,failed to compare two nodes,unknown instance type");
- return false;
- }
-
- private static boolean isNodeEqual(List<SqlNode> operands0, List<SqlNode> operands1) {
- if (operands0.size() != operands1.size()) {
- return false;
- }
- for (int i = 0; i < operands0.size(); i++) {
- if (!isNodeEqual(operands0.get(i), operands1.get(i))) {
- return false;
- }
- }
- return true;
- }
-
private static String getTableAlias(SqlNode node) {
if (node instanceof SqlCall) {
SqlCall call = (SqlCall) node;
@@ -350,4 +263,4 @@ class SqlTreeVisitor implements SqlVisitor<SqlNode> {
public SqlNode visit(SqlIntervalQualifier intervalQualifier) {
return null;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d61f867f/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java b/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
index c3efe8d..f9b8d8b 100644
--- a/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
@@ -21,8 +21,6 @@ package org.apache.kylin.query.util;
import java.util.HashMap;
import java.util.Map;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.parser.SqlParseException;
import org.junit.Assert;
import org.junit.Test;
@@ -30,22 +28,6 @@ import org.junit.Test;
import com.google.common.collect.ImmutableSortedMap;
public class ConvertToComputedColumnTest {
- @Test
- public void testEqual() throws SqlParseException {
- String sql0 = "select a.a + a.b + a.c from t as a";
- String sql1 = "select (((a . a + a.b + a.c))) from t as a";
- String sql2 = "select (a + b) + c from t";
- String sql3 = "select a.a + (a.b + a.c) from t as a";
-
- SqlNode sn0 = getSelectNode(sql0);
- SqlNode sn1 = getSelectNode(sql1);
- SqlNode sn2 = getSelectNode(sql2);
- SqlNode sn3 = getSelectNode(sql3);
-
- Assert.assertEquals(true, ConvertToComputedColumn.isNodeEqual(sn0, sn1));
- Assert.assertEquals(true, ConvertToComputedColumn.isNodeEqual(sn0, sn2));
- Assert.assertEquals(false, ConvertToComputedColumn.isNodeEqual(sn0, sn3));
- }
@Test
public void testErrorCase() {
@@ -101,10 +83,6 @@ public class ConvertToComputedColumnTest {
}
- private static SqlNode getSelectNode(String sql) throws SqlParseException {
- return ((SqlSelect) ConvertToComputedColumn.parse(sql)).getSelectList().get(0);
- }
-
@Test
public void testTwoCCHasSameSubExp() {
String sql0 = "select a + b + c from t order by a + b";
[42/50] [abbrv] kylin git commit: Merge commit
'40e4d627227f584d6212ef588eb3fd658bf159dc'
Posted by ka...@apache.org.
Merge commit '40e4d627227f584d6212ef588eb3fd658bf159dc'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/44961356
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/44961356
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/44961356
Branch: refs/heads/KYLIN-2606
Commit: 44961356c1f63fd3d91fb2ded3362219e578c279
Parents: 82b578a 40e4d62
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Jul 3 16:07:49 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Jul 3 16:07:49 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
.../org/apache/kylin/engine/spark/SparkExecutable.java | 7 ++++++-
.../java/org/apache/kylin/source/hive/HiveMRInput.java | 2 ++
.../kylin/source/kafka/hadoop/KafkaFlatTableJob.java | 2 +-
.../cube/v2/coprocessor/endpoint/CubeVisitService.java | 11 ++---------
webapp/app/js/controllers/models.js | 13 ++++++++-----
webapp/app/partials/models/models_tree.html | 4 ++--
7 files changed, 22 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
[25/50] [abbrv] kylin git commit: minor,
remove unused API in QueryUtil
Posted by ka...@apache.org.
minor, remove unused API in QueryUtil
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/40054dd4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/40054dd4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/40054dd4
Branch: refs/heads/KYLIN-2606
Commit: 40054dd4b263a44bfacf809d614fb9a9d5cce77e
Parents: 052d609
Author: lidongsjtu <li...@apache.org>
Authored: Thu Jun 29 21:29:43 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 21:33:04 2017 +0800
----------------------------------------------------------------------
query/src/main/java/org/apache/kylin/query/util/QueryUtil.java | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/40054dd4/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 7794f94..eff9f6d 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -41,11 +41,6 @@ public class QueryUtil {
String transform(String sql, String project);
}
- // for mockup test
- public static String massageSql(String sql) {
- return massageSql(sql, null, 0, 0);
- }
-
public static String massageSql(String sql, String project, int limit, int offset) {
sql = sql.trim();
sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator"));
[45/50] [abbrv] kylin git commit: remove links of system and remove
useless debug log
Posted by ka...@apache.org.
remove links of system and remove useless debug log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0cc68b7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0cc68b7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0cc68b7
Branch: refs/heads/KYLIN-2606
Commit: b0cc68b767098f1b9d7dc714e3129ae1efdfb321
Parents: 4b66751
Author: luguosheng1314 <55...@qq.com>
Authored: Tue Jul 4 16:16:08 2017 +0800
Committer: chenzhx <34...@qq.com>
Committed: Tue Jul 4 22:01:42 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeMeasures.js | 1 -
webapp/app/js/controllers/modelDataModel.js | 1 -
webapp/app/js/controllers/sourceMeta.js | 2 --
webapp/app/partials/admin/admin.html | 16 ++++++++--------
4 files changed, 8 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0cc68b7/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 824e2ca..c598fde 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -114,7 +114,6 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
$scope.nextParameters[i].next_parameter=$scope.nextParameters[i+1];
}
$scope.newMeasure.function.parameter.next_parameter = $scope.nextParameters[0];
- console.log($scope.newMeasure.function.parameter);
}
$scope.editNextParameter = function(parameter){
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0cc68b7/webapp/app/js/controllers/modelDataModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelDataModel.js b/webapp/app/js/controllers/modelDataModel.js
index 15fb20a..c895218 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -218,7 +218,6 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
lookupList.splice(lookupList.indexOf(lookup), 1);
$scope.aliasName.splice($scope.aliasName.indexOf(lookup.alias),1);
}
- console.log($scope.aliasName);
};
$scope.changeKey = function(index){
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0cc68b7/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
index c2122ad..4f341f0 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -644,12 +644,10 @@ KylinApp
$scope.streamingOnChange = function () {
$scope.table.schemaChecked = true;
- console.log($scope.streaming.sourceSchema);
try {
$scope.streaming.parseResult = JSON.parse($scope.streaming.sourceSchema);
} catch (error) {
$scope.table.sourceValid = false;
- console.log(error);
return;
}
$scope.table.sourceValid = true;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0cc68b7/webapp/app/partials/admin/admin.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/admin/admin.html b/webapp/app/partials/admin/admin.html
index b4bca8d..be0156a 100644
--- a/webapp/app/partials/admin/admin.html
+++ b/webapp/app/partials/admin/admin.html
@@ -64,14 +64,14 @@
<a ng-click="downloadBadQueryFiles();" tooltip="Download Diagnosis Info For Current Project" class="btn btn-primary btn-lg btn-block"><i class="fa fa-ambulance"></i> Diagnosis</a>
</div>
- <div>
- <h3>Links</h3>
- </div>
- <div style="padding-top: 10px" ng-if="config.reference_links['hadoop']">
- <a class="label-lg label-yellow arrowed-right"style="font-size:18px;" tooltip="Cluster Resource Monitoring" href="{{config.reference_links['hadoop'].link}}" >
- Hadoop Monitor
- </a>
- </div>
+ <!--<div>-->
+ <!--<h3>Links</h3>-->
+ <!--</div>-->
+ <!--<div style="padding-top: 10px" ng-if="config.reference_links['hadoop']">-->
+ <!--<a class="label-lg label-yellow arrowed-right"style="font-size:18px;" tooltip="Cluster Resource Monitoring" href="{{config.reference_links['hadoop'].link}}" >-->
+ <!--Hadoop Monitor-->
+ <!--</a>-->
+ <!--</div>-->
</div>
</div>
[50/50] [abbrv] kylin git commit: KYLIN-2606 Only return counter for
precise count_distinct if query is exactAggregate
Posted by ka...@apache.org.
KYLIN-2606 Only return counter for precise count_distinct if query is exactAggregate
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1f40225d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1f40225d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1f40225d
Branch: refs/heads/KYLIN-2606
Commit: 1f40225d1a80ece1babf107b3ef5cc7de8baacb7
Parents: d509542
Author: kangkaisen <ka...@163.com>
Authored: Wed Feb 15 19:53:17 2017 +0800
Committer: 康凯森 <ka...@meituan.com>
Committed: Fri Jul 7 13:47:12 2017 +0800
----------------------------------------------------------------------
.../kylin/cube/gridtable/CubeCodeSystem.java | 4 +
.../org/apache/kylin/gridtable/GTRecord.java | 8 +
.../kylin/gridtable/GTSampleCodeSystem.java | 4 +
.../apache/kylin/gridtable/IGTCodeSystem.java | 3 +
.../measure/bitmap/BitmapCounterFactory.java | 2 +
.../kylin/measure/bitmap/BitmapSerializer.java | 37 ++++-
.../measure/bitmap/RoaringBitmapCounter.java | 10 ++
.../bitmap/RoaringBitmapCounterFactory.java | 5 +
.../metadata/datatype/DataTypeSerializer.java | 13 ++
.../gtrecord/GTCubeStorageQueryBase.java | 53 ++++++-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 1 +
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 2 +-
.../hbase/cube/v2/HBaseReadonlyStore.java | 33 +++-
.../coprocessor/endpoint/CubeVisitService.java | 2 +-
.../endpoint/generated/CubeVisitProtos.java | 151 +++++++++++++++----
.../endpoint/protobuf/CubeVisit.proto | 1 +
16 files changed, 291 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index aaa12a7..9eae6f3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -177,4 +177,8 @@ public class CubeCodeSystem implements IGTCodeSystem {
return result;
}
+ @Override
+ public DataTypeSerializer<?> getSerializer(int col) {
+ return serializers[col];
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 3e62ea7..f65e4b5 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -296,4 +296,12 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable {
}
}
+ /** change pointers to point to data in given buffer, this
+ * method allows to defined specific column to load */
+ public void loadColumns(int selectedCol, ByteBuffer buf) {
+ int pos = buf.position();
+ int len = info.codeSystem.codeLength(selectedCol, buf);
+ cols[selectedCol].set(buf.array(), buf.arrayOffset() + pos, len);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index 3f3c844..2a5e791 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -118,4 +118,8 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
}
};
+ @Override
+ public DataTypeSerializer<?> getSerializer(int col) {
+ return serializers[col];
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
index 89dfc99..9c8ad6b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
public interface IGTCodeSystem {
@@ -62,4 +63,6 @@ public interface IGTCodeSystem {
/** Return aggregators for metrics */
MeasureAggregator<?>[] newMetricsAggregators(ImmutableBitSet columns, String[] aggrFunctions);
+ /** Return specific DataTypeSerializer */
+ DataTypeSerializer<?> getSerializer(int col);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounterFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounterFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounterFactory.java
index da7748e..39aa1be 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounterFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounterFactory.java
@@ -26,5 +26,7 @@ public interface BitmapCounterFactory {
BitmapCounter newBitmap(int... values);
+ BitmapCounter newBitmap(long counter);
+
BitmapCounter newBitmap(ByteBuffer in) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
index c1b260d..e2ec4cc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
@@ -28,6 +28,9 @@ public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
private static final BitmapCounterFactory factory = RoaringBitmapCounterFactory.INSTANCE;
private static final BitmapCounter DELEGATE = factory.newBitmap();
+ private static final int IS_RESULT_FLAG = 1;
+ private static final int RESULT_SIZE = 12;
+
// called by reflection
public BitmapSerializer(DataType type) {
}
@@ -44,8 +47,12 @@ public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
@Override
public BitmapCounter deserialize(ByteBuffer in) {
try {
- return factory.newBitmap(in);
-
+ if (peekLength(in) == RESULT_SIZE) {
+ int flag = in.getInt();
+ return factory.newBitmap(in.getLong());
+ } else {
+ return factory.newBitmap(in);
+ }
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -53,7 +60,12 @@ public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
@Override
public int peekLength(ByteBuffer in) {
- return DELEGATE.peekLength(in);
+ ByteBuffer buffer = in.slice();
+ if (buffer.getInt(0) == IS_RESULT_FLAG) {
+ return RESULT_SIZE;
+ } else {
+ return DELEGATE.peekLength(in);
+ }
}
@Override
@@ -71,4 +83,23 @@ public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
// It's difficult to decide the size before data was ingested, comparing with HLLCounter(16) as 64KB, here is assumption
return 8 * 1024;
}
+
+ @Override
+ public boolean supportDirectReturnResult() {
+ return true;
+ }
+
+ @Override
+ public ByteBuffer getFinalResult(ByteBuffer in) {
+ ByteBuffer out = ByteBuffer.allocate(RESULT_SIZE);
+ try {
+ BitmapCounter counter = factory.newBitmap(in);
+ out.putInt(IS_RESULT_FLAG);
+ out.putLong(counter.getCount());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ out.flip();
+ return out;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
index eec45f2..9929e24 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -35,6 +35,7 @@ import java.util.Iterator;
public class RoaringBitmapCounter implements BitmapCounter, Serializable {
private ImmutableRoaringBitmap bitmap;
+ private Long counter;
RoaringBitmapCounter() {
bitmap = new MutableRoaringBitmap();
@@ -44,6 +45,11 @@ public class RoaringBitmapCounter implements BitmapCounter, Serializable {
this.bitmap = bitmap;
}
+ RoaringBitmapCounter(long counter) {
+ this.counter = counter;
+ }
+
+
private MutableRoaringBitmap getMutableBitmap() {
if (bitmap instanceof MutableRoaringBitmap) {
return (MutableRoaringBitmap) bitmap;
@@ -86,6 +92,10 @@ public class RoaringBitmapCounter implements BitmapCounter, Serializable {
@Override
public long getCount() {
+ if (counter != null) {
+ return counter;
+ }
+
return bitmap.getCardinality();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
index 822afa2..8ab908a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
@@ -40,6 +40,11 @@ public class RoaringBitmapCounterFactory implements BitmapCounterFactory, Serial
}
@Override
+ public BitmapCounter newBitmap(long counter) {
+ return new RoaringBitmapCounter(counter);
+ }
+
+ @Override
public BitmapCounter newBitmap(ByteBuffer in) throws IOException {
RoaringBitmapCounter counter = new RoaringBitmapCounter();
counter.readFields(in);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
index a4a35a4..2de38c0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
@@ -89,6 +89,19 @@ abstract public class DataTypeSerializer<T> implements BytesSerializer<T>, java.
throw new UnsupportedOperationException();
}
+ /** If the query is exactAggregation and has some memory hungry measures,
+ * we could directly return final result to speed up the query.
+ * If the DataTypeSerializer support this,
+ * which should override the getFinalResult method, besides that, the deserialize and peekLength method should also support it, like {@link org.apache.kylin.measure.bitmap.BitmapSerializer} */
+ public boolean supportDirectReturnResult() {
+ return false;
+ }
+
+ /** An optional method that converts a expensive buffer to lightweight buffer containing final result (for memory hungry measures) */
+ public ByteBuffer getFinalResult(ByteBuffer in) {
+ throw new UnsupportedOperationException();
+ }
+
/** Convert from obj to string */
public String toString(T value) {
if (value == null)
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index f89fc47..a21446a 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -44,6 +44,7 @@ import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TblColRef;
@@ -135,6 +136,10 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
Set<TblColRef> singleValuesD = findSingleValueColumns(filter);
context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, groupsD, singleValuesD));
+ // exactAggregation mean: needn't aggregation at storage and query engine both.
+ boolean exactAggregation = isExactAggregation(context, cuboid, groups, otherDimsD, singleValuesD, derivedPostAggregation, sqlDigest.aggregations);
+ context.setExactAggregation(exactAggregation);
+
// replace derived columns in filter with host columns; columns on loosened condition must be added to group by
Set<TblColRef> loosenedColumnD = Sets.newHashSet();
Set<TblColRef> filterColumnD = Sets.newHashSet();
@@ -428,7 +433,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest);
}
}
-
+
private TupleFilter checkHavingCanPushDown(TupleFilter havingFilter, Set<TblColRef> groupsD, List<FunctionDesc> aggregations, Set<FunctionDesc> metrics) {
// must have only one segment
Segments<CubeSegment> readySegs = cubeInstance.getSegments(SegmentStatusEnum.READY);
@@ -465,4 +470,50 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
return havingFilter;
}
+ private boolean isExactAggregation(StorageContext context, Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation, Collection<FunctionDesc> functionDescs) {
+ if (context.isNeedStorageAggregation()) {
+ logger.info("exactAggregation is false because need storage aggregation");
+ return false;
+ }
+
+ if (cuboid.requirePostAggregation()) {
+ logger.info("exactAggregation is false because cuboid " + cuboid.getInputID() + "=> " + cuboid.getId());
+ return false;
+ }
+
+ // derived aggregation is bad, unless expanded columns are already in group by
+ if (groups.containsAll(derivedPostAggregation) == false) {
+ logger.info("exactAggregation is false because derived column require post aggregation: " + derivedPostAggregation);
+ return false;
+ }
+
+ // other columns (from filter) is bad, unless they are ensured to have single value
+ if (singleValuesD.containsAll(othersD) == false) {
+ logger.info("exactAggregation is false because some column not on group by: " + othersD //
+ + " (single value column: " + singleValuesD + ")");
+ return false;
+ }
+
+ //for DimensionAsMetric like max(cal_dt), the dimension column maybe not in real group by
+ for (FunctionDesc functionDesc : functionDescs) {
+ if (functionDesc.isDimensionAsMetric()) {
+ logger.info("exactAggregation is false because has DimensionAsMetric");
+ return false;
+ }
+ }
+
+ // for partitioned cube, the partition column must belong to group by or has single value
+ PartitionDesc partDesc = cuboid.getCubeDesc().getModel().getPartitionDesc();
+ if (partDesc.isPartitioned()) {
+ TblColRef col = partDesc.getPartitionDateColumnRef();
+ if (!groups.contains(col) && !singleValuesD.contains(col)) {
+ logger.info("exactAggregation is false because cube is partitioned and " + col + " is not on group by");
+ return false;
+ }
+ }
+
+ logger.info("exactAggregation is true, cuboid id is " + cuboid.getId());
+ return true;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index e822ada..af8754d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -163,6 +163,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
builder.setSpillEnabled(cubeSeg.getConfig().getQueryCoprocessorSpillEnabled());
builder.setMaxScanBytes(cubeSeg.getConfig().getPartitionMaxScanBytes());
+ builder.setIsExactAggregate(storageContext.isExactAggregation());
for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
executorService.submit(new Runnable() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 951e2ef..a8f4fd8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -193,7 +193,7 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
};
- IGTStore store = new HBaseReadonlyStore(cellListIterator, scanRequest, rawScans.get(0).hbaseColumns, hbaseColumnsToGT, cubeSeg.getRowKeyPreambleSize(), false);
+ IGTStore store = new HBaseReadonlyStore(cellListIterator, scanRequest, rawScans.get(0).hbaseColumns, hbaseColumnsToGT, cubeSeg.getRowKeyPreambleSize(), false, storageContext.isExactAggregation());
IGTScanner rawScanner = store.scan(scanRequest);
final IGTScanner decorateScanner = scanRequest.decorateScanner(rawScanner);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
index 631e8e8..4ec0c9d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
@@ -20,6 +20,7 @@ package org.apache.kylin.storage.hbase.cube.v2;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -44,18 +45,19 @@ public class HBaseReadonlyStore implements IGTStore {
private List<List<Integer>> hbaseColumnsToGT;
private int rowkeyPreambleSize;
private boolean withDelay = false;
-
+ private boolean isExactAggregation;
/**
* @param withDelay is for test use
*/
- public HBaseReadonlyStore(CellListIterator cellListIterator, GTScanRequest gtScanRequest, List<Pair<byte[], byte[]>> hbaseColumns, List<List<Integer>> hbaseColumnsToGT, int rowkeyPreambleSize, boolean withDelay) {
+ public HBaseReadonlyStore(CellListIterator cellListIterator, GTScanRequest gtScanRequest, List<Pair<byte[], byte[]>> hbaseColumns, List<List<Integer>> hbaseColumnsToGT, int rowkeyPreambleSize, boolean withDelay, boolean isExactAggregation) {
this.cellListIterator = cellListIterator;
this.info = gtScanRequest.getInfo();
this.hbaseColumns = hbaseColumns;
this.hbaseColumnsToGT = hbaseColumnsToGT;
this.rowkeyPreambleSize = rowkeyPreambleSize;
this.withDelay = withDelay;
+ this.isExactAggregation = isExactAggregation;
}
@Override
@@ -132,6 +134,12 @@ public class HBaseReadonlyStore implements IGTStore {
buf = byteBuffer(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
oneRecord.loadColumns(hbaseColumnsToGT.get(i), buf);
}
+
+
+ if (isExactAggregation && getDirectReturnResultColumns().size() > 0) {
+ trimGTRecord(oneRecord);
+ }
+
return oneRecord;
}
@@ -145,6 +153,27 @@ public class HBaseReadonlyStore implements IGTStore {
return ByteBuffer.wrap(array, offset, length);
}
+ private List<Integer> getDirectReturnResultColumns() {
+ List<Integer> columns = new ArrayList<>();
+ for (int i = 0; i < info.getColumnCount(); i++) {
+ if (info.getCodeSystem().getSerializer(i).supportDirectReturnResult()) {
+ columns.add(i);
+ }
+ }
+ return columns;
+ }
+
+ private void trimGTRecord(GTRecord record) {
+ List<Integer> directReturnResultColumns = getDirectReturnResultColumns();
+ for (Integer i : directReturnResultColumns) {
+ ByteBuffer recordBuffer = record.get(i).asBuffer();
+ if (recordBuffer!= null) {
+ ByteBuffer trimmedBuffer = info.getCodeSystem().getSerializer(i).getFinalResult(recordBuffer);
+ record.loadColumns(i, trimmedBuffer);
+ }
+ }
+ }
+
};
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 3b26b71..bb6555c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -295,7 +295,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
!request.hasMaxScanBytes() ? Long.MAX_VALUE : request.getMaxScanBytes(), // for new client
scanReq.getTimeout());
- IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, request.getRowkeyPreambleSize(), behavior.delayToggledOn());
+ IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, request.getRowkeyPreambleSize(), behavior.delayToggledOn(), request.getIsExactAggregate());
IGTScanner rawScanner = store.scan(scanReq);
IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, behavior.filterToggledOn(), behavior.aggrToggledOn(), false, request.getSpillEnabled());
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index b9f2771..4c662c9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -123,7 +123,7 @@ public final class CubeVisitProtos {
* <code>optional int64 maxScanBytes = 8;</code>
*
* <pre>
- * 0 means no limit
+ * must be positive
* </pre>
*/
boolean hasMaxScanBytes();
@@ -131,10 +131,20 @@ public final class CubeVisitProtos {
* <code>optional int64 maxScanBytes = 8;</code>
*
* <pre>
- * 0 means no limit
+ * must be positive
* </pre>
*/
long getMaxScanBytes();
+
+ // optional bool isExactAggregate = 9 [default = false];
+ /**
+ * <code>optional bool isExactAggregate = 9 [default = false];</code>
+ */
+ boolean hasIsExactAggregate();
+ /**
+ * <code>optional bool isExactAggregate = 9 [default = false];</code>
+ */
+ boolean getIsExactAggregate();
}
/**
* Protobuf type {@code CubeVisitRequest}
@@ -230,6 +240,11 @@ public final class CubeVisitProtos {
maxScanBytes_ = input.readInt64();
break;
}
+ case 72: {
+ bitField0_ |= 0x00000080;
+ isExactAggregate_ = input.readBool();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -982,7 +997,7 @@ public final class CubeVisitProtos {
* <code>optional int64 maxScanBytes = 8;</code>
*
* <pre>
- * 0 means no limit
+ * must be positive
* </pre>
*/
public boolean hasMaxScanBytes() {
@@ -992,13 +1007,29 @@ public final class CubeVisitProtos {
* <code>optional int64 maxScanBytes = 8;</code>
*
* <pre>
- * 0 means no limit
+ * must be positive
* </pre>
*/
public long getMaxScanBytes() {
return maxScanBytes_;
}
+ // optional bool isExactAggregate = 9 [default = false];
+ public static final int ISEXACTAGGREGATE_FIELD_NUMBER = 9;
+ private boolean isExactAggregate_;
+ /**
+ * <code>optional bool isExactAggregate = 9 [default = false];</code>
+ */
+ public boolean hasIsExactAggregate() {
+ return ((bitField0_ & 0x00000080) == 0x00000080);
+ }
+ /**
+ * <code>optional bool isExactAggregate = 9 [default = false];</code>
+ */
+ public boolean getIsExactAggregate() {
+ return isExactAggregate_;
+ }
+
private void initFields() {
gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
@@ -1008,6 +1039,7 @@ public final class CubeVisitProtos {
queryId_ = "";
spillEnabled_ = true;
maxScanBytes_ = 0L;
+ isExactAggregate_ = false;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -1061,6 +1093,9 @@ public final class CubeVisitProtos {
if (((bitField0_ & 0x00000040) == 0x00000040)) {
output.writeInt64(8, maxScanBytes_);
}
+ if (((bitField0_ & 0x00000080) == 0x00000080)) {
+ output.writeBool(9, isExactAggregate_);
+ }
getUnknownFields().writeTo(output);
}
@@ -1102,6 +1137,10 @@ public final class CubeVisitProtos {
size += com.google.protobuf.CodedOutputStream
.computeInt64Size(8, maxScanBytes_);
}
+ if (((bitField0_ & 0x00000080) == 0x00000080)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(9, isExactAggregate_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -1162,6 +1201,11 @@ public final class CubeVisitProtos {
result = result && (getMaxScanBytes()
== other.getMaxScanBytes());
}
+ result = result && (hasIsExactAggregate() == other.hasIsExactAggregate());
+ if (hasIsExactAggregate()) {
+ result = result && (getIsExactAggregate()
+ == other.getIsExactAggregate());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -1207,6 +1251,10 @@ public final class CubeVisitProtos {
hash = (37 * hash) + MAXSCANBYTES_FIELD_NUMBER;
hash = (53 * hash) + hashLong(getMaxScanBytes());
}
+ if (hasIsExactAggregate()) {
+ hash = (37 * hash) + ISEXACTAGGREGATE_FIELD_NUMBER;
+ hash = (53 * hash) + hashBoolean(getIsExactAggregate());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -1337,6 +1385,8 @@ public final class CubeVisitProtos {
bitField0_ = (bitField0_ & ~0x00000040);
maxScanBytes_ = 0L;
bitField0_ = (bitField0_ & ~0x00000080);
+ isExactAggregate_ = false;
+ bitField0_ = (bitField0_ & ~0x00000100);
return this;
}
@@ -1402,6 +1452,10 @@ public final class CubeVisitProtos {
to_bitField0_ |= 0x00000040;
}
result.maxScanBytes_ = maxScanBytes_;
+ if (((from_bitField0_ & 0x00000100) == 0x00000100)) {
+ to_bitField0_ |= 0x00000080;
+ }
+ result.isExactAggregate_ = isExactAggregate_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -1469,6 +1523,9 @@ public final class CubeVisitProtos {
if (other.hasMaxScanBytes()) {
setMaxScanBytes(other.getMaxScanBytes());
}
+ if (other.hasIsExactAggregate()) {
+ setIsExactAggregate(other.getIsExactAggregate());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -2068,7 +2125,7 @@ public final class CubeVisitProtos {
* <code>optional int64 maxScanBytes = 8;</code>
*
* <pre>
- * 0 means no limit
+ * must be positive
* </pre>
*/
public boolean hasMaxScanBytes() {
@@ -2078,7 +2135,7 @@ public final class CubeVisitProtos {
* <code>optional int64 maxScanBytes = 8;</code>
*
* <pre>
- * 0 means no limit
+ * must be positive
* </pre>
*/
public long getMaxScanBytes() {
@@ -2088,7 +2145,7 @@ public final class CubeVisitProtos {
* <code>optional int64 maxScanBytes = 8;</code>
*
* <pre>
- * 0 means no limit
+ * must be positive
* </pre>
*/
public Builder setMaxScanBytes(long value) {
@@ -2101,7 +2158,7 @@ public final class CubeVisitProtos {
* <code>optional int64 maxScanBytes = 8;</code>
*
* <pre>
- * 0 means no limit
+ * must be positive
* </pre>
*/
public Builder clearMaxScanBytes() {
@@ -2111,6 +2168,39 @@ public final class CubeVisitProtos {
return this;
}
+ // optional bool isExactAggregate = 9 [default = false];
+ private boolean isExactAggregate_ ;
+ /**
+ * <code>optional bool isExactAggregate = 9 [default = false];</code>
+ */
+ public boolean hasIsExactAggregate() {
+ return ((bitField0_ & 0x00000100) == 0x00000100);
+ }
+ /**
+ * <code>optional bool isExactAggregate = 9 [default = false];</code>
+ */
+ public boolean getIsExactAggregate() {
+ return isExactAggregate_;
+ }
+ /**
+ * <code>optional bool isExactAggregate = 9 [default = false];</code>
+ */
+ public Builder setIsExactAggregate(boolean value) {
+ bitField0_ |= 0x00000100;
+ isExactAggregate_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bool isExactAggregate = 9 [default = false];</code>
+ */
+ public Builder clearIsExactAggregate() {
+ bitField0_ = (bitField0_ & ~0x00000100);
+ isExactAggregate_ = false;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:CubeVisitRequest)
}
@@ -5516,32 +5606,33 @@ public final class CubeVisitProtos {
java.lang.String[] descriptorData = {
"\npstorage-hbase/src/main/java/org/apache" +
"/kylin/storage/hbase/cube/v2/coprocessor" +
- "/endpoint/protobuf/CubeVisit.proto\"\205\002\n\020C" +
+ "/endpoint/protobuf/CubeVisit.proto\"\246\002\n\020C" +
"ubeVisitRequest\022\025\n\rgtScanRequest\030\001 \002(\014\022\024" +
"\n\014hbaseRawScan\030\002 \002(\014\022\032\n\022rowkeyPreambleSi" +
"ze\030\003 \002(\005\0223\n\020hbaseColumnsToGT\030\004 \003(\0132\031.Cub" +
"eVisitRequest.IntList\022\027\n\017kylinProperties" +
"\030\005 \002(\t\022\017\n\007queryId\030\006 \001(\t\022\032\n\014spillEnabled\030" +
- "\007 \001(\010:\004true\022\024\n\014maxScanBytes\030\010 \001(\003\032\027\n\007Int" +
- "List\022\014\n\004ints\030\001 \003(\005\"\253\004\n\021CubeVisitResponse",
- "\022\026\n\016compressedRows\030\001 \002(\014\022\'\n\005stats\030\002 \002(\0132" +
- "\030.CubeVisitResponse.Stats\022/\n\terrorInfo\030\003" +
- " \001(\0132\034.CubeVisitResponse.ErrorInfo\032\220\002\n\005S" +
- "tats\022\030\n\020serviceStartTime\030\001 \001(\003\022\026\n\016servic" +
- "eEndTime\030\002 \001(\003\022\027\n\017scannedRowCount\030\003 \001(\003\022" +
- "\032\n\022aggregatedRowCount\030\004 \001(\003\022\025\n\rsystemCpu" +
- "Load\030\005 \001(\001\022\036\n\026freePhysicalMemorySize\030\006 \001" +
- "(\001\022\031\n\021freeSwapSpaceSize\030\007 \001(\001\022\020\n\010hostnam" +
- "e\030\010 \001(\t\022\016\n\006etcMsg\030\t \001(\t\022\026\n\016normalComplet" +
- "e\030\n \001(\005\022\024\n\014scannedBytes\030\013 \001(\003\032H\n\tErrorIn",
- "fo\022*\n\004type\030\001 \002(\0162\034.CubeVisitResponse.Err" +
- "orType\022\017\n\007message\030\002 \002(\t\"G\n\tErrorType\022\020\n\014" +
- "UNKNOWN_TYPE\020\000\022\013\n\007TIMEOUT\020\001\022\033\n\027RESOURCE_" +
- "LIMIT_EXCEEDED\020\0022F\n\020CubeVisitService\0222\n\t" +
- "visitCube\022\021.CubeVisitRequest\032\022.CubeVisit" +
- "ResponseB`\nEorg.apache.kylin.storage.hba" +
- "se.cube.v2.coprocessor.endpoint.generate" +
- "dB\017CubeVisitProtosH\001\210\001\001\240\001\001"
+ "\007 \001(\010:\004true\022\024\n\014maxScanBytes\030\010 \001(\003\022\037\n\020isE" +
+ "xactAggregate\030\t \001(\010:\005false\032\027\n\007IntList\022\014\n",
+ "\004ints\030\001 \003(\005\"\253\004\n\021CubeVisitResponse\022\026\n\016com" +
+ "pressedRows\030\001 \002(\014\022\'\n\005stats\030\002 \002(\0132\030.CubeV" +
+ "isitResponse.Stats\022/\n\terrorInfo\030\003 \001(\0132\034." +
+ "CubeVisitResponse.ErrorInfo\032\220\002\n\005Stats\022\030\n" +
+ "\020serviceStartTime\030\001 \001(\003\022\026\n\016serviceEndTim" +
+ "e\030\002 \001(\003\022\027\n\017scannedRowCount\030\003 \001(\003\022\032\n\022aggr" +
+ "egatedRowCount\030\004 \001(\003\022\025\n\rsystemCpuLoad\030\005 " +
+ "\001(\001\022\036\n\026freePhysicalMemorySize\030\006 \001(\001\022\031\n\021f" +
+ "reeSwapSpaceSize\030\007 \001(\001\022\020\n\010hostname\030\010 \001(\t" +
+ "\022\016\n\006etcMsg\030\t \001(\t\022\026\n\016normalComplete\030\n \001(\005",
+ "\022\024\n\014scannedBytes\030\013 \001(\003\032H\n\tErrorInfo\022*\n\004t" +
+ "ype\030\001 \002(\0162\034.CubeVisitResponse.ErrorType\022" +
+ "\017\n\007message\030\002 \002(\t\"G\n\tErrorType\022\020\n\014UNKNOWN" +
+ "_TYPE\020\000\022\013\n\007TIMEOUT\020\001\022\033\n\027RESOURCE_LIMIT_E" +
+ "XCEEDED\020\0022F\n\020CubeVisitService\0222\n\tvisitCu" +
+ "be\022\021.CubeVisitRequest\032\022.CubeVisitRespons" +
+ "eB`\nEorg.apache.kylin.storage.hbase.cube" +
+ ".v2.coprocessor.endpoint.generatedB\017Cube" +
+ "VisitProtosH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -5553,7 +5644,7 @@ public final class CubeVisitProtos {
internal_static_CubeVisitRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_CubeVisitRequest_descriptor,
- new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "RowkeyPreambleSize", "HbaseColumnsToGT", "KylinProperties", "QueryId", "SpillEnabled", "MaxScanBytes", });
+ new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "RowkeyPreambleSize", "HbaseColumnsToGT", "KylinProperties", "QueryId", "SpillEnabled", "MaxScanBytes", "IsExactAggregate", });
internal_static_CubeVisitRequest_IntList_descriptor =
internal_static_CubeVisitRequest_descriptor.getNestedTypes().get(0);
internal_static_CubeVisitRequest_IntList_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f40225d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
index aa83595..8ca8756 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -38,6 +38,7 @@ message CubeVisitRequest {
optional string queryId = 6;
optional bool spillEnabled = 7 [default = true];
optional int64 maxScanBytes = 8; // must be positive
+ optional bool isExactAggregate = 9 [default = false];
message IntList {
repeated int32 ints = 1;
}
[06/50] [abbrv] kylin git commit: Fix for KYLIN-2668. Added Regex to accept after ‘jdbc:kylin:’ up to the ‘//‘. and parsed out as properties.
Posted by ka...@apache.org.
Fix for KYLIN-2668. Added Regex to accept after ‘jdbc:kylin:’ up to the ‘//‘. and parsed out as properties.
Added test to test the url, project and ssl is enabled.
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f084601
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f084601
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f084601
Branch: refs/heads/KYLIN-2606
Commit: 2f084601b938d6051c0fe02a9fc075be549547f0
Parents: 2bc90f9
Author: Joe Swingle <jo...@hobsons.com>
Authored: Tue Jun 13 10:56:22 2017 -0400
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jun 29 10:39:58 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/jdbc/KylinConnection.java | 4 ++--
jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java | 10 ++++++++++
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f084601/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
index 6852998..6fb228a 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
@@ -51,9 +51,9 @@ public class KylinConnection extends AvaticaConnection {
super(driver, factory, url, info);
String odbcUrl = url;
- odbcUrl = odbcUrl.replace(Driver.CONNECT_STRING_PREFIX + "//", "");
+ odbcUrl = odbcUrl.replaceAll((Driver.CONNECT_STRING_PREFIX + "[[A-Za-z0-9]*=[A-Za-z0-9]*;]*//").toString(), "");
+
String[] temps = odbcUrl.split("/");
-
assert temps.length == 2;
this.baseUrl = temps[0];
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f084601/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
index 0887634..4fd9d33 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.jdbc;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -188,6 +189,15 @@ public class DriverTest {
state.close();
conn.close();
}
+
+ @Test
+ public void testSSLFromURL() throws SQLException{
+ Driver driver = new DummyDriver();
+ Connection conn = driver.connect("jdbc:kylin:ssl=True;//test_url/test_db", null);
+ assertEquals("test_url", ((KylinConnection)conn).getBaseUrl());
+ assertEquals("test_db", ((KylinConnection)conn).getProject());
+ assertTrue(Boolean.parseBoolean( (String)((KylinConnection)conn).getConnectionProperties().get("ssl")));
+ }
private void printResultSet(ResultSet rs) throws SQLException {
ResultSetMetaData meta = rs.getMetaData();
[07/50] [abbrv] kylin git commit: Merge commit
'2f084601b938d6051c0fe02a9fc075be549547f0' into mergekylin
Posted by ka...@apache.org.
Merge commit '2f084601b938d6051c0fe02a9fc075be549547f0' into mergekylin
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/69532cd5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/69532cd5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/69532cd5
Branch: refs/heads/KYLIN-2606
Commit: 69532cd5ff3dc7da9a4670795f1230faa489c389
Parents: a0b7e74 2f08460
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 29 13:43:13 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Jun 29 13:43:13 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 118 +++----
.../apache/kylin/common/KylinConfigBase.java | 28 ++
.../java/org/apache/kylin/cube/CubeManager.java | 21 +-
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 10 +-
.../InMemCubeBuilderInputConverter.java | 5 +
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
.../apache/kylin/cube/model/RowKeyColDesc.java | 9 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 52 +++-
.../kylin/metadata/model/ISourceAware.java | 1 +
.../kylin/source/ISampleDataDeployer.java | 47 +++
.../java/org/apache/kylin/source/ISource.java | 6 +
.../source/datagen/ModelDataGenerator.java | 7 +-
.../engine/spark/KylinKryoRegistrator.java | 1 +
.../org/apache/kylin/jdbc/KylinConnection.java | 4 +-
.../java/org/apache/kylin/jdbc/DriverTest.java | 10 +
.../kylin/provision/BuildCubeWithEngine.java | 14 +-
.../kylin/source/hive/BeelineHiveClient.java | 2 +-
.../apache/kylin/source/hive/DBConnConf.java | 86 ++++++
.../kylin/source/hive/HiveClientFactory.java | 1 +
.../apache/kylin/source/hive/HiveMRInput.java | 41 ++-
.../kylin/source/hive/HiveMetadataExplorer.java | 78 ++++-
.../apache/kylin/source/hive/HiveSource.java | 17 ++
.../apache/kylin/source/hive/IHiveClient.java | 7 +-
.../org/apache/kylin/source/jdbc/CmdStep.java | 69 +++++
.../apache/kylin/source/jdbc/HiveCmdStep.java | 77 +++++
.../apache/kylin/source/jdbc/JdbcExplorer.java | 305 +++++++++++++++++++
.../kylin/source/jdbc/JdbcHiveMRInput.java | 93 ++++++
.../apache/kylin/source/jdbc/JdbcSource.java | 66 ++++
.../org/apache/kylin/source/jdbc/JdbcTable.java | 67 ++++
.../kylin/source/jdbc/JdbcTableReader.java | 107 +++++++
.../org/apache/kylin/source/jdbc/SqlUtil.java | 107 +++++++
.../apache/kylin/source/kafka/KafkaSource.java | 8 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 7 +-
.../apache/kylin/tool/KylinConfigCLITest.java | 12 +-
webapp/app/js/controllers/modelEdit.js | 5 +-
webapp/app/js/controllers/modelMeasures.js | 33 +-
webapp/app/js/controllers/page.js | 5 +
webapp/app/js/controllers/projects.js | 38 +--
webapp/app/js/controllers/sourceMeta.js | 9 +
webapp/app/js/filters/filter.js | 17 ++
.../cubeDesigner/advanced_settings.html | 2 +-
webapp/app/partials/cubeDesigner/measures.html | 2 +-
webapp/app/partials/cubes/cubes.html | 10 +-
.../app/partials/modelDesigner/data_model.html | 2 +-
.../partials/modelDesigner/model_measures.html | 2 +-
webapp/app/partials/models/model_json_edit.html | 44 +++
webapp/app/partials/models/model_schema.html | 4 +-
webapp/app/partials/models/models_tree.html | 2 +
.../app/partials/projects/project_detail.html | 29 ++
webapp/app/routes.json | 8 +
50 files changed, 1499 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/69532cd5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/69532cd5/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/69532cd5/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
[24/50] [abbrv] kylin git commit: KYLIN-2674 Should not catch
OutOfMemoryError in coprocessor
Posted by ka...@apache.org.
KYLIN-2674 Should not catch OutOfMemoryError in coprocessor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1eb32c3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1eb32c3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1eb32c3e
Branch: refs/heads/KYLIN-2606
Commit: 1eb32c3e62d92142b4f7330ac10edd606c870307
Parents: a1e055a
Author: kangkaisen <ka...@live.com>
Authored: Mon Jun 12 12:17:20 2017 +0800
Committer: kangkaisen <ka...@live.com>
Committed: Thu Jun 29 21:23:43 2017 +0800
----------------------------------------------------------------------
.../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1eb32c3e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index c66af66..3b26b71 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -394,10 +394,6 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
logger.error(ioe.toString(), ioe);
IOException wrapped = new IOException("Error in coprocessor " + debugGitTag, ioe);
ResponseConverter.setControllerException(controller, wrapped);
- } catch (OutOfMemoryError oom) {
- logger.error(oom.toString(), oom);
- IOException wrapped = new IOException("OOM in coprocessor " + debugGitTag, oom);
- ResponseConverter.setControllerException(controller, wrapped);
} finally {
for (RegionScanner innerScanner : regionScanners) {
IOUtils.closeQuietly(innerScanner);
[18/50] [abbrv] kylin git commit: minor, fix ACL NPE
Posted by ka...@apache.org.
minor, fix ACL NPE
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/052d6096
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/052d6096
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/052d6096
Branch: refs/heads/KYLIN-2606
Commit: 052d6096779d5826ba24517b0b4cc9a45b78bf4a
Parents: 8ca0d32
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 17:23:13 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 18:24:43 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/rest/service/AccessService.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/052d6096/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index affa508..ae7ac6e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.service;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.persistence.AclEntity;
@@ -297,8 +298,11 @@ public class AccessService {
}
public List<AccessEntryResponse> generateAceResponses(Acl acl) {
- List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
+ if (null == acl) {
+ return Collections.emptyList();
+ }
+ List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
for (AccessControlEntry ace : acl.getEntries()) {
result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
}
[09/50] [abbrv] kylin git commit: fix ci
Posted by ka...@apache.org.
fix ci
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/86032971
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/86032971
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/86032971
Branch: refs/heads/KYLIN-2606
Commit: 86032971f766a893a28fce9017227999f5defe2d
Parents: 56a0fa5
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 29 11:46:04 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 13:51:45 2017 +0800
----------------------------------------------------------------------
.../query/sql_computedcolumn/query03.sql | 21 ++++++++++++++++++++
.../sql_computedcolumn/query03.sql.compare | 21 ++++++++++++++++++++
.../org/apache/kylin/rest/util/AdHocUtil.java | 19 ++++--------------
3 files changed, 46 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/86032971/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql
new file mode 100644
index 0000000..ff8a570
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql
@@ -0,0 +1,21 @@
+SELECT
+
+count(*) as cnt, sum(price) as sum_price, sum(DEAL_AMOUNT) as deal_amount, SELLER_COUNTRY.NAME, DEAL_YEAR as deal_year
+
+FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
+INNER JOIN TEST_ORDER as TEST_ORDER
+ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
+ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
+ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
+ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
+ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
+
+
+where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and "SELLER_ACCOUNT"."COUNTRY_ABBR" in ('T', 'R') and BUYER_ACCOUNT.COUNTRY_ABBR in ('T', 'R')
+group by SELLER_COUNTRY.NAME, DEAL_YEAR
http://git-wip-us.apache.org/repos/asf/kylin/blob/86032971/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare
new file mode 100644
index 0000000..160375c
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare
@@ -0,0 +1,21 @@
+SELECT
+
+count(*) as cnt, sum(price) as sum_price, sum(TEST_KYLIN_FACT.PRICE * TEST_KYLIN_FACT.ITEM_COUNT) as deal_amount, SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT) as deal_year
+
+FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
+INNER JOIN TEST_ORDER as TEST_ORDER
+ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
+ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
+ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
+ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
+ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
+
+
+where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SUBSTR("SELLER_ACCOUNT"."ACCOUNT_COUNTRY",1,1) in ('T', 'R') and SUBSTR(BUYER_ACCOUNT.ACCOUNT_COUNTRY,1,1) in ('T', 'R')
+group by SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT)
http://git-wip-us.apache.org/repos/asf/kylin/blob/86032971/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
index 9eecdc1..9694a21 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
@@ -35,6 +35,7 @@ import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.MetadataManager.CCInfo;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
@@ -185,23 +186,11 @@ public class AdHocUtil {
return sql;
}
- // identifier in expr must be DB.TABLE.COLUMN, all TABLE in expr should be guaranteed to be same
static String replaceIdentifierInExpr(String expr, String tableAlias, boolean quoted) {
- List<Triple<Integer, Integer, String>> replacements = Lists.newArrayList();
- Matcher matcher = identifierInExprPattern.matcher(expr);
- while (matcher.find()) {
-
- String t = tableAlias == null ? StringUtils.strip(matcher.group(3), ".") : tableAlias;
- String c = matcher.group(4);
-
- String replacement = quoted ? "\"" + t.toUpperCase() + "\".\"" + c.toUpperCase() + "\"" : t + "." + c;
- replacements.add(Triple.of(matcher.start(1), matcher.end(1), replacement));
+ if (tableAlias == null) {
+ return expr;
}
- Collections.reverse(replacements);
- for (Triple<Integer, Integer, String> triple : replacements) {
- expr = expr.substring(0, triple.getLeft()) + triple.getRight() + expr.substring(triple.getMiddle());
- }
- return expr;
+ return CalciteParser.insertAliasInExpr(expr, tableAlias);
}
}
[44/50] [abbrv] kylin git commit: minor,refine no result position
Posted by ka...@apache.org.
minor,refine no result position
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b66751d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b66751d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b66751d
Branch: refs/heads/KYLIN-2606
Commit: 4b66751d4fc9cc77c6bd8f22a899900c66fd31df
Parents: 46d144a
Author: luguosheng1314 <55...@qq.com>
Authored: Tue Jul 4 16:04:49 2017 +0800
Committer: chenzhx <34...@qq.com>
Committed: Tue Jul 4 21:56:33 2017 +0800
----------------------------------------------------------------------
webapp/app/partials/modelDesigner/model_info.html | 2 +-
webapp/app/partials/projects/project_detail.html | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b66751d/webapp/app/partials/modelDesigner/model_info.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/model_info.html b/webapp/app/partials/modelDesigner/model_info.html
index ae8237f..59a565b 100644
--- a/webapp/app/partials/modelDesigner/model_info.html
+++ b/webapp/app/partials/modelDesigner/model_info.html
@@ -35,7 +35,7 @@
ng-model="modelsManager.selectedModel.name" required
placeholder="You can use letters, numbers, and '_'"
ng-maxlength=100 ng-pattern="/^\w+$/" />
- <small class="help-block" ng-show="forms.model_info_form.model_name.$error.required && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$submitted)">Model name is required.</small>
+ <small style="color:red" class="help-block" ng-show="forms.model_info_form.model_name.$error.required && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$submitted)">Model name is required.</small>
<small class="help-block" ng-show="!forms.model_info_form.model_name.$error.required&&forms.model_info_form.model_name.$invalid && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$submitted)">Model name is invalid.</small>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b66751d/webapp/app/partials/projects/project_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/projects/project_detail.html b/webapp/app/partials/projects/project_detail.html
index 431073a..5cfc091 100644
--- a/webapp/app/partials/projects/project_detail.html
+++ b/webapp/app/partials/projects/project_detail.html
@@ -76,7 +76,7 @@
</tbody>
</table>
</div>
- <div no-result ng-if="getMapLength(project.override_kylin_properties) == 0"></div>
+ <div no-result ng-if="getMapLength(project.override_kylin_properties) == 0" style="clear:both"></div>
<div class="col-xs-1"></div>
</div>
</div>
[13/50] [abbrv] kylin git commit: Minor change, format code
Posted by ka...@apache.org.
Minor change, format code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8acbc62
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8acbc62
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8acbc62
Branch: refs/heads/KYLIN-2606
Commit: c8acbc620b2e52fd9d2ba3fc6fc478a6418b0f97
Parents: 2f7aa45
Author: auphyroc99 <45...@qq.com>
Authored: Thu Jun 29 14:48:38 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 14:50:21 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/jdbc/DriverTest.java | 20 +++++++++++++-------
1 file changed, 13 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8acbc62/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
index 4fd9d33..70978d7 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
@@ -178,7 +178,8 @@ public class DriverTest {
info.put("password", "KYLIN");
Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
- PreparedStatement state = conn.prepareStatement("select cal_dt, count(*) from test_kylin_fact where seller_id=? group by cal_dt");
+ PreparedStatement state = conn
+ .prepareStatement("select cal_dt, count(*) from test_kylin_fact where seller_id=? group by cal_dt");
state.setLong(1, 10000001);
ResultSet resultSet = state.executeQuery();
@@ -189,14 +190,14 @@ public class DriverTest {
state.close();
conn.close();
}
-
+
@Test
- public void testSSLFromURL() throws SQLException{
+ public void testSSLFromURL() throws SQLException {
Driver driver = new DummyDriver();
Connection conn = driver.connect("jdbc:kylin:ssl=True;//test_url/test_db", null);
- assertEquals("test_url", ((KylinConnection)conn).getBaseUrl());
- assertEquals("test_db", ((KylinConnection)conn).getProject());
- assertTrue(Boolean.parseBoolean( (String)((KylinConnection)conn).getConnectionProperties().get("ssl")));
+ assertEquals("test_url", ((KylinConnection) conn).getBaseUrl());
+ assertEquals("test_db", ((KylinConnection) conn).getProject());
+ assertTrue(Boolean.parseBoolean((String) ((KylinConnection) conn).getConnectionProperties().get("ssl")));
}
private void printResultSet(ResultSet rs) throws SQLException {
@@ -221,7 +222,12 @@ public class DriverTest {
System.out.println("Metadata:");
for (int i = 0; i < metadata.getColumnCount(); i++) {
- String metaStr = metadata.getCatalogName(i + 1) + " " + metadata.getColumnClassName(i + 1) + " " + metadata.getColumnDisplaySize(i + 1) + " " + metadata.getColumnLabel(i + 1) + " " + metadata.getColumnName(i + 1) + " " + metadata.getColumnType(i + 1) + " " + metadata.getColumnTypeName(i + 1) + " " + metadata.getPrecision(i + 1) + " " + metadata.getScale(i + 1) + " " + metadata.getSchemaName(i + 1) + " " + metadata.getTableName(i + 1);
+ String metaStr = metadata.getCatalogName(i + 1) + " " + metadata.getColumnClassName(i + 1) + " "
+ + metadata.getColumnDisplaySize(i + 1) + " " + metadata.getColumnLabel(i + 1) + " "
+ + metadata.getColumnName(i + 1) + " " + metadata.getColumnType(i + 1) + " "
+ + metadata.getColumnTypeName(i + 1) + " " + metadata.getPrecision(i + 1) + " "
+ + metadata.getScale(i + 1) + " " + metadata.getSchemaName(i + 1) + " "
+ + metadata.getTableName(i + 1);
System.out.println(metaStr);
}
}
[36/50] [abbrv] kylin git commit: minor, roll back validate join key
Posted by ka...@apache.org.
minor, roll back validate join key
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ae8ca7c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ae8ca7c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ae8ca7c
Branch: refs/heads/KYLIN-2606
Commit: 3ae8ca7cd3796583787e0657edec31b8bdb30ce1
Parents: dc232d1
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Jun 30 20:26:31 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 20:29:11 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 31 --------------------
1 file changed, 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ae8ca7c/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index abee71b..bc35e2a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -617,38 +617,7 @@ public class DataModelDesc extends RootPersistentEntity {
+ fkCols[i].getDatatype());
}
}
-
- if (!isSpecialTestModel()) {
- for (TblColRef t : fkCols) {
- if (!isJoinKeyDimension(t.getIdentity())) {
- throw new IllegalStateException(
- "Foreigner key: " + t.getIdentity() + " should be a dimension.");
- }
- }
-
- for (TblColRef t : pkCols) {
- if (!isJoinKeyDimension(t.getIdentity())) {
- throw new IllegalStateException("Primary key: " + t.getIdentity() + " should be a dimension.");
- }
- }
- }
- }
- }
-
- private boolean isJoinKeyDimension(String key) {
- for (ModelDimensionDesc d : dimensions) {
- for (String col : d.getColumns()) {
- if (key.equals(d.getTable() + '.' + col))
- return true;
- }
}
- return false;
- }
-
- private boolean isSpecialTestModel() {
- if (config.isDevEnv() && (name.startsWith("test_kylin_") || name.startsWith("test_streaming")))
- return true;
- return false;
}
private void initJoinsTree() {
[05/50] [abbrv] kylin git commit: KYLIN-2686 The same project's
computed column's definition can not be same
Posted by ka...@apache.org.
KYLIN-2686 The same project's computed column's definition can not be same
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a0b7e74c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a0b7e74c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a0b7e74c
Branch: refs/heads/KYLIN-2606
Commit: a0b7e74c16718bb7bab3eb093d6953c393f3444a
Parents: b2fc2c2
Author: Aron.tao <24...@qq.com>
Authored: Wed Jun 28 12:41:34 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Wed Jun 28 14:04:24 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 43 ++++++++++++++------
1 file changed, 31 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0b7e74c/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 7e08f1c..7de955e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -471,33 +471,52 @@ public class DataModelDesc extends RootPersistentEntity {
}
private void initComputedColumns(Map<String, CCInfo> ccInfoMap) {
+ if (ccInfoMap == null) {
+ logger.error("cc info map is null");
+ }
+
Set<String> ccSet = Sets.newHashSet();//make sure cc name does not duplicate within this model
- for (ComputedColumnDesc computedColumnDesc : this.computedColumnDescs) {
- computedColumnDesc.init();
+ for (ComputedColumnDesc thisCCDesc : this.computedColumnDescs) {
+ thisCCDesc.init();
+ String thisCCName = thisCCDesc.getFullName();
- if (ccSet.contains(computedColumnDesc.getFullName())) {
- throw new IllegalArgumentException(
- String.format("More than one computed column named %s exist in model %s",
- computedColumnDesc.getFullName(), this.getName()));
+ if (ccSet.contains(thisCCName)) {
+ throw new IllegalArgumentException(String.format(
+ "More than one computed column named %s exist in model %s", thisCCName, this.getName()));
} else {
- ccSet.add(computedColumnDesc.getFullName());
+ ccSet.add(thisCCName);
}
- CCInfo other = ccInfoMap.get(computedColumnDesc.getFullName());
+ CCInfo other = ccInfoMap.get(thisCCName);
if (other == null || (other.getDataModelDescs().size() == 1 && other.getDataModelDescs().contains(this))) {
- ccInfoMap.put(computedColumnDesc.getFullName(),
- new CCInfo(computedColumnDesc, Sets.<DataModelDesc> newHashSet(this)));
- } else if (other.getComputedColumnDesc().equals(computedColumnDesc)) {
+ //check whether two computer columns's definition is the same.
+ for (CCInfo sysCCInfo : ccInfoMap.values()) {
+ String definition0 = thisCCDesc.getExpression();
+ String definition1 = sysCCInfo.getComputedColumnDesc().getExpression();
+ if (isTwoCCDefinitionEquals(definition0, definition1)) {
+ throw new IllegalStateException(String.format(
+ "Computed column %s'definition: %s is already defined in other models: %s. Please change another definition, or try to keep consistent definition",
+ thisCCName, definition0, sysCCInfo.getDataModelDescs()));
+ }
+ }
+ ccInfoMap.put(thisCCName, new CCInfo(thisCCDesc, Sets.<DataModelDesc> newHashSet(this)));
+ } else if (other.getComputedColumnDesc().equals(thisCCDesc)) {
other.getDataModelDescs().add(this);
} else {
throw new IllegalStateException(String.format(
"Computed column named %s is already defined in other models: %s. Please change another name, or try to keep consistent definition", //
- computedColumnDesc.getFullName(), other.getDataModelDescs()));
+ thisCCName, other.getDataModelDescs()));
}
}
}
+ private boolean isTwoCCDefinitionEquals(String definition0, String definition1) {
+ definition0 = definition0.replaceAll("\\s*", "");
+ definition1 = definition1.replaceAll("\\s*", "");
+ return definition0.equalsIgnoreCase(definition1);
+ }
+
private void initJoinColumns() {
for (JoinTableDesc joinTable : joinTables) {
[17/50] [abbrv] kylin git commit: #476 enable model read permission
when grant cube permissions
Posted by ka...@apache.org.
#476 enable model read permission when grant cube permissions
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ca0d321
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ca0d321
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ca0d321
Branch: refs/heads/KYLIN-2606
Commit: 8ca0d3217453929d0ac0c3078d9e78e769f852b0
Parents: 7f79083
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 17:23:13 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 18:05:01 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/controller2/AccessControllerV2.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ca0d321/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
index 1ca8652..bd7d897 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
@@ -21,12 +21,15 @@ package org.apache.kylin.rest.controller2;
import java.io.IOException;
import org.apache.kylin.common.persistence.AclEntity;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.rest.controller.BasicController;
import org.apache.kylin.rest.request.AccessRequest;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.ResponseCode;
+import org.apache.kylin.rest.security.AclEntityType;
import org.apache.kylin.rest.security.AclPermissionFactory;
import org.apache.kylin.rest.service.AccessService;
+import org.apache.kylin.rest.service.CubeService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.acls.model.Acl;
@@ -51,6 +54,10 @@ public class AccessControllerV2 extends BasicController {
@Qualifier("accessService")
private AccessService accessService;
+ @Autowired
+ @Qualifier("cubeMgmtService")
+ private CubeService cubeService;
+
/**
* Get access entry list of a domain object
*
@@ -100,6 +107,15 @@ public class AccessControllerV2 extends BasicController {
Permission permission = AclPermissionFactory.getPermission(accessRequest.getPermission());
Acl acl = accessService.grant(ae, permission, sid);
+ if (AclEntityType.CUBE_INSTANCE.equals(type)) {
+ CubeInstance instance = cubeService.getCubeManager().getCubeByUuid(uuid);
+ if (instance != null) {
+ AclEntity model = accessService.getAclEntity(AclEntityType.DATA_MODEL_DESC, instance.getModel().getUuid());
+ // FIXME: should not always grant
+ accessService.grant(model, permission, sid);
+ }
+ }
+
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, accessService.generateAceResponses(acl), "");
}
[15/50] [abbrv] kylin git commit: #1294 refine acl
Posted by ka...@apache.org.
#1294 refine acl
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7f790837
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7f790837
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7f790837
Branch: refs/heads/KYLIN-2606
Commit: 7f79083722b634df8f4f814c6a59e0472dbcbe74
Parents: c490e6c
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 16:35:05 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 16:36:57 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/controller2/AccessControllerV2.java | 2 +-
.../src/main/java/org/apache/kylin/rest/service/AccessService.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f790837/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
index a61e321..1ca8652 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
@@ -80,7 +80,7 @@ public class AccessControllerV2 extends BasicController {
public EnvelopeResponse listAccessEntitiesV2(@PathVariable String type, @PathVariable String uuid) {
AclEntity ae = accessService.getAclEntity(type, uuid);
Acl acl = accessService.getAcl(ae);
- return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, accessService.generateListAceResponses(acl), "");
+ return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, accessService.generateAllAceResponses(acl), "");
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f790837/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 3a6d723..affa508 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -322,7 +322,7 @@ public class AccessService {
}
}
- public Object generateListAceResponses(Acl acl) {
+ public Object generateAllAceResponses(Acl acl) {
List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
while (acl != null) {
[37/50] [abbrv] kylin git commit: minor, refine acl
Posted by ka...@apache.org.
minor, refine acl
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d91f5229
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d91f5229
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d91f5229
Branch: refs/heads/KYLIN-2606
Commit: d91f522904424f59d817bbcde10c47cc68f04d9d
Parents: 3ae8ca7
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Jun 30 20:52:03 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 30 20:57:18 2017 +0800
----------------------------------------------------------------------
.../hbase/ITAclTableMigrationToolTest.java | 10 ++--
.../rest/controller2/ProjectControllerV2.java | 4 ++
.../org/apache/kylin/rest/msg/CnMessage.java | 2 +-
.../java/org/apache/kylin/rest/msg/Message.java | 2 +-
.../apache/kylin/rest/security/ManagedUser.java | 54 +++++++++++++++++---
.../rest/service/AclTableMigrationTool.java | 2 +-
.../kylin/rest/service/ServiceTestBase.java | 10 ++--
tool-assembly/pom.xml | 1 +
tool/pom.xml | 12 +++++
9 files changed, 79 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
index 2cb671e..05f437d 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
@@ -44,7 +44,6 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclConstant;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.AclTableMigrationTool;
-import org.apache.kylin.rest.service.UserGrantedAuthority;
import org.apache.kylin.rest.service.UserService;
import org.apache.kylin.rest.util.Serializer;
import org.junit.After;
@@ -53,6 +52,7 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.authority.SimpleGrantedAuthority;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -70,7 +70,7 @@ public class ITAclTableMigrationToolTest extends HBaseMetadataTestCase {
private TableName userTable = TableName.valueOf(STORE_WITH_OLD_TABLE + AclConstant.USER_TABLE_NAME);
- private Serializer<UserGrantedAuthority[]> ugaSerializer = new Serializer<UserGrantedAuthority[]>(UserGrantedAuthority[].class);
+ private Serializer<SimpleGrantedAuthority[]> ugaSerializer = new Serializer<>(SimpleGrantedAuthority[].class);
private AclTableMigrationTool aclTableMigrationJob;
@@ -192,13 +192,13 @@ public class ITAclTableMigrationToolTest extends HBaseMetadataTestCase {
if (authorities == null)
authorities = Collections.emptyList();
- UserGrantedAuthority[] serializing = new UserGrantedAuthority[authorities.size() + 1];
+ SimpleGrantedAuthority[] serializing = new SimpleGrantedAuthority[authorities.size() + 1];
// password is stored as the [0] authority
- serializing[0] = new UserGrantedAuthority(AclConstant.PWD_PREFIX + "password");
+ serializing[0] = new SimpleGrantedAuthority(AclConstant.PWD_PREFIX + "password");
int i = 1;
for (GrantedAuthority a : authorities) {
- serializing[i++] = new UserGrantedAuthority(a.getAuthority());
+ serializing[i++] = new SimpleGrantedAuthority(a.getAuthority());
}
byte[] value = ugaSerializer.serialize(serializing);
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
index a25e5b1..d6ac8f2 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
@@ -142,6 +142,10 @@ public class ProjectControllerV2 extends BasicController {
if (projectDesc.getName().equals(currentProject.getName())) {
updatedProj = projectService.updateProject(projectDesc, currentProject);
} else {
+ if (!isProjectEmpty(formerProjectName)) {
+ throw new BadRequestException(msg.getDELETE_PROJECT_NOT_EMPTY());
+ }
+ // disable project rename
updatedProj = projectService.renameProject(projectDesc, currentProject);
}
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, updatedProj, "");
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
index a828aa0..e086b68 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
@@ -246,7 +246,7 @@ public class CnMessage extends Message {
}
public String getDELETE_PROJECT_NOT_EMPTY() {
- return "不能删除该项目，如需要删除请先清空其中的Cube和Model";
+ return "不能修改该项目，如需要修改请先清空其中的Cube和Model";
}
public String getRENAME_PROJECT_NOT_EMPTY() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
index 45c1a65..f48a217 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
@@ -246,7 +246,7 @@ public class Message {
}
public String getDELETE_PROJECT_NOT_EMPTY() {
- return "Cannot delete non-empty project";
+ return "Cannot modify non-empty project";
}
// Table
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java b/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
index 280339e..69326a7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
@@ -18,6 +18,7 @@
package org.apache.kylin.rest.security;
+import java.io.IOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -25,10 +26,20 @@ import java.util.List;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.rest.service.UserGrantedAuthority;
import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.authority.SimpleGrantedAuthority;
import org.springframework.security.core.userdetails.UserDetails;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.google.common.collect.Lists;
@SuppressWarnings("serial")
@@ -40,7 +51,9 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
@JsonProperty
private String password;
@JsonProperty
- private List<UserGrantedAuthority> authorities = Lists.newArrayList();
+ @JsonSerialize(using = SimpleGrantedAuthoritySerializer.class)
+ @JsonDeserialize(using = SimpleGrantedAuthorityDeserializer.class)
+ private List<SimpleGrantedAuthority> authorities = Lists.newArrayList();
@JsonProperty
private boolean disabled = false;
@JsonProperty
@@ -60,7 +73,7 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
}
public ManagedUser(@JsonProperty String username, @JsonProperty String password,
- @JsonProperty List<UserGrantedAuthority> authorities, @JsonProperty boolean disabled,
+ @JsonProperty List<SimpleGrantedAuthority> authorities, @JsonProperty boolean disabled,
@JsonProperty boolean defaultPassword, @JsonProperty boolean locked, @JsonProperty long lockedTime,
@JsonProperty int wrongTime) {
this.username = username;
@@ -82,7 +95,7 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
this.authorities = Lists.newArrayList();
for (String a : authoritiesStr) {
- authorities.add(new UserGrantedAuthority(a));
+ authorities.add(new SimpleGrantedAuthority(a));
}
caterLegacy();
@@ -116,7 +129,7 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
}
private void caterLegacy() {
- Iterator<UserGrantedAuthority> iterator = authorities.iterator();
+ Iterator<SimpleGrantedAuthority> iterator = authorities.iterator();
while (iterator.hasNext()) {
if (DISABLED_ROLE.equals(iterator.next().getAuthority())) {
iterator.remove();
@@ -125,14 +138,14 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
}
}
- public List<UserGrantedAuthority> getAuthorities() {
+ public List<SimpleGrantedAuthority> getAuthorities() {
return this.authorities;
}
public void setGrantedAuthorities(Collection<? extends GrantedAuthority> grantedAuthorities) {
this.authorities = Lists.newArrayList();
for (GrantedAuthority grantedAuthority : grantedAuthorities) {
- this.authorities.add(new UserGrantedAuthority(grantedAuthority.getAuthority()));
+ this.authorities.add(new SimpleGrantedAuthority(grantedAuthority.getAuthority()));
}
}
@@ -228,4 +241,33 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
public String toString() {
return "ManagedUser [username=" + username + ", authorities=" + authorities + "]";
}
+
+ private static class SimpleGrantedAuthoritySerializer extends JsonSerializer<List<SimpleGrantedAuthority>> {
+
+ @Override
+ public void serialize(List<SimpleGrantedAuthority> value, JsonGenerator gen, SerializerProvider serializers)
+ throws IOException, JsonProcessingException {
+ List<UserGrantedAuthority> ugaList = Lists.newArrayList();
+ for (SimpleGrantedAuthority sga : value) {
+ ugaList.add(new UserGrantedAuthority(sga.getAuthority()));
+ }
+
+ gen.writeObject(ugaList);
+ }
+ }
+
+ private static class SimpleGrantedAuthorityDeserializer extends JsonDeserializer<List<SimpleGrantedAuthority>> {
+
+ @Override
+ public List<SimpleGrantedAuthority> deserialize(JsonParser p, DeserializationContext ctxt)
+ throws IOException, JsonProcessingException {
+ UserGrantedAuthority[] ugaArray = p.readValueAs(UserGrantedAuthority[].class);
+ List<SimpleGrantedAuthority> sgaList = Lists.newArrayList();
+ for (UserGrantedAuthority uga : ugaArray) {
+ sgaList.add(new SimpleGrantedAuthority(uga.getAuthority()));
+ }
+
+ return sgaList;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
index 64bac23..029efdc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
@@ -193,7 +193,7 @@ public class AclTableMigrationTool {
}
private DomainObjectInfo getDomainObjectInfoFromRs(Result result) {
- String type = String.valueOf(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY),
+ String type = new String(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY),
Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_TYPE_COLUMN)));
String id = new String(result.getRow());
DomainObjectInfo newInfo = new DomainObjectInfo();
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index 1d60a53..e2f5258 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -35,6 +35,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.Authentication;
+import org.springframework.security.core.authority.SimpleGrantedAuthority;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
@@ -72,18 +73,19 @@ public class ServiceTestBase extends LocalFileMetadataTestCase {
if (!userService.userExists("ADMIN")) {
userService.createUser(new ManagedUser("ADMIN", "KYLIN", false, Arrays.asList(//
- new UserGrantedAuthority(Constant.ROLE_ADMIN), new UserGrantedAuthority(Constant.ROLE_ANALYST),
- new UserGrantedAuthority(Constant.ROLE_MODELER))));
+ new SimpleGrantedAuthority(Constant.ROLE_ADMIN), new SimpleGrantedAuthority(Constant.ROLE_ANALYST),
+ new SimpleGrantedAuthority(Constant.ROLE_MODELER))));
}
if (!userService.userExists("MODELER")) {
userService.createUser(new ManagedUser("MODELER", "MODELER", false, Arrays.asList(//
- new UserGrantedAuthority(Constant.ROLE_ANALYST), new UserGrantedAuthority(Constant.ROLE_MODELER))));
+ new SimpleGrantedAuthority(Constant.ROLE_ANALYST),
+ new SimpleGrantedAuthority(Constant.ROLE_MODELER))));
}
if (!userService.userExists("ANALYST")) {
userService.createUser(new ManagedUser("ANALYST", "ANALYST", false, Arrays.asList(//
- new UserGrantedAuthority(Constant.ROLE_ANALYST))));
+ new SimpleGrantedAuthority(Constant.ROLE_ANALYST))));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/tool-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml
index 0595bdd..df0725b 100644
--- a/tool-assembly/pom.xml
+++ b/tool-assembly/pom.xml
@@ -104,6 +104,7 @@
<include>org.apache.kylin:*</include>
<include>org.springframework.security:spring-security-core</include>
<include>org.springframework.security:spring-security-acl</include>
+ <include>org.springframework:spring-core</include>
</includes>
</artifactSet>
<relocations>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d91f5229/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index cced5d2..06a7e5a 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -32,6 +32,10 @@
<version>2.1.0-SNAPSHOT</version>
</parent>
+ <properties>
+ <spring.framework.version>4.2.8.RELEASE</spring.framework.version>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
@@ -72,6 +76,14 @@
<scope>provided</scope>
</dependency>
+ <!--Spring-->
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ <version>${spring.framework.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
<!-- Env & Test -->
<dependency>
<groupId>org.apache.kylin</groupId>
[38/50] [abbrv] kylin git commit: minor,
rename adhoc to pushdown in KYLIN
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 8d18901..f3402ef 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -94,7 +94,7 @@ import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.util.AclUtil;
-import org.apache.kylin.rest.util.AdHocUtil;
+import org.apache.kylin.rest.util.PushDownUtil;
import org.apache.kylin.rest.util.TableauInterceptor;
import org.apache.kylin.storage.hybrid.HybridInstance;
import org.slf4j.Logger;
@@ -220,7 +220,8 @@ public class QueryService extends BasicService {
return null;
}
List<Query> queries = new ArrayList<Query>();
- QueryRecord record = queryStore.getResource(getQueryKeyById(creator), QueryRecord.class, QueryRecordSerializer.getInstance());
+ QueryRecord record = queryStore.getResource(getQueryKeyById(creator), QueryRecord.class,
+ QueryRecordSerializer.getInstance());
if (record != null) {
for (Query query : record.getQueries()) {
queries.add(query);
@@ -235,7 +236,7 @@ public class QueryService extends BasicService {
final Set<Long> cuboidIds = new HashSet<Long>();
float duration = response.getDuration() / (float) 1000;
boolean storageCacheUsed = response.isStorageCacheUsed();
- boolean isAdHoc = response.isAdHoc();
+ boolean isPushDown = response.isPushDown();
if (!response.isHitExceptionCache() && null != OLAPContext.getThreadLocalContexts()) {
for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) {
@@ -276,7 +277,7 @@ public class QueryService extends BasicService {
stringBuilder.append("Is Partial Result: ").append(response.isPartial()).append(newLine);
stringBuilder.append("Hit Exception Cache: ").append(response.isHitExceptionCache()).append(newLine);
stringBuilder.append("Storage cache used: ").append(storageCacheUsed).append(newLine);
- stringBuilder.append("Is Ad-hoc Query: ").append(isAdHoc).append(newLine);
+ stringBuilder.append("Is Query Push-Down: ").append(isPushDown).append(newLine);
stringBuilder.append("Message: ").append(response.getExceptionMessage()).append(newLine);
stringBuilder.append("==========================[QUERY]===============================").append(newLine);
@@ -292,14 +293,16 @@ public class QueryService extends BasicService {
return;
}
} catch (AccessDeniedException e) {
- logger.warn("Current user {} has no READ permission on current project {}, please ask Administrator for permission granting.");
+ logger.warn(
+ "Current user {} has no READ permission on current project {}, please ask Administrator for permission granting.");
//just continue
}
String realizationsStr = sqlResponse.getCube();//CUBE[name=abc],HYBRID[name=xyz]
if (StringUtils.isEmpty(realizationsStr)) {
- throw new AccessDeniedException("Ad-hoc query requires having READ permission on project, please ask Administrator to grant you permissions");
+ throw new AccessDeniedException(
+ "Query pushdown requires having READ permission on project, please ask Administrator to grant you permissions");
}
String[] splits = StringUtils.split(realizationsStr, ",");
@@ -344,7 +347,8 @@ public class QueryService extends BasicService {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
String serverMode = kylinConfig.getServerMode();
- if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase()))) {
+ if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase())
+ || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase()))) {
throw new BadRequestException(String.format(msg.getQUERY_NOT_ALLOWED(), serverMode));
}
if (StringUtils.isBlank(sqlRequest.getProject())) {
@@ -370,7 +374,8 @@ public class QueryService extends BasicService {
long startTime = System.currentTimeMillis();
SQLResponse sqlResponse = null;
- boolean queryCacheEnabled = checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in KylinConfig") && //
+ boolean queryCacheEnabled = checkCondition(kylinConfig.isQueryCacheEnabled(),
+ "query cache disabled in KylinConfig") && //
checkCondition(!BackdoorToggles.getDisableCache(), "query cache disabled in BackdoorToggles");
if (queryCacheEnabled) {
@@ -386,13 +391,22 @@ public class QueryService extends BasicService {
long scanBytesThreshold = kylinConfig.getQueryScanBytesCacheThreshold();
sqlResponse.setDuration(System.currentTimeMillis() - startTime);
logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", //
- String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()));
+ String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()),
+ String.valueOf(sqlResponse.getTotalScanCount()));
if (checkCondition(queryCacheEnabled, "query cache is disabled") //
&& checkCondition(!sqlResponse.getIsException(), "query has exception") //
- && checkCondition(sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scanCountThreshold || sqlResponse.getTotalScanBytes() > scanBytesThreshold, //
- "query is too lightweight with duration: {} (threshold {}), scan count: {} (threshold {}), scan bytes: {} (threshold {})", sqlResponse.getDuration(), durationThreshold, sqlResponse.getTotalScanCount(), scanCountThreshold, sqlResponse.getTotalScanBytes(), scanBytesThreshold)
- && checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(), "query response is too large: {} ({})", sqlResponse.getResults().size(), kylinConfig.getLargeQueryThreshold())) {
- cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest.getCacheKey(), sqlResponse));
+ && checkCondition(
+ sqlResponse.getDuration() > durationThreshold
+ || sqlResponse.getTotalScanCount() > scanCountThreshold
+ || sqlResponse.getTotalScanBytes() > scanBytesThreshold, //
+ "query is too lightweight with duration: {} (threshold {}), scan count: {} (threshold {}), scan bytes: {} (threshold {})",
+ sqlResponse.getDuration(), durationThreshold, sqlResponse.getTotalScanCount(),
+ scanCountThreshold, sqlResponse.getTotalScanBytes(), scanBytesThreshold)
+ && checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(),
+ "query response is too large: {} ({})", sqlResponse.getResults().size(),
+ kylinConfig.getLargeQueryThreshold())) {
+ cacheManager.getCache(SUCCESS_QUERY_CACHE)
+ .put(new Element(sqlRequest.getCacheKey(), sqlResponse));
}
} else {
@@ -411,7 +425,8 @@ public class QueryService extends BasicService {
sqlResponse.setTotalScanCount(queryContext.getScannedRows());
sqlResponse.setTotalScanBytes(queryContext.getScannedBytes());
- if (queryCacheEnabled && e.getCause() != null && ExceptionUtils.getRootCause(e) instanceof ResourceLimitExceededException) {
+ if (queryCacheEnabled && e.getCause() != null
+ && ExceptionUtils.getRootCause(e) instanceof ResourceLimitExceededException) {
Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
exceptionCache.put(new Element(sqlRequest.getCacheKey(), sqlResponse));
}
@@ -459,7 +474,8 @@ public class QueryService extends BasicService {
private SQLResponse queryWithSqlMassage(SQLRequest sqlRequest) throws Exception {
String userInfo = SecurityContextHolder.getContext().getAuthentication().getName();
- final Collection<? extends GrantedAuthority> grantedAuthorities = SecurityContextHolder.getContext().getAuthentication().getAuthorities();
+ final Collection<? extends GrantedAuthority> grantedAuthorities = SecurityContextHolder.getContext()
+ .getAuthentication().getAuthorities();
for (GrantedAuthority grantedAuthority : grantedAuthorities) {
userInfo += ",";
userInfo += grantedAuthority.getAuthority();
@@ -514,9 +530,12 @@ public class QueryService extends BasicService {
String schemaName = JDBCTableMeta.getString(2);
// Not every JDBC data provider offers full 10 columns, e.g., PostgreSQL has only 5
- TableMeta tblMeta = new TableMeta(catalogName == null ? Constant.FakeCatalogName : catalogName, schemaName == null ? Constant.FakeSchemaName : schemaName, JDBCTableMeta.getString(3), JDBCTableMeta.getString(4), JDBCTableMeta.getString(5), null, null, null, null, null);
+ TableMeta tblMeta = new TableMeta(catalogName == null ? Constant.FakeCatalogName : catalogName,
+ schemaName == null ? Constant.FakeSchemaName : schemaName, JDBCTableMeta.getString(3),
+ JDBCTableMeta.getString(4), JDBCTableMeta.getString(5), null, null, null, null, null);
- if (!cubedOnly || getProjectManager().isExposedTable(project, schemaName + "." + tblMeta.getTABLE_NAME())) {
+ if (!cubedOnly
+ || getProjectManager().isExposedTable(project, schemaName + "." + tblMeta.getTABLE_NAME())) {
tableMetas.add(tblMeta);
tableMap.put(tblMeta.getTABLE_SCHEM() + "#" + tblMeta.getTABLE_NAME(), tblMeta);
}
@@ -529,9 +548,18 @@ public class QueryService extends BasicService {
String schemaName = columnMeta.getString(2);
// kylin(optiq) is not strictly following JDBC specification
- ColumnMeta colmnMeta = new ColumnMeta(catalogName == null ? Constant.FakeCatalogName : catalogName, schemaName == null ? Constant.FakeSchemaName : schemaName, columnMeta.getString(3), columnMeta.getString(4), columnMeta.getInt(5), columnMeta.getString(6), columnMeta.getInt(7), getInt(columnMeta.getString(8)), columnMeta.getInt(9), columnMeta.getInt(10), columnMeta.getInt(11), columnMeta.getString(12), columnMeta.getString(13), getInt(columnMeta.getString(14)), getInt(columnMeta.getString(15)), columnMeta.getInt(16), columnMeta.getInt(17), columnMeta.getString(18), columnMeta.getString(19), columnMeta.getString(20), columnMeta.getString(21), getShort(columnMeta.getString(22)), columnMeta.getString(23));
-
- if (!cubedOnly || getProjectManager().isExposedColumn(project, schemaName + "." + colmnMeta.getTABLE_NAME(), colmnMeta.getCOLUMN_NAME())) {
+ ColumnMeta colmnMeta = new ColumnMeta(catalogName == null ? Constant.FakeCatalogName : catalogName,
+ schemaName == null ? Constant.FakeSchemaName : schemaName, columnMeta.getString(3),
+ columnMeta.getString(4), columnMeta.getInt(5), columnMeta.getString(6), columnMeta.getInt(7),
+ getInt(columnMeta.getString(8)), columnMeta.getInt(9), columnMeta.getInt(10),
+ columnMeta.getInt(11), columnMeta.getString(12), columnMeta.getString(13),
+ getInt(columnMeta.getString(14)), getInt(columnMeta.getString(15)), columnMeta.getInt(16),
+ columnMeta.getInt(17), columnMeta.getString(18), columnMeta.getString(19),
+ columnMeta.getString(20), columnMeta.getString(21), getShort(columnMeta.getString(22)),
+ columnMeta.getString(23));
+
+ if (!cubedOnly || getProjectManager().isExposedColumn(project,
+ schemaName + "." + colmnMeta.getTABLE_NAME(), colmnMeta.getCOLUMN_NAME())) {
tableMap.get(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME()).addColumn(colmnMeta);
}
}
@@ -551,7 +579,8 @@ public class QueryService extends BasicService {
}
@SuppressWarnings("checkstyle:methodlength")
- protected List<TableMetaWithType> getMetadataV2(CubeManager cubeMgr, String project, boolean cubedOnly) throws SQLException, IOException {
+ protected List<TableMetaWithType> getMetadataV2(CubeManager cubeMgr, String project, boolean cubedOnly)
+ throws SQLException, IOException {
//Message msg = MsgPicker.getMsg();
Connection conn = null;
@@ -578,9 +607,13 @@ public class QueryService extends BasicService {
String schemaName = JDBCTableMeta.getString(2);
// Not every JDBC data provider offers full 10 columns, e.g., PostgreSQL has only 5
- TableMetaWithType tblMeta = new TableMetaWithType(catalogName == null ? Constant.FakeCatalogName : catalogName, schemaName == null ? Constant.FakeSchemaName : schemaName, JDBCTableMeta.getString(3), JDBCTableMeta.getString(4), JDBCTableMeta.getString(5), null, null, null, null, null);
+ TableMetaWithType tblMeta = new TableMetaWithType(
+ catalogName == null ? Constant.FakeCatalogName : catalogName,
+ schemaName == null ? Constant.FakeSchemaName : schemaName, JDBCTableMeta.getString(3),
+ JDBCTableMeta.getString(4), JDBCTableMeta.getString(5), null, null, null, null, null);
- if (!cubedOnly || getProjectManager().isExposedTable(project, schemaName + "." + tblMeta.getTABLE_NAME())) {
+ if (!cubedOnly
+ || getProjectManager().isExposedTable(project, schemaName + "." + tblMeta.getTABLE_NAME())) {
tableMetas.add(tblMeta);
tableMap.put(tblMeta.getTABLE_SCHEM() + "#" + tblMeta.getTABLE_NAME(), tblMeta);
}
@@ -593,11 +626,22 @@ public class QueryService extends BasicService {
String schemaName = columnMeta.getString(2);
// kylin(optiq) is not strictly following JDBC specification
- ColumnMetaWithType colmnMeta = new ColumnMetaWithType(catalogName == null ? Constant.FakeCatalogName : catalogName, schemaName == null ? Constant.FakeSchemaName : schemaName, columnMeta.getString(3), columnMeta.getString(4), columnMeta.getInt(5), columnMeta.getString(6), columnMeta.getInt(7), getInt(columnMeta.getString(8)), columnMeta.getInt(9), columnMeta.getInt(10), columnMeta.getInt(11), columnMeta.getString(12), columnMeta.getString(13), getInt(columnMeta.getString(14)), getInt(columnMeta.getString(15)), columnMeta.getInt(16), columnMeta.getInt(17), columnMeta.getString(18), columnMeta.getString(19), columnMeta.getString(20), columnMeta.getString(21), getShort(columnMeta.getString(22)), columnMeta.getString(23));
-
- if (!cubedOnly || getProjectManager().isExposedColumn(project, schemaName + "." + colmnMeta.getTABLE_NAME(), colmnMeta.getCOLUMN_NAME())) {
+ ColumnMetaWithType colmnMeta = new ColumnMetaWithType(
+ catalogName == null ? Constant.FakeCatalogName : catalogName,
+ schemaName == null ? Constant.FakeSchemaName : schemaName, columnMeta.getString(3),
+ columnMeta.getString(4), columnMeta.getInt(5), columnMeta.getString(6), columnMeta.getInt(7),
+ getInt(columnMeta.getString(8)), columnMeta.getInt(9), columnMeta.getInt(10),
+ columnMeta.getInt(11), columnMeta.getString(12), columnMeta.getString(13),
+ getInt(columnMeta.getString(14)), getInt(columnMeta.getString(15)), columnMeta.getInt(16),
+ columnMeta.getInt(17), columnMeta.getString(18), columnMeta.getString(19),
+ columnMeta.getString(20), columnMeta.getString(21), getShort(columnMeta.getString(22)),
+ columnMeta.getString(23));
+
+ if (!cubedOnly || getProjectManager().isExposedColumn(project,
+ schemaName + "." + colmnMeta.getTABLE_NAME(), colmnMeta.getCOLUMN_NAME())) {
tableMap.get(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME()).addColumn(colmnMeta);
- columnMap.put(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME() + "#" + colmnMeta.getCOLUMN_NAME(), colmnMeta);
+ columnMap.put(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME() + "#"
+ + colmnMeta.getCOLUMN_NAME(), colmnMeta);
}
}
@@ -638,7 +682,8 @@ public class QueryService extends BasicService {
for (JoinTableDesc joinTableDesc : dataModelDesc.getJoinTables()) {
JoinDesc joinDesc = joinTableDesc.getJoin();
for (String pk : joinDesc.getPrimaryKey()) {
- String columnIdentity = (dataModelDesc.findTable(pk.substring(0, pk.indexOf("."))).getTableIdentity() + pk.substring(pk.indexOf("."))).replace('.', '#');
+ String columnIdentity = (dataModelDesc.findTable(pk.substring(0, pk.indexOf(".")))
+ .getTableIdentity() + pk.substring(pk.indexOf("."))).replace('.', '#');
if (columnMap.containsKey(columnIdentity)) {
columnMap.get(columnIdentity).getTYPE().add(ColumnMetaWithType.columnTypeEnum.PK);
} else {
@@ -647,7 +692,8 @@ public class QueryService extends BasicService {
}
for (String fk : joinDesc.getForeignKey()) {
- String columnIdentity = (dataModelDesc.findTable(fk.substring(0, fk.indexOf("."))).getTableIdentity() + fk.substring(fk.indexOf("."))).replace('.', '#');
+ String columnIdentity = (dataModelDesc.findTable(fk.substring(0, fk.indexOf(".")))
+ .getTableIdentity() + fk.substring(fk.indexOf("."))).replace('.', '#');
if (columnMap.containsKey(columnIdentity)) {
columnMap.get(columnIdentity).getTYPE().add(ColumnMetaWithType.columnTypeEnum.FK);
} else {
@@ -660,7 +706,8 @@ public class QueryService extends BasicService {
List<ModelDimensionDesc> dimensions = dataModelDesc.getDimensions();
for (ModelDimensionDesc dimension : dimensions) {
for (String column : dimension.getColumns()) {
- String columnIdentity = (dataModelDesc.findTable(dimension.getTable()).getTableIdentity() + "." + column).replace('.', '#');
+ String columnIdentity = (dataModelDesc.findTable(dimension.getTable()).getTableIdentity() + "."
+ + column).replace('.', '#');
if (columnMap.containsKey(columnIdentity)) {
columnMap.get(columnIdentity).getTYPE().add(ColumnMetaWithType.columnTypeEnum.DIMENSION);
} else {
@@ -672,7 +719,8 @@ public class QueryService extends BasicService {
String[] measures = dataModelDesc.getMetrics();
for (String measure : measures) {
- String columnIdentity = (dataModelDesc.findTable(measure.substring(0, measure.indexOf("."))).getTableIdentity() + measure.substring(measure.indexOf("."))).replace('.', '#');
+ String columnIdentity = (dataModelDesc.findTable(measure.substring(0, measure.indexOf(".")))
+ .getTableIdentity() + measure.substring(measure.indexOf("."))).replace('.', '#');
if (columnMap.containsKey(columnIdentity)) {
columnMap.get(columnIdentity).getTYPE().add(ColumnMetaWithType.columnTypeEnum.MEASURE);
} else {
@@ -703,7 +751,7 @@ public class QueryService extends BasicService {
Connection conn = null;
Statement stat = null;
ResultSet resultSet = null;
- Boolean isAdHoc = false;
+ Boolean isPushDown = false;
List<List<String>> results = Lists.newArrayList();
List<SelectedColumnMeta> columnMetas = Lists.newArrayList();
@@ -713,7 +761,7 @@ public class QueryService extends BasicService {
// special case for prepare query.
if (BackdoorToggles.getPrepareOnly()) {
- return getPrepareOnlySqlResponse(correctedSql, conn, isAdHoc, results, columnMetas);
+ return getPrepareOnlySqlResponse(correctedSql, conn, isPushDown, results, columnMetas);
}
stat = conn.createStatement();
@@ -725,7 +773,13 @@ public class QueryService extends BasicService {
// Fill in selected column meta
for (int i = 1; i <= columnCount; ++i) {
- columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), metaData.isSearchable(i), metaData.isCurrency(i), metaData.isNullable(i), metaData.isSigned(i), metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i), metaData.getSchemaName(i), metaData.getCatalogName(i), metaData.getTableName(i), metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), metaData.getColumnTypeName(i), metaData.isReadOnly(i), metaData.isWritable(i), metaData.isDefinitelyWritable(i)));
+ columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i),
+ metaData.isSearchable(i), metaData.isCurrency(i), metaData.isNullable(i), metaData.isSigned(i),
+ metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i),
+ metaData.getSchemaName(i), metaData.getCatalogName(i), metaData.getTableName(i),
+ metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i),
+ metaData.getColumnTypeName(i), metaData.isReadOnly(i), metaData.isWritable(i),
+ metaData.isDefinitelyWritable(i)));
}
// fill in results
@@ -738,17 +792,17 @@ public class QueryService extends BasicService {
results.add(oneRow);
}
} catch (SQLException sqlException) {
- isAdHoc = AdHocUtil.doAdHocQuery(sqlRequest.getProject(), correctedSql, results, columnMetas, sqlException);
+ isPushDown = PushDownUtil.doPushDownQuery(sqlRequest.getProject(), correctedSql, results, columnMetas,
+ sqlException);
} finally {
close(resultSet, stat, conn);
}
- return getSqlResponse(isAdHoc, results, columnMetas);
+ return getSqlResponse(isPushDown, results, columnMetas);
}
- private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection conn,
- Boolean isAdHoc, List<List<String>> results, List<SelectedColumnMeta> columnMetas)
- throws SQLException {
+ private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection conn, Boolean isPushDown,
+ List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws SQLException {
CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(true);
@@ -772,7 +826,12 @@ public class QueryService extends BasicService {
String columnName = field.getKey();
BasicSqlType basicSqlType = (BasicSqlType) field.getValue();
- columnMetas.add(new SelectedColumnMeta(false, config.caseSensitive(), false, false, basicSqlType.isNullable() ? 1 : 0, true, basicSqlType.getPrecision(), columnName, columnName, null, null, null, basicSqlType.getPrecision(), basicSqlType.getScale() < 0 ? 0 : basicSqlType.getScale(), basicSqlType.getSqlTypeName().getJdbcOrdinal(), basicSqlType.getSqlTypeName().getName(), true, false, false));
+ columnMetas.add(new SelectedColumnMeta(false, config.caseSensitive(), false, false,
+ basicSqlType.isNullable() ? 1 : 0, true, basicSqlType.getPrecision(), columnName,
+ columnName, null, null, null, basicSqlType.getPrecision(),
+ basicSqlType.getScale() < 0 ? 0 : basicSqlType.getScale(),
+ basicSqlType.getSqlTypeName().getJdbcOrdinal(), basicSqlType.getSqlTypeName().getName(),
+ true, false, false));
}
} else {
@@ -782,10 +841,11 @@ public class QueryService extends BasicService {
CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(false);
}
- return getSqlResponse(isAdHoc, results, columnMetas);
+ return getSqlResponse(isPushDown, results, columnMetas);
}
- private SQLResponse getSqlResponse(Boolean isAdHoc, List<List<String>> results, List<SelectedColumnMeta> columnMetas) {
+ private SQLResponse getSqlResponse(Boolean isPushDown, List<List<String>> results,
+ List<SelectedColumnMeta> columnMetas) {
boolean isPartialResult = false;
StringBuilder cubeSb = new StringBuilder();
@@ -804,7 +864,8 @@ public class QueryService extends BasicService {
}
logger.info(logSb.toString());
- SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, false, null, isPartialResult, isAdHoc);
+ SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, false, null, isPartialResult,
+ isPushDown);
response.setTotalScanCount(QueryContext.current().getScannedRows());
response.setTotalScanBytes(QueryContext.current().getScannedBytes());
return response;
@@ -815,7 +876,8 @@ public class QueryService extends BasicService {
* @param param
* @throws SQLException
*/
- private void setParam(PreparedStatement preparedState, int index, PrepareSqlRequest.StateParam param) throws SQLException {
+ private void setParam(PreparedStatement preparedState, int index, PrepareSqlRequest.StateParam param)
+ throws SQLException {
boolean isNull = (null == param.getValue());
Class<?> clazz;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
deleted file mode 100644
index 0eff508..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.util;
-
-import java.sql.SQLException;
-import java.util.Collections;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.apache.commons.lang3.tuple.Triple;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
-import org.apache.kylin.query.routing.NoRealizationFoundException;
-import org.apache.kylin.source.adhocquery.IAdHocConverter;
-import org.apache.kylin.source.adhocquery.IAdHocRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class AdHocUtil {
- private static final Logger logger = LoggerFactory.getLogger(AdHocUtil.class);
-
- public static boolean doAdHocQuery(String project, String sql, List<List<String>> results,
- List<SelectedColumnMeta> columnMetas, SQLException sqlException) throws Exception {
-
- boolean isExpectedCause = (ExceptionUtils.getRootCause(sqlException).getClass()
- .equals(NoRealizationFoundException.class));
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
-
- if (isExpectedCause && kylinConfig.isAdhocEnabled()) {
-
- logger.info("Query failed to utilize pre-calculation, routing to other engines", sqlException);
- IAdHocRunner runner = (IAdHocRunner) ClassUtil.newInstance(kylinConfig.getAdHocRunnerClassName());
- IAdHocConverter converter = (IAdHocConverter) ClassUtil
- .newInstance(kylinConfig.getAdHocConverterClassName());
-
- runner.init(kylinConfig);
-
- logger.debug("Ad-hoc query runner {}", runner);
-
- String expandCC = restoreComputedColumnToExpr(sql, project);
- if (!StringUtils.equals(expandCC, sql)) {
- logger.info("computed column in sql is expanded to: " + expandCC);
- }
- String adhocSql = converter.convert(expandCC);
- if (!adhocSql.equals(expandCC)) {
- logger.info("the query is converted to {} according to kylin.query.ad-hoc.converter-class-name",
- adhocSql);
- }
-
- runner.executeQuery(adhocSql, results, columnMetas);
-
- return true;
- } else {
- throw sqlException;
- }
- }
-
- private final static Pattern identifierInSqlPattern = Pattern.compile(
- //find pattern like "table"."column" or "column"
- "((?<![\\p{L}_0-9\\.\\\"])(\\\"[\\p{L}_0-9]+\\\"\\.)?(\\\"[\\p{L}_0-9]+\\\")(?![\\p{L}_0-9\\.\\\"]))" + "|"
- //find pattern like table.column or column
- + "((?<![\\p{L}_0-9\\.\\\"])([\\p{L}_0-9]+\\.)?([\\p{L}_0-9]+)(?![\\p{L}_0-9\\.\\\"]))");
-
- private final static Pattern identifierInExprPattern = Pattern.compile(
- // a.b.c
- "((?<![\\p{L}_0-9\\.\\\"])([\\p{L}_0-9]+\\.)([\\p{L}_0-9]+\\.)([\\p{L}_0-9]+)(?![\\p{L}_0-9\\.\\\"]))");
-
- private final static Pattern endWithAsPattern = Pattern.compile("\\s+as\\s+$", Pattern.CASE_INSENSITIVE);
-
- public static String restoreComputedColumnToExpr(String beforeSql, String project) {
- final MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- List<DataModelDesc> dataModelDescs = metadataManager.getModels(project);
-
- String afterSql = beforeSql;
- for (DataModelDesc dataModelDesc : dataModelDescs) {
- for (ComputedColumnDesc computedColumnDesc : dataModelDesc.getComputedColumnDescs()) {
- afterSql = restoreComputedColumnToExpr(afterSql, computedColumnDesc);
- }
- }
-
- return afterSql;
- }
-
- static String restoreComputedColumnToExpr(String sql, ComputedColumnDesc computedColumnDesc) {
-
- String ccName = computedColumnDesc.getColumnName();
- List<Triple<Integer, Integer, String>> replacements = Lists.newArrayList();
- Matcher matcher = identifierInSqlPattern.matcher(sql);
-
- while (matcher.find()) {
- if (matcher.group(1) != null) { //with quote case: "TABLE"."COLUMN"
-
- String quotedColumnName = matcher.group(3);
- Preconditions.checkNotNull(quotedColumnName);
- String columnName = StringUtils.strip(quotedColumnName, "\"");
- if (!columnName.equalsIgnoreCase(ccName)) {
- continue;
- }
-
- if (matcher.group(2) != null) { // table name exist
- String quotedTableAlias = StringUtils.strip(matcher.group(2), ".");
- String tableAlias = StringUtils.strip(quotedTableAlias, "\"");
- replacements.add(Triple.of(matcher.start(1), matcher.end(1),
- replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, true)));
- } else { //only column
- if (endWithAsPattern.matcher(sql.substring(0, matcher.start(1))).find()) {
- //select DEAL_AMOUNT as "deal_amount" case
- continue;
- }
- replacements.add(Triple.of(matcher.start(1), matcher.end(1),
- replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, true)));
- }
- } else if (matcher.group(4) != null) { //without quote case: table.column or simply column
- String columnName = matcher.group(6);
- Preconditions.checkNotNull(columnName);
- if (!columnName.equalsIgnoreCase(ccName)) {
- continue;
- }
-
- if (matcher.group(5) != null) { //table name exist
- String tableAlias = StringUtils.strip(matcher.group(5), ".");
- replacements.add(Triple.of(matcher.start(4), matcher.end(4),
- replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, false)));
-
- } else { //only column
- if (endWithAsPattern.matcher(sql.substring(0, matcher.start(4))).find()) {
- //select DEAL_AMOUNT as deal_amount case
- continue;
- }
- replacements.add(Triple.of(matcher.start(4), matcher.end(4),
- replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, false)));
- }
- }
- }
-
- Collections.reverse(replacements);
- for (Triple<Integer, Integer, String> triple : replacements) {
- sql = sql.substring(0, triple.getLeft()) + "(" + triple.getRight() + ")"
- + sql.substring(triple.getMiddle());
- }
- return sql;
- }
-
- static String replaceIdentifierInExpr(String expr, String tableAlias, boolean quoted) {
- if (tableAlias == null) {
- return expr;
- }
-
- return CalciteParser.insertAliasInExpr(expr, tableAlias);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
new file mode 100644
index 0000000..5d7f47a
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.util;
+
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
+import org.apache.kylin.query.routing.NoRealizationFoundException;
+import org.apache.kylin.source.adhocquery.IPushDownConverter;
+import org.apache.kylin.source.adhocquery.IPushDownRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class PushDownUtil {
+ private static final Logger logger = LoggerFactory.getLogger(PushDownUtil.class);
+
+ public static boolean doPushDownQuery(String project, String sql, List<List<String>> results,
+ List<SelectedColumnMeta> columnMetas, SQLException sqlException) throws Exception {
+
+ boolean isExpectedCause = (ExceptionUtils.getRootCause(sqlException).getClass()
+ .equals(NoRealizationFoundException.class));
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+
+ if (isExpectedCause && kylinConfig.isPushDownEnabled()) {
+
+ logger.info("Query failed to utilize pre-calculation, routing to other engines", sqlException);
+ IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(kylinConfig.getPushDownRunnerClassName());
+ IPushDownConverter converter = (IPushDownConverter) ClassUtil
+ .newInstance(kylinConfig.getPushDownConverterClassName());
+
+ runner.init(kylinConfig);
+
+ logger.debug("Query pushdown runner {}", runner);
+
+ String expandCC = restoreComputedColumnToExpr(sql, project);
+ if (!StringUtils.equals(expandCC, sql)) {
+ logger.info("computed column in sql is expanded to: " + expandCC);
+ }
+ String adhocSql = converter.convert(expandCC);
+ if (!adhocSql.equals(expandCC)) {
+ logger.info("the query is converted to {} according to kylin.query.pushdown.converter-class-name",
+ adhocSql);
+ }
+
+ runner.executeQuery(adhocSql, results, columnMetas);
+
+ return true;
+ } else {
+ throw sqlException;
+ }
+ }
+
+ private final static Pattern identifierInSqlPattern = Pattern.compile(
+ //find pattern like "table"."column" or "column"
+ "((?<![\\p{L}_0-9\\.\\\"])(\\\"[\\p{L}_0-9]+\\\"\\.)?(\\\"[\\p{L}_0-9]+\\\")(?![\\p{L}_0-9\\.\\\"]))" + "|"
+ //find pattern like table.column or column
+ + "((?<![\\p{L}_0-9\\.\\\"])([\\p{L}_0-9]+\\.)?([\\p{L}_0-9]+)(?![\\p{L}_0-9\\.\\\"]))");
+
+ private final static Pattern identifierInExprPattern = Pattern.compile(
+ // a.b.c
+ "((?<![\\p{L}_0-9\\.\\\"])([\\p{L}_0-9]+\\.)([\\p{L}_0-9]+\\.)([\\p{L}_0-9]+)(?![\\p{L}_0-9\\.\\\"]))");
+
+ private final static Pattern endWithAsPattern = Pattern.compile("\\s+as\\s+$", Pattern.CASE_INSENSITIVE);
+
+ public static String restoreComputedColumnToExpr(String beforeSql, String project) {
+ final MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ List<DataModelDesc> dataModelDescs = metadataManager.getModels(project);
+
+ String afterSql = beforeSql;
+ for (DataModelDesc dataModelDesc : dataModelDescs) {
+ for (ComputedColumnDesc computedColumnDesc : dataModelDesc.getComputedColumnDescs()) {
+ afterSql = restoreComputedColumnToExpr(afterSql, computedColumnDesc);
+ }
+ }
+ return afterSql;
+ }
+
+ static String restoreComputedColumnToExpr(String sql, ComputedColumnDesc computedColumnDesc) {
+
+ String ccName = computedColumnDesc.getColumnName();
+ List<Triple<Integer, Integer, String>> replacements = Lists.newArrayList();
+ Matcher matcher = identifierInSqlPattern.matcher(sql);
+
+ while (matcher.find()) {
+ if (matcher.group(1) != null) { //with quote case: "TABLE"."COLUMN"
+
+ String quotedColumnName = matcher.group(3);
+ Preconditions.checkNotNull(quotedColumnName);
+ String columnName = StringUtils.strip(quotedColumnName, "\"");
+ if (!columnName.equalsIgnoreCase(ccName)) {
+ continue;
+ }
+
+ if (matcher.group(2) != null) { // table name exist
+ String quotedTableAlias = StringUtils.strip(matcher.group(2), ".");
+ String tableAlias = StringUtils.strip(quotedTableAlias, "\"");
+ replacements.add(Triple.of(matcher.start(1), matcher.end(1),
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, true)));
+ } else { //only column
+ if (endWithAsPattern.matcher(sql.substring(0, matcher.start(1))).find()) {
+ //select DEAL_AMOUNT as "deal_amount" case
+ continue;
+ }
+ replacements.add(Triple.of(matcher.start(1), matcher.end(1),
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, true)));
+ }
+ } else if (matcher.group(4) != null) { //without quote case: table.column or simply column
+ String columnName = matcher.group(6);
+ Preconditions.checkNotNull(columnName);
+ if (!columnName.equalsIgnoreCase(ccName)) {
+ continue;
+ }
+
+ if (matcher.group(5) != null) { //table name exist
+ String tableAlias = StringUtils.strip(matcher.group(5), ".");
+ replacements.add(Triple.of(matcher.start(4), matcher.end(4),
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, false)));
+
+ } else { //only column
+ if (endWithAsPattern.matcher(sql.substring(0, matcher.start(4))).find()) {
+ //select DEAL_AMOUNT as deal_amount case
+ continue;
+ }
+ replacements.add(Triple.of(matcher.start(4), matcher.end(4),
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, false)));
+ }
+ }
+ }
+
+ Collections.reverse(replacements);
+ for (Triple<Integer, Integer, String> triple : replacements) {
+ sql = sql.substring(0, triple.getLeft()) + "(" + triple.getRight() + ")"
+ + sql.substring(triple.getMiddle());
+ }
+ return sql;
+ }
+
+ static String replaceIdentifierInExpr(String expr, String tableAlias, boolean quoted) {
+ if (tableAlias == null) {
+ return expr;
+ }
+
+ return CalciteParser.insertAliasInExpr(expr, tableAlias);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
deleted file mode 100644
index 05b135a..0000000
--- a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.rest.util;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class AdHocUtilTest {
-
- @Test
- public void testReplaceIdentifierInExpr() {
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("x * y", null, false);
- Assert.assertEquals("x * y", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("x_3 * y_3", "b_2", false);
- Assert.assertEquals("b_2.x_3 * b_2.y_3", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("substr(x,1,3)>y", "c", true);
- Assert.assertEquals("substr(c.x,1,3)>c.y", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", "c", true);
- Assert.assertEquals("strcmp(substr(c.x,1,3),c.y)", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, true);
- Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, false);
- Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
- }
- }
-
- @Test
- public void testRestoreComputedColumnToExpr() {
-
- ComputedColumnDesc computedColumnDesc = mock(ComputedColumnDesc.class);
- when(computedColumnDesc.getColumnName()).thenReturn("DEAL_AMOUNT");
- when(computedColumnDesc.getExpression()).thenReturn("price * number");
-
- {
- String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", computedColumnDesc);
- Assert.assertEquals("select (price * number) from DB.TABLE group by date order by (price * number)", ret);
- }
- {
- String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select DEAL_AMOUNT as DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT",
- computedColumnDesc);
- Assert.assertEquals(
- "select (price * number) as DEAL_AMOUNT from DB.TABLE group by date order by (price * number)",
- ret);
- }
- {
- String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select \"DEAL_AMOUNT\" AS deal_amount from DB.TABLE group by date order by DEAL_AMOUNT",
- computedColumnDesc);
- Assert.assertEquals(
- "select (price * number) AS deal_amount from DB.TABLE group by date order by (price * number)",
- ret);
- }
- {
- String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select x.DEAL_AMOUNT AS deal_amount from DB.TABLE x group by date order by x.DEAL_AMOUNT",
- computedColumnDesc);
- Assert.assertEquals(
- "select (x.price * x.number) AS deal_amount from DB.TABLE x group by date order by (x.price * x.number)",
- ret);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/test/java/org/apache/kylin/rest/util/PushDownUtilTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/PushDownUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/PushDownUtilTest.java
new file mode 100644
index 0000000..5302a70
--- /dev/null
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/PushDownUtilTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.rest.util;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class PushDownUtilTest {
+
+ @Test
+ public void testReplaceIdentifierInExpr() {
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("x * y", null, false);
+ Assert.assertEquals("x * y", ret);
+ }
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("x_3 * y_3", "b_2", false);
+ Assert.assertEquals("b_2.x_3 * b_2.y_3", ret);
+ }
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("substr(x,1,3)>y", "c", true);
+ Assert.assertEquals("substr(c.x,1,3)>c.y", ret);
+ }
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", "c", true);
+ Assert.assertEquals("strcmp(substr(c.x,1,3),c.y)", ret);
+ }
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, true);
+ Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
+ }
+ {
+ String ret = PushDownUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, false);
+ Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
+ }
+ }
+
+ @Test
+ public void testRestoreComputedColumnToExpr() {
+
+ ComputedColumnDesc computedColumnDesc = mock(ComputedColumnDesc.class);
+ when(computedColumnDesc.getColumnName()).thenReturn("DEAL_AMOUNT");
+ when(computedColumnDesc.getExpression()).thenReturn("price * number");
+
+ {
+ String ret = PushDownUtil.restoreComputedColumnToExpr(
+ "select DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", computedColumnDesc);
+ Assert.assertEquals("select (price * number) from DB.TABLE group by date order by (price * number)", ret);
+ }
+ {
+ String ret = PushDownUtil.restoreComputedColumnToExpr(
+ "select DEAL_AMOUNT as DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT",
+ computedColumnDesc);
+ Assert.assertEquals(
+ "select (price * number) as DEAL_AMOUNT from DB.TABLE group by date order by (price * number)",
+ ret);
+ }
+ {
+ String ret = PushDownUtil.restoreComputedColumnToExpr(
+ "select \"DEAL_AMOUNT\" AS deal_amount from DB.TABLE group by date order by DEAL_AMOUNT",
+ computedColumnDesc);
+ Assert.assertEquals(
+ "select (price * number) AS deal_amount from DB.TABLE group by date order by (price * number)",
+ ret);
+ }
+ {
+ String ret = PushDownUtil.restoreComputedColumnToExpr(
+ "select x.DEAL_AMOUNT AS deal_amount from DB.TABLE x group by date order by x.DEAL_AMOUNT",
+ computedColumnDesc);
+ Assert.assertEquals(
+ "select (x.price * x.number) AS deal_amount from DB.TABLE x group by date order by (x.price * x.number)",
+ ret);
+ }
+ }
+}
[31/50] [abbrv] kylin git commit: minor,
fix computed column and adhoc issues
Posted by ka...@apache.org.
minor, fix computed column and adhoc issues
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cf1ba956
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cf1ba956
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cf1ba956
Branch: refs/heads/KYLIN-2606
Commit: cf1ba9566c2b8ae8aa0f69dc8c27293bee156ab5
Parents: 5fb4f35
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Jun 30 15:50:09 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 30 16:00:37 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/MetadataManager.java | 32 +-------
.../kylin/metadata/model/DataModelDesc.java | 80 +++++++++-----------
.../metadata/model/ModelDimensionDesc.java | 12 ++-
.../metadata/model/tool/CalciteParser.java | 9 ++-
.../kylin/model/tool/CalciteParserTest.java | 26 +++++--
.../query/util/ConvertToComputedColumn.java | 36 +++------
.../apache/kylin/rest/service/AclService.java | 4 +-
.../apache/kylin/rest/service/ModelService.java | 2 +-
.../org/apache/kylin/rest/util/AdHocUtil.java | 58 +++++---------
.../apache/kylin/rest/util/AdHocUtilTest.java | 23 +++---
.../kylin/rest/service/ModelServiceTest.java | 4 +-
11 files changed, 116 insertions(+), 170 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 96ae294..6c4faa3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -25,7 +25,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -40,7 +39,6 @@ import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.ExternalFilterDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -120,26 +118,6 @@ public class MetadataManager {
// name => External Filter Desc
private CaseInsensitiveStringCache<ExternalFilterDesc> extFilterMap;
- public static class CCInfo {
- private ComputedColumnDesc computedColumnDesc;
- private Set<DataModelDesc> dataModelDescs;
-
- public CCInfo(ComputedColumnDesc computedColumnDesc, Set<DataModelDesc> dataModelDescs) {
- this.computedColumnDesc = computedColumnDesc;
- this.dataModelDescs = dataModelDescs;
- }
-
- public ComputedColumnDesc getComputedColumnDesc() {
- return computedColumnDesc;
- }
-
- public Set<DataModelDesc> getDataModelDescs() {
- return dataModelDescs;
- }
- }
-
- private Map<String, CCInfo> ccInfoMap = Maps.newHashMap();// this is to check any two models won't conflict computed columns
-
private MetadataManager(KylinConfig config) throws IOException {
init(config);
}
@@ -181,9 +159,6 @@ public class MetadataManager {
return Collections.unmodifiableMap(srcTableMap.getMap());
}
- public Map<String, CCInfo> getCcInfoMap() {
- return ccInfoMap;
- }
public Map<String, TableExtDesc> listAllTableExdMap() {
return srcTableExdMap.getMap();
@@ -373,7 +348,6 @@ public class MetadataManager {
@Override
public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
- ccInfoMap.clear();
for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) {
reloadDataModelDescAt(DataModelDesc.concatResourcePath(model));
}
@@ -546,7 +520,7 @@ public class MetadataManager {
return new ArrayList<>(dataModelDescMap.values());
}
- public List<DataModelDesc> getModels(String projectName) throws IOException {
+ public List<DataModelDesc> getModels(String projectName) {
ProjectInstance projectInstance = ProjectManager.getInstance(config).getProject(projectName);
ArrayList<DataModelDesc> ret = new ArrayList<>();
@@ -618,7 +592,7 @@ public class MetadataManager {
DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER);
if (!dataModelDesc.isDraft())
- dataModelDesc.init(config, this.getAllTablesMap(), this.ccInfoMap);
+ dataModelDesc.init(config, this.getAllTablesMap(), listDataModels());
dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc);
return dataModelDesc;
@@ -669,7 +643,7 @@ public class MetadataManager {
private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) throws IOException {
if (!dataModelDesc.isDraft())
- dataModelDesc.init(config, this.getAllTablesMap(), this.ccInfoMap);
+ dataModelDesc.init(config, this.getAllTablesMap(), listDataModels());
String path = dataModelDesc.getResourcePath();
getStore().putResource(path, dataModelDesc, MODELDESC_SERIALIZER);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 341f36e..bc35e2a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -18,8 +18,6 @@
package org.apache.kylin.metadata.model;
-import static org.apache.kylin.metadata.MetadataManager.CCInfo;
-
import java.io.Serializable;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -38,6 +36,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.JoinsTree.Chain;
@@ -49,6 +48,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
@@ -234,6 +234,7 @@ public class DataModelDesc extends RootPersistentEntity {
return factTableRefs.contains(t);
}
+ //TODO: different from isFactTable(TableRef t)
public boolean isFactTable(String fullTableName) {
for (TableRef t : factTableRefs) {
if (t.getTableIdentity().equals(fullTableName))
@@ -332,7 +333,7 @@ public class DataModelDesc extends RootPersistentEntity {
throw new IllegalArgumentException("Table not found by " + tableIdentity + " in model " + name);
}
- public void init(KylinConfig config, Map<String, TableDesc> originalTables, Map<String, CCInfo> ccInfoMap) {
+ public void init(KylinConfig config, Map<String, TableDesc> originalTables, List<DataModelDesc> dataModelDescs) {
//tweak the tables according to Computed Columns defined in model
Map<String, TableDesc> tables = Maps.newHashMap();
for (Map.Entry<String, TableDesc> entry : originalTables.entrySet()) {
@@ -351,12 +352,12 @@ public class DataModelDesc extends RootPersistentEntity {
initJoinsTree();
initDimensionsAndMetrics();
initPartitionDesc();
- initComputedColumns(ccInfoMap);
+ initComputedColumns(dataModelDescs);
initFilterCondition();
boolean reinit = validate();
if (reinit) { // model slightly changed by validate() and must init() again
- init(config, tables, ccInfoMap);
+ init(config, tables, dataModelDescs);
}
}
@@ -471,38 +472,43 @@ public class DataModelDesc extends RootPersistentEntity {
this.partitionDesc.init(this);
}
- private void initComputedColumns(Map<String, CCInfo> ccInfoMap) {
- if (ccInfoMap == null) {
- logger.error("cc info map is null");
+ private void initComputedColumns(List<DataModelDesc> allDataModelDescs) {
+ Preconditions.checkNotNull(allDataModelDescs);
+
+ List<Pair<ComputedColumnDesc, DataModelDesc>> existingCCs = Lists.newArrayList();
+
+ for (DataModelDesc dataModelDesc : allDataModelDescs) {
+ if (!StringUtils.equals(dataModelDesc.getName(), this.getName())) {
+ for (ComputedColumnDesc cc : dataModelDesc.getComputedColumnDescs()) {
+ existingCCs.add(Pair.newPair(cc, dataModelDesc));
+ }
+ }
}
- Set<String> ccSet = Sets.newHashSet();//make sure cc name does not duplicate within this model
+ for (ComputedColumnDesc newCC : this.computedColumnDescs) {
- for (ComputedColumnDesc thisCCDesc : this.computedColumnDescs) {
- thisCCDesc.init();
- String thisCCName = thisCCDesc.getFullName();
+ newCC.init();
+ final String newCCName = newCC.getFullName();
+ final String newCCColumnName = newCC.getColumnName();
- if (ccSet.contains(thisCCName)) {
- throw new IllegalArgumentException(String.format(
- "More than one computed column named %s exist in model %s", thisCCName, this.getName()));
- } else {
- ccSet.add(thisCCName);
- }
+ for (Pair<ComputedColumnDesc, DataModelDesc> pair : existingCCs) {
+ DataModelDesc dataModelDesc = pair.getSecond();
+ ComputedColumnDesc cc = pair.getFirst();
- CCInfo other = ccInfoMap.get(thisCCName);
+ if (StringUtils.equals(cc.getFullName(), newCCName) && !(cc.equals(newCC))) {
+ throw new IllegalArgumentException(
+ String.format("Computed column named %s is defined differently in model %s", newCCName,
+ dataModelDesc.getName()));
+ }
- if (other == null) {
- checkSameCCDefinition(ccInfoMap, thisCCDesc, thisCCName);
- ccInfoMap.put(thisCCName, new CCInfo(thisCCDesc, Sets.<DataModelDesc> newHashSet(this)));
- } else if (other.getDataModelDescs().size() == 1 && other.getDataModelDescs().contains(this)) {
- ccInfoMap.put(thisCCName, new CCInfo(thisCCDesc, Sets.<DataModelDesc> newHashSet(this)));
- } else if (other.getComputedColumnDesc().equals(thisCCDesc)) {
- other.getDataModelDescs().add(this);
- } else {
- throw new IllegalStateException(String.format(
- "Computed column named %s is already defined in other models: %s. Please change another name, or try to keep consistent definition", //
- thisCCName, other.getDataModelDescs()));
+ if (isTwoCCDefinitionEquals(cc.getExpression(), newCC.getExpression())
+ && !StringUtils.equals(cc.getColumnName(), newCCColumnName)) {
+ throw new IllegalArgumentException(String.format(
+ "Duplicate expression of %s with computed column %s in model %s, keep same computed column name could suppress this",
+ newCCColumnName, cc.getColumnName(), dataModelDesc.getName()));
+ }
}
+ existingCCs.add(Pair.newPair(newCC, this));
}
}
@@ -548,20 +554,6 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
- private void checkSameCCDefinition(Map<String, CCInfo> ccInfoMap, ComputedColumnDesc thisCCDesc,
- String thisCCName) {
- //check whether two computer columns's definition is the same.
- for (CCInfo sysCCInfo : ccInfoMap.values()) {
- String definition0 = thisCCDesc.getExpression();
- String definition1 = sysCCInfo.getComputedColumnDesc().getExpression();
- if (isTwoCCDefinitionEquals(definition0, definition1)) {
- throw new IllegalStateException(String.format(
- "Computed column %s's definition: %s is already defined in other models: %s. Please change another definition, or try to keep consistent definition",
- thisCCName, definition0, sysCCInfo.getDataModelDescs()));
- }
- }
- }
-
private boolean isTwoCCDefinitionEquals(String definition0, String definition1) {
definition0 = definition0.replaceAll("\\s*", "");
definition1 = definition1.replaceAll("\\s*", "");
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
index c0ddbad..283c39c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
@@ -31,7 +31,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class ModelDimensionDesc implements Serializable {
private static final long serialVersionUID = 1L;
-
+
@JsonProperty("table")
private String table;
@JsonProperty("columns")
@@ -58,12 +58,18 @@ public class ModelDimensionDesc implements Serializable {
if (columns != null) {
StringUtil.toUpperCaseArray(columns, columns);
}
-
+
if (model != null) {
table = model.findTable(table).getAlias();
if (columns != null) {
for (int i = 0; i < columns.length; i++) {
- columns[i] = model.findColumn(table, columns[i]).getName();
+ TblColRef column = model.findColumn(table, columns[i]);
+
+ if (column.getColumnDesc().isComputedColumnn() && !model.isFactTable(column.getTableRef())) {
+ throw new RuntimeException("Computed Column on lookup table is not allowed");
+ }
+
+ columns[i] = column.getName();
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
index 47a90bf..9b80969 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
@@ -22,7 +22,9 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
+import java.util.Set;
+import com.google.common.collect.Lists;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlLiteral;
@@ -37,7 +39,7 @@ import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.commons.lang3.tuple.Pair;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
public class CalciteParser {
public static SqlNode parse(String sql) throws SqlParseException {
@@ -151,19 +153,20 @@ public class CalciteParser {
String sql = prefix + expr + suffix;
SqlNode sqlNode = getOnlySelectNode(sql);
- final List<SqlIdentifier> sqlIdentifiers = Lists.newArrayList();
+ final Set<SqlIdentifier> s = Sets.newHashSet();
SqlVisitor sqlVisitor = new SqlBasicVisitor() {
@Override
public Object visit(SqlIdentifier id) {
if (id.names.size() > 1) {
throw new IllegalArgumentException("SqlIdentifier " + id + " contains DB/Table name");
}
- sqlIdentifiers.add(id);
+ s.add(id);
return null;
}
};
sqlNode.accept(sqlVisitor);
+ List<SqlIdentifier> sqlIdentifiers = Lists.newArrayList(s);
Collections.sort(sqlIdentifiers, new Comparator<SqlIdentifier>() {
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
index acf7b5a..81c1c97 100644
--- a/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
@@ -18,12 +18,13 @@
package org.apache.kylin.model.tool;
+import static org.junit.Assert.assertEquals;
+
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -38,13 +39,13 @@ public class CalciteParserTest {
@Test
public void testNoTableNameExists() throws SqlParseException {
String expr1 = "a + b";
- Assert.assertEquals("x.a + x.b", CalciteParser.insertAliasInExpr(expr1, "x"));
+ assertEquals("x.a + x.b", CalciteParser.insertAliasInExpr(expr1, "x"));
String expr2 = "a + year(b)";
- Assert.assertEquals("x.a + year(x.b)", CalciteParser.insertAliasInExpr(expr2, "x"));
+ assertEquals("x.a + year(x.b)", CalciteParser.insertAliasInExpr(expr2, "x"));
String expr3 = "a + hiveudf(b)";
- Assert.assertEquals("x.a + hiveudf(x.b)", CalciteParser.insertAliasInExpr(expr3, "x"));
+ assertEquals("x.a + hiveudf(x.b)", CalciteParser.insertAliasInExpr(expr3, "x"));
}
@Test
@@ -75,6 +76,17 @@ public class CalciteParserTest {
}
@Test
+ public void testCasewhen() {
+ String expr = "(CASE LSTG_FORMAT_NAME WHEN 'Auction' THEN 'x' WHEN 'y' THEN '222' ELSE 'z' END)";
+ String alias = "TEST_KYLIN_FACT";
+ String s = CalciteParser.insertAliasInExpr(expr, alias);
+ System.out.println(s);
+ assertEquals(
+ "(CASE TEST_KYLIN_FACT.LSTG_FORMAT_NAME WHEN 'Auction' THEN 'x' WHEN 'y' THEN '222' ELSE 'z' END)",
+ s);
+ }
+
+ @Test
public void testPos() throws SqlParseException {
String[] sqls = new String[] { "select \n a \n + \n b \n from t", //
"select\na\n+\nb\nfrom t", //
@@ -104,8 +116,8 @@ public class CalciteParserTest {
SqlNode sn2 = CalciteParser.getOnlySelectNode(sql2);
SqlNode sn3 = CalciteParser.getOnlySelectNode(sql3);
- Assert.assertEquals(true, CalciteParser.isNodeEqual(sn0, sn1));
- Assert.assertEquals(true, CalciteParser.isNodeEqual(sn0, sn2));
- Assert.assertEquals(false, CalciteParser.isNodeEqual(sn0, sn3));
+ assertEquals(true, CalciteParser.isNodeEqual(sn0, sn1));
+ assertEquals(true, CalciteParser.isNodeEqual(sn0, sn2));
+ assertEquals(false, CalciteParser.isNodeEqual(sn0, sn3));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
index 7f188c1..a9ff360 100644
--- a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++ b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -24,8 +24,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import javax.annotation.Nullable;
-
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlDynamicParam;
@@ -40,18 +38,15 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Functions;
-import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
@@ -65,7 +60,7 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
ImmutableSortedMap<String, String> computedColumns = getSortedComputedColumnWithProject(project);
String s = replaceComputedColumn(sql, computedColumns);
if (!StringUtils.equals(sql, s)) {
- logger.debug("change sql to " + s);
+ logger.debug("sql changed");
}
return s;
}
@@ -162,28 +157,15 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
}
private ImmutableSortedMap<String, String> getSortedComputedColumnWithProject(String project) {
- MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- Map<String, MetadataManager.CCInfo> ccInfoMap = metadataManager.getCcInfoMap();
- final ProjectInstance projectInstance = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(project);
-
- Iterable<MetadataManager.CCInfo> projectCCInfo = Iterables.filter(ccInfoMap.values(),
- new Predicate<MetadataManager.CCInfo>() {
- @Override
- public boolean apply(@Nullable MetadataManager.CCInfo ccInfo) {
- return Iterables.any(ccInfo.getDataModelDescs(), new Predicate<DataModelDesc>() {
- @Override
- public boolean apply(@Nullable DataModelDesc model) {
- return projectInstance.containsModel(model.getName());
- }
- });
- }
- });
Map<String, String> computedColumns = new HashMap<>();
- for (MetadataManager.CCInfo ccInfo : projectCCInfo) {
- computedColumns.put(ccInfo.getComputedColumnDesc().getColumnName(),
- ccInfo.getComputedColumnDesc().getExpression());
+
+ MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ List<DataModelDesc> dataModelDescs = metadataManager.getModels(project);
+ for (DataModelDesc dataModelDesc : dataModelDescs) {
+ for (ComputedColumnDesc computedColumnDesc : dataModelDesc.getComputedColumnDescs()) {
+ computedColumns.put(computedColumnDesc.getColumnName(), computedColumnDesc.getExpression());
+ }
}
return getMapSortedByValue(computedColumns);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index 02b8e9f..79ed4aa 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -281,9 +281,7 @@ public class AclService implements MutableAclService {
} else {
aceInfos.addAll(allAceInfos.values());
}
- } else {
- logger.warn("Get null AllAceInfos from AclRecord");
- }
+ }
List<AccessControlEntry> newAces = new ArrayList<AccessControlEntry>();
for (int i = 0; i < aceInfos.size(); i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index fa54eaa..89aee26 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -222,7 +222,7 @@ public class ModelService extends BasicService {
List<CubeInstance> cubes = cubeService.listAllCubes(null, null, modelName, true);
if (cubes != null && cubes.size() != 0) {
dataModelDesc.init(getConfig(), getMetadataManager().getAllTablesMap(),
- getMetadataManager().getCcInfoMap());
+ getMetadataManager().listDataModels());
List<String> dimCols = new ArrayList<String>();
List<String> dimAndMCols = new ArrayList<String>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
index 9694a21..0eff508 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java
@@ -21,23 +21,18 @@ package org.apache.kylin.rest.util;
import java.sql.SQLException;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import javax.annotation.Nullable;
-
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.MetadataManager.CCInfo;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.query.routing.NoRealizationFoundException;
import org.apache.kylin.source.adhocquery.IAdHocConverter;
@@ -46,8 +41,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
public class AdHocUtil {
@@ -72,9 +65,13 @@ public class AdHocUtil {
logger.debug("Ad-hoc query runner {}", runner);
String expandCC = restoreComputedColumnToExpr(sql, project);
+ if (!StringUtils.equals(expandCC, sql)) {
+ logger.info("computed column in sql is expanded to: " + expandCC);
+ }
String adhocSql = converter.convert(expandCC);
- if (!adhocSql.equals(sql)) {
- logger.info("before delegating to adhoc, the query is converted to {} ", adhocSql);
+ if (!adhocSql.equals(expandCC)) {
+ logger.info("the query is converted to {} according to kylin.query.ad-hoc.converter-class-name",
+ adhocSql);
}
runner.executeQuery(adhocSql, results, columnMetas);
@@ -98,37 +95,22 @@ public class AdHocUtil {
private final static Pattern endWithAsPattern = Pattern.compile("\\s+as\\s+$", Pattern.CASE_INSENSITIVE);
public static String restoreComputedColumnToExpr(String beforeSql, String project) {
- MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- Map<String, CCInfo> ccInfoMap = metadataManager.getCcInfoMap();
- final ProjectInstance projectInstance = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(project);
-
- Iterable<CCInfo> projectCCInfo = Iterables.filter(ccInfoMap.values(), new Predicate<CCInfo>() {
- @Override
- public boolean apply(@Nullable CCInfo ccInfo) {
- return Iterables.any(ccInfo.getDataModelDescs(), new Predicate<DataModelDesc>() {
- @Override
- public boolean apply(@Nullable DataModelDesc model) {
- return projectInstance.containsModel(model.getName());
- }
- });
- }
- });
+ final MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ List<DataModelDesc> dataModelDescs = metadataManager.getModels(project);
String afterSql = beforeSql;
- for (CCInfo ccInfo : projectCCInfo) {
- afterSql = restoreComputedColumnToExpr(afterSql, ccInfo);
+ for (DataModelDesc dataModelDesc : dataModelDescs) {
+ for (ComputedColumnDesc computedColumnDesc : dataModelDesc.getComputedColumnDescs()) {
+ afterSql = restoreComputedColumnToExpr(afterSql, computedColumnDesc);
+ }
}
- if (!StringUtils.equals(beforeSql, afterSql)) {
- logger.info("computed column in sql is expanded before sending to adhoc engine: " + afterSql);
- }
return afterSql;
}
- static String restoreComputedColumnToExpr(String sql, CCInfo ccInfo) {
+ static String restoreComputedColumnToExpr(String sql, ComputedColumnDesc computedColumnDesc) {
- String ccName = ccInfo.getComputedColumnDesc().getColumnName();
+ String ccName = computedColumnDesc.getColumnName();
List<Triple<Integer, Integer, String>> replacements = Lists.newArrayList();
Matcher matcher = identifierInSqlPattern.matcher(sql);
@@ -146,14 +128,14 @@ public class AdHocUtil {
String quotedTableAlias = StringUtils.strip(matcher.group(2), ".");
String tableAlias = StringUtils.strip(quotedTableAlias, "\"");
replacements.add(Triple.of(matcher.start(1), matcher.end(1),
- replaceIdentifierInExpr(ccInfo.getComputedColumnDesc().getExpression(), tableAlias, true)));
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, true)));
} else { //only column
if (endWithAsPattern.matcher(sql.substring(0, matcher.start(1))).find()) {
//select DEAL_AMOUNT as "deal_amount" case
continue;
}
replacements.add(Triple.of(matcher.start(1), matcher.end(1),
- replaceIdentifierInExpr(ccInfo.getComputedColumnDesc().getExpression(), null, true)));
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, true)));
}
} else if (matcher.group(4) != null) { //without quote case: table.column or simply column
String columnName = matcher.group(6);
@@ -164,8 +146,8 @@ public class AdHocUtil {
if (matcher.group(5) != null) { //table name exist
String tableAlias = StringUtils.strip(matcher.group(5), ".");
- replacements.add(Triple.of(matcher.start(4), matcher.end(4), replaceIdentifierInExpr(
- ccInfo.getComputedColumnDesc().getExpression(), tableAlias, false)));
+ replacements.add(Triple.of(matcher.start(4), matcher.end(4),
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), tableAlias, false)));
} else { //only column
if (endWithAsPattern.matcher(sql.substring(0, matcher.start(4))).find()) {
@@ -173,7 +155,7 @@ public class AdHocUtil {
continue;
}
replacements.add(Triple.of(matcher.start(4), matcher.end(4),
- replaceIdentifierInExpr(ccInfo.getComputedColumnDesc().getExpression(), null, false)));
+ replaceIdentifierInExpr(computedColumnDesc.getExpression(), null, false)));
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
index 724844c..05b135a 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.kylin.rest.util;
-import static org.apache.kylin.metadata.MetadataManager.CCInfo;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -62,36 +61,34 @@ public class AdHocUtilTest {
when(computedColumnDesc.getColumnName()).thenReturn("DEAL_AMOUNT");
when(computedColumnDesc.getExpression()).thenReturn("price * number");
- CCInfo ccInfo = mock(CCInfo.class);
- when(ccInfo.getComputedColumnDesc()).thenReturn(computedColumnDesc);
-
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
- Assert.assertEquals(
- "select (price * number) from DB.TABLE group by date order by (price * number)",
- ret);
+ "select DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", computedColumnDesc);
+ Assert.assertEquals("select (price * number) from DB.TABLE group by date order by (price * number)", ret);
}
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select DEAL_AMOUNT as DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
+ "select DEAL_AMOUNT as DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT",
+ computedColumnDesc);
Assert.assertEquals(
"select (price * number) as DEAL_AMOUNT from DB.TABLE group by date order by (price * number)",
ret);
}
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select \"DEAL_AMOUNT\" AS deal_amount from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
+ "select \"DEAL_AMOUNT\" AS deal_amount from DB.TABLE group by date order by DEAL_AMOUNT",
+ computedColumnDesc);
Assert.assertEquals(
"select (price * number) AS deal_amount from DB.TABLE group by date order by (price * number)",
ret);
}
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
- "select x.DEAL_AMOUNT AS deal_amount from DB.TABLE x group by date order by x.DEAL_AMOUNT", ccInfo);
+ "select x.DEAL_AMOUNT AS deal_amount from DB.TABLE x group by date order by x.DEAL_AMOUNT",
+ computedColumnDesc);
Assert.assertEquals(
- "select (x.price * x.number) AS deal_amount from DB.TABLE x group by date order by (x.price * x.number)",
- ret);
+ "select (x.price * x.number) AS deal_amount from DB.TABLE x group by date order by (x.price * x.number)",
+ ret);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf1ba956/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index b78b18e..b8633fb 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -80,8 +80,8 @@ public class ModelServiceTest extends ServiceTestBase {
@Test
public void testFailureModelUpdateDueToComputedColumnConflict() throws IOException, JobException, NoSuchFieldException, IllegalAccessException {
- expectedEx.expect(IllegalStateException.class);
- expectedEx.expectMessage("Computed column named DEFAULT.TEST_KYLIN_FACT.DEAL_AMOUNT is already defined in other models: [DataModelDesc [name=ci_left_join_model], DataModelDesc [name=ci_inner_join_model]]. Please change another name, or try to keep consistent definition");
+ expectedEx.expect(IllegalArgumentException.class);
+ expectedEx.expectMessage("Computed column named DEFAULT.TEST_KYLIN_FACT.DEAL_AMOUNT is defined differently in model ci_inner_join_model");
List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default", true);
Assert.assertTrue(dataModelDescs.size() == 1);
[41/50] [abbrv] kylin git commit: minor, rename adhoc to pushdown
Posted by ka...@apache.org.
minor, rename adhoc to pushdown
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/82b578a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/82b578a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/82b578a5
Branch: refs/heads/KYLIN-2606
Commit: 82b578a561de76e93dcd0b56473bbd229e1292b9
Parents: 4f4382d
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Jul 1 12:25:21 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Sat Jul 1 12:30:52 2017 +0800
----------------------------------------------------------------------
build/smoke-test/sql/sql1.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/82b578a5/build/smoke-test/sql/sql1.json
----------------------------------------------------------------------
diff --git a/build/smoke-test/sql/sql1.json b/build/smoke-test/sql/sql1.json
index 7b90d0f..a8d01b0 100644
--- a/build/smoke-test/sql/sql1.json
+++ b/build/smoke-test/sql/sql1.json
@@ -3,7 +3,7 @@
"partial": false,
"affectedRowCount": 0,
"isException": false,
- "adHoc": false,
+ "pushDown": false,
"results": [
[
"10000"
[10/50] [abbrv] kylin git commit: update pom
Posted by ka...@apache.org.
update pom
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56a0fa5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56a0fa5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56a0fa5f
Branch: refs/heads/KYLIN-2606
Commit: 56a0fa5f1d1bebde0297612ee9a4af0354ce672e
Parents: d61f867
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jun 28 21:30:52 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 13:51:45 2017 +0800
----------------------------------------------------------------------
core-metadata/pom.xml | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/56a0fa5f/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index d45a838..9e06ecf 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -70,9 +70,6 @@
<artifactId>avatica-core</artifactId>
</exclusion>
</exclusions>
- <!--set to provided on purpose, so that dependant on metadata module will not depend on calcite unnecessarily-->
- <!--https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html-->
- <scope>provided</scope>
</dependency>
<!-- Compiled -->
[48/50] [abbrv] kylin git commit: KYLIN-1351 Use different h2
instances in tests
Posted by ka...@apache.org.
KYLIN-1351 Use different h2 instances in tests
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8d3c0ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8d3c0ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8d3c0ee
Branch: refs/heads/KYLIN-2606
Commit: f8d3c0eed2f04604abcb9e7972634755ac673e2d
Parents: ad47dc0
Author: auphyroc99 <45...@qq.com>
Authored: Thu Jul 6 16:32:15 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Jul 6 17:18:11 2017 +0800
----------------------------------------------------------------------
.../jdbc/ITJdbcSourceTableLoaderTest.java | 49 ++++++++------------
.../source/jdbc/ITJdbcTableReaderTest.java | 49 ++++++++------------
2 files changed, 40 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8d3c0ee/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
index a84dca9..977c0f4 100644
--- a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.source.jdbc;
import static org.junit.Assert.assertTrue;
@@ -5,7 +23,6 @@ import static org.junit.Assert.assertTrue;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
-import java.sql.Types;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
@@ -19,12 +36,6 @@ import org.apache.kylin.source.ISource;
import org.apache.kylin.source.ISourceMetadataExplorer;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.datagen.ModelDataGenerator;
-import org.dbunit.DatabaseUnitException;
-import org.dbunit.database.DatabaseConfig;
-import org.dbunit.dataset.datatype.DataType;
-import org.dbunit.dataset.datatype.DataTypeException;
-import org.dbunit.ext.h2.H2Connection;
-import org.dbunit.ext.h2.H2DataTypeFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -39,14 +50,14 @@ public class ITJdbcSourceTableLoaderTest extends LocalFileMetadataTestCase imple
super.createTestMetadata();
- System.setProperty("kylin.source.jdbc.connection-url", "jdbc:h2:mem:db");
+ System.setProperty("kylin.source.jdbc.connection-url", "jdbc:h2:mem:db" + "_jdbc_source_table_loader");
System.setProperty("kylin.source.jdbc.driver", "org.h2.Driver");
System.setProperty("kylin.source.jdbc.user", "sa");
System.setProperty("kylin.source.jdbc.pass", "");
config = KylinConfig.getInstanceFromEnv();
- h2Connection = DriverManager.getConnection("jdbc:h2:mem:db", "sa", "");
+ h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + "_jdbc_source_table_loader", "sa", "");
H2Database h2DB = new H2Database(h2Connection, config);
@@ -98,24 +109,4 @@ public class ITJdbcSourceTableLoaderTest extends LocalFileMetadataTestCase imple
return ISourceAware.ID_JDBC;
}
- @SuppressWarnings("deprecation")
- protected static H2Connection newH2Connection() throws DatabaseUnitException {
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
- return h2Conn;
- }
-
- public static class TestH2DataTypeFactory extends H2DataTypeFactory {
- @Override
- public DataType createDataType(int sqlType, String sqlTypeName, String tableName, String columnName)
- throws DataTypeException {
-
- if ((columnName.startsWith("COL") || columnName.startsWith("col")) && sqlType == Types.BIGINT) {
- return DataType.INTEGER;
- }
- return super.createDataType(sqlType, sqlTypeName);
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8d3c0ee/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
index 7850952..22e0b14 100644
--- a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
@@ -1,9 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.source.jdbc;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
-import java.sql.Types;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
@@ -11,12 +28,6 @@ import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.query.H2Database;
import org.apache.kylin.source.datagen.ModelDataGenerator;
-import org.dbunit.DatabaseUnitException;
-import org.dbunit.database.DatabaseConfig;
-import org.dbunit.dataset.datatype.DataType;
-import org.dbunit.dataset.datatype.DataTypeException;
-import org.dbunit.ext.h2.H2Connection;
-import org.dbunit.ext.h2.H2DataTypeFactory;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -32,14 +43,14 @@ public class ITJdbcTableReaderTest extends LocalFileMetadataTestCase implements
super.createTestMetadata();
- System.setProperty("kylin.source.jdbc.connection-url", "jdbc:h2:mem:db");
+ System.setProperty("kylin.source.jdbc.connection-url", "jdbc:h2:mem:db" + "_jdbc_table_reader");
System.setProperty("kylin.source.jdbc.driver", "org.h2.Driver");
System.setProperty("kylin.source.jdbc.user", "sa");
System.setProperty("kylin.source.jdbc.pass", "");
config = KylinConfig.getInstanceFromEnv();
- h2Connection = DriverManager.getConnection("jdbc:h2:mem:db", "sa", "");
+ h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + "_jdbc_table_reader", "sa", "");
H2Database h2DB = new H2Database(h2Connection, config);
@@ -93,24 +104,4 @@ public class ITJdbcTableReaderTest extends LocalFileMetadataTestCase implements
return ISourceAware.ID_JDBC;
}
- @SuppressWarnings("deprecation")
- protected static H2Connection newH2Connection() throws DatabaseUnitException {
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
- return h2Conn;
- }
-
- public static class TestH2DataTypeFactory extends H2DataTypeFactory {
- @Override
- public DataType createDataType(int sqlType, String sqlTypeName, String tableName, String columnName)
- throws DataTypeException {
-
- if ((columnName.startsWith("COL") || columnName.startsWith("col")) && sqlType == Types.BIGINT) {
- return DataType.INTEGER;
- }
- return super.createDataType(sqlType, sqlTypeName);
- }
- }
-
}
[21/50] [abbrv] kylin git commit: KYLIN-2694 Fix
ArrayIndexOutOfBoundsException in SparkCubingByLayer
Posted by ka...@apache.org.
KYLIN-2694 Fix ArrayIndexOutOfBoundsException in SparkCubingByLayer
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/547722d6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/547722d6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/547722d6
Branch: refs/heads/KYLIN-2606
Commit: 547722d6a35344a5675d6d6ce9b885d8fdc58fc7
Parents: 804a1aa
Author: kangkaisen <ka...@live.com>
Authored: Mon Jun 26 15:33:15 2017 +0800
Committer: kangkaisen <ka...@live.com>
Committed: Thu Jun 29 21:13:06 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/547722d6/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index f70fd30..91aa9f7 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -269,7 +269,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
saveToHDFS(allRDDs[level], vCubeDesc.getValue(), outputPath, level, confOverwrite);
allRDDs[level - 1].unpersist();
}
- allRDDs[totalLevels - 1].unpersist();
+ allRDDs[totalLevels].unpersist();
logger.info("Finished on calculating all level cuboids.");
}
[14/50] [abbrv] kylin git commit: minor,
prevent to drop job if it is stopped
Posted by ka...@apache.org.
minor, prevent to drop job if it is stopped
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8e1a7d51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8e1a7d51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8e1a7d51
Branch: refs/heads/KYLIN-2606
Commit: 8e1a7d514519bedfbadccd3ba252ca240b56137f
Parents: c8acbc6
Author: Cheng Wang <ch...@kyligence.io>
Authored: Thu Jun 29 15:52:09 2017 +0800
Committer: 成 <ch...@kyligence.io>
Committed: Thu Jun 29 16:08:04 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller2/JobControllerV2.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e1a7d51/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
index f9db6d8..a86082c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
@@ -282,8 +282,10 @@ public class JobControllerV2 extends BasicController {
JobInstance jobInstance = jobService.getJobInstance(jobId);
JobStatusEnum status = jobInstance.getStatus();
- if (status == JobStatusEnum.NEW || status == JobStatusEnum.PENDING || status == JobStatusEnum.RUNNING) {
- throw new BadRequestException("Cannot drop running job " + jobInstance.getName());
+ if (status == JobStatusEnum.NEW || status == JobStatusEnum.PENDING || status == JobStatusEnum.RUNNING
+ || status == JobStatusEnum.STOPPED) {
+ throw new BadRequestException(
+ "Cannot drop running job " + jobInstance.getName() + ", please discard it first.");
}
jobService.dropJob(jobInstance);
[19/50] [abbrv] kylin git commit: KYLIN-2688 When the model has a
ready cube, should not allow user to edit model JSON in web.
Posted by ka...@apache.org.
KYLIN-2688 When the model has a ready cube, should not allow user to edit model JSON in web.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8627e8a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8627e8a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8627e8a
Branch: refs/heads/KYLIN-2606
Commit: a8627e8a9f264cc138124f3825d490463fa58480
Parents: d851c51
Author: 10069681 <pe...@zte.com.cn>
Authored: Tue Jun 27 15:09:56 2017 +0800
Committer: chenzhx <34...@qq.com>
Committed: Thu Jun 29 19:16:02 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/models.js | 13 ++++++++-----
webapp/app/partials/models/models_tree.html | 4 ++--
2 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a8627e8a/webapp/app/js/controllers/models.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/models.js b/webapp/app/js/controllers/models.js
index fd21845..1cc0c84 100644
--- a/webapp/app/js/controllers/models.js
+++ b/webapp/app/js/controllers/models.js
@@ -127,7 +127,7 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, $routeParams, $location,
});
};
- $scope.editModel = function(model){
+ $scope.editModel = function(model, isEditJson){
var cubename = [];
var modelstate=false;
var i=0;
@@ -145,10 +145,13 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, $routeParams, $location,
})
}
- if(modelstate==false){
- $location.path("/models/edit/"+model.name);
- }
- else{
+ if (modelstate==false){
+ if (isEditJson) {
+ $location.path("/models/edit/" + model.name + "/descriptionjson");
+ } else {
+ $location.path("/models/edit/" + model.name);
+ }
+ } else {
SweetAlert.swal('Sorry','This model is still used by '+ cubename.join(','));
}
})
http://git-wip-us.apache.org/repos/asf/kylin/blob/a8627e8a/webapp/app/partials/models/models_tree.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/models/models_tree.html b/webapp/app/partials/models/models_tree.html
index c7b55a3..399f839 100644
--- a/webapp/app/partials/models/models_tree.html
+++ b/webapp/app/partials/models/models_tree.html
@@ -53,11 +53,11 @@
Action <span class="ace-icon fa fa-caret-down icon-on-right"></span>
</button>
<ul class="dropdown-menu" role="menu" ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission(model, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask))">
- <li><a ng-click="editModel(model)" title="Edit Model" style="cursor:pointer;margin-right: 8px;" >Edit</a></li>
+ <li><a ng-click="editModel(model, false)" title="Edit Model" style="cursor:pointer;margin-right: 8px;" >Edit</a></li>
<li><a ng-click="cloneModel(model)" title="Clone Model" style="cursor:pointer;margin-right: 8px;" >Clone </a></li>
<li><a ng-click="dropModel(model)" title="Drop Model" style="cursor:pointer;margin-right: 8px;">Drop</a></li>
<li ng-if="userService.hasRole('ROLE_ADMIN')">
- <a href="models/edit/{{model.name}}/descriptionjson">Edit(JSON)</a></li>
+ <a ng-click="editModel(model, true)">Edit(JSON)</a></li>
</ul>
</div>
</div>
[27/50] [abbrv] kylin git commit: minor, fix agg combination bug
Posted by ka...@apache.org.
minor, fix agg combination bug
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eca7f6b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eca7f6b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eca7f6b
Branch: refs/heads/KYLIN-2606
Commit: 5eca7f6b59454df4681deb17f361d4708a2affca
Parents: 40054dd
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 23:07:16 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 23:11:00 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/model/AggregationGroup.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eca7f6b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 064d657..0533ea1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -320,7 +320,7 @@ public class AggregationGroup implements Serializable {
normalDims.removeAll(jointDims);
combination = combination * (1L << normalDims.size());
- if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid()) {
+ if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid() && !mandatoryDims.isEmpty()) {
combination += 1;
}
combination -= 1; // not include cuboid 0
[43/50] [abbrv] kylin git commit: minor,
clean unnecessary line feed and wrong cancel btn
Posted by ka...@apache.org.
minor, clean unnecessary line feed and wrong cancel btn
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46d144a0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46d144a0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46d144a0
Branch: refs/heads/KYLIN-2606
Commit: 46d144a027a75d9cab71581da2ec9ad4f674c477
Parents: 4496135
Author: luguosheng1314 <55...@qq.com>
Authored: Tue Jul 4 16:52:55 2017 +0800
Committer: chenzhx <34...@qq.com>
Committed: Tue Jul 4 18:14:08 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/access.js | 1 -
webapp/app/js/model/cubeDescModel.js | 6 ------
webapp/app/js/model/cubeListModel.js | 1 -
webapp/app/js/model/jobListModel.js | 4 ----
webapp/app/js/model/streamingListModel.js | 6 ------
webapp/app/partials/cubes/cube_json_edit.html | 2 +-
webapp/app/partials/projects/project_create.html | 2 +-
7 files changed, 2 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/46d144a0/webapp/app/js/controllers/access.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/access.js b/webapp/app/js/controllers/access.js
index 6187f9a..61e8f3e 100644
--- a/webapp/app/js/controllers/access.js
+++ b/webapp/app/js/controllers/access.js
@@ -28,7 +28,6 @@ KylinApp.controller('AccessCtrl', function ($scope, AccessService, MessageServic
"<li>CUBE ADMIN: Full access to cube and jobs, including access management.</li></ul></div>";
$scope.authorities = null;
-
AuthenticationService.authorities({}, function (authorities) {
$scope.authorities = authorities.stringList;
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/46d144a0/webapp/app/js/model/cubeDescModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeDescModel.js b/webapp/app/js/model/cubeDescModel.js
index d0f5250..b9b95da 100644
--- a/webapp/app/js/model/cubeDescModel.js
+++ b/webapp/app/js/model/cubeDescModel.js
@@ -19,8 +19,6 @@
KylinApp.service('CubeDescModel', function (kylinConfig) {
this.cubeMetaFrame = {};
-
- //
this.createNew = function (defaultPara) {
var cubeMeta = {
"name": "",
@@ -100,10 +98,6 @@ KylinApp.service('CubeDescModel', function (kylinConfig) {
"joint_dims" : [ ]//2 dim array
}
}
-
return group;
}
-
-
-
})
http://git-wip-us.apache.org/repos/asf/kylin/blob/46d144a0/webapp/app/js/model/cubeListModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeListModel.js b/webapp/app/js/model/cubeListModel.js
index 39853c0..b08ddbe 100755
--- a/webapp/app/js/model/cubeListModel.js
+++ b/webapp/app/js/model/cubeListModel.js
@@ -47,7 +47,6 @@ KylinApp.service('CubeList',function(CubeService,$q,AccessService){
defer.reject("Failed to load cubes");
});
return defer.promise;
-
};
this.removeCube = function(cube){
http://git-wip-us.apache.org/repos/asf/kylin/blob/46d144a0/webapp/app/js/model/jobListModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/jobListModel.js b/webapp/app/js/model/jobListModel.js
index ee75d2f..95e0be3 100755
--- a/webapp/app/js/model/jobListModel.js
+++ b/webapp/app/js/model/jobListModel.js
@@ -25,7 +25,6 @@ KylinApp.service('JobList',function(JobService,$q){
this.jobs={};
this.list = function(jobRequest){
-
var defer = $q.defer();
JobService.list(jobRequest, function (jobs) {
angular.forEach(jobs, function (job) {
@@ -45,13 +44,10 @@ KylinApp.service('JobList',function(JobService,$q){
},function(){
defer.reject("failed to load jobs");
});
-
return defer.promise;
-
};
this.removeAll = function(){
_this.jobs={};
};
-
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/46d144a0/webapp/app/js/model/streamingListModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/streamingListModel.js b/webapp/app/js/model/streamingListModel.js
index 113ffe6..d397b8b 100644
--- a/webapp/app/js/model/streamingListModel.js
+++ b/webapp/app/js/model/streamingListModel.js
@@ -20,15 +20,10 @@ KylinApp.service('StreamingList', function (CubeService, $q, AccessService, Stre
var _this = this;
this.streamingConfigs = [];
this.kafkaConfigs = [];
-
-
this.list = function () {
var defer = $q.defer();
-
var streamingPromises = [];
var kafkaPromises = [];
-
-
kafkaPromises.push(StreamingService.getKfkConfig({}, function (kfkConfigs) {
_this.kafkaConfigs = kfkConfigs;
},function(){
@@ -47,7 +42,6 @@ KylinApp.service('StreamingList', function (CubeService, $q, AccessService, Stre
defer.resolve("failed");
})
return defer.promise;
-
};
this.checkCubeExist = function(cubeName){
http://git-wip-us.apache.org/repos/asf/kylin/blob/46d144a0/webapp/app/partials/cubes/cube_json_edit.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cube_json_edit.html b/webapp/app/partials/cubes/cube_json_edit.html
index ac08d1d..feb81c1 100644
--- a/webapp/app/partials/cubes/cube_json_edit.html
+++ b/webapp/app/partials/cubes/cube_json_edit.html
@@ -39,7 +39,7 @@
}" ng-model="state.cubeSchema">
</div>
</section>
- <button class="btn btn-primary" ng-disabled="cube_form.$invalid" ng-click="state.project = projectModel.selectedProject;saveCube()">
+ <button class="btn btn-primary" style="float:right;margin-bottom: 10px;" ng-disabled="cube_form.$invalid" ng-click="state.project = projectModel.selectedProject;saveCube()">
Save
</button>
</form>
http://git-wip-us.apache.org/repos/asf/kylin/blob/46d144a0/webapp/app/partials/projects/project_create.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/projects/project_create.html b/webapp/app/partials/projects/project_create.html
index 2dc76da..ceba015 100644
--- a/webapp/app/partials/projects/project_create.html
+++ b/webapp/app/partials/projects/project_create.html
@@ -81,7 +81,7 @@
</div>
</div>
<div class="modal-footer">
- <button class="btn btn-primary" ng-click="cancel()">Close</button>
+ <button class="btn" ng-click="cancel()">Close</button>
<button class="btn btn-success" ng-click="createOrUpdate()" ng-disabled="proj_create_form.$invalid">
Submit
</button>
[26/50] [abbrv] kylin git commit: minor,
fix bug "skip all steps after resume"
Posted by ka...@apache.org.
minor, fix bug "skip all steps after resume"
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7f9dab7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7f9dab7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7f9dab7
Branch: refs/heads/KYLIN-2606
Commit: e7f9dab77f087ce1701974abaae1b8f541448de4
Parents: 5eca7f6
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 23:08:27 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 23:11:00 2017 +0800
----------------------------------------------------------------------
.../kylin/job/execution/DefaultChainedExecutable.java | 1 +
.../org/apache/kylin/job/execution/ExecutableManager.java | 9 +++++++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7f9dab7/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index ae129ab..4f5c02b 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -75,6 +75,7 @@ public class DefaultChainedExecutable extends AbstractExecutable implements Chai
final long endTime = getEndTime();
if (endTime > 0) {
long interruptTime = System.currentTimeMillis() - endTime + getInterruptTime();
+ info.putAll(getManager().getJobOutput(getId()).getInfo());
info.put(START_TIME, Long.toString(startTime));
info.put(INTERRUPT_TIME, Long.toString(interruptTime));
getManager().updateJobOutput(getId(), ExecutableState.RUNNING, info, null);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7f9dab7/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index f16cfde..f8fb980 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -389,6 +389,15 @@ public class ExecutableManager {
updateJobOutput(jobId, ExecutableState.STOPPED, null, null);
}
+ public ExecutableOutputPO getJobOutput(String jobId) {
+ try {
+ return executableDao.getJobOutput(jobId);
+ } catch (PersistentException e) {
+ logger.error("Can't get output of Job " + jobId);
+ throw new RuntimeException(e);
+ }
+ }
+
public void updateJobOutput(String jobId, ExecutableState newStatus, Map<String, String> info, String output) {
try {
final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
[49/50] [abbrv] kylin git commit: drop jasper dependency in kylin-it
Posted by ka...@apache.org.
drop jasper dependency in kylin-it
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d509542f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d509542f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d509542f
Branch: refs/heads/KYLIN-2606
Commit: d509542fca8f5b6e3af6383bb34f6bf33dcf2467
Parents: f8d3c0e
Author: Li Yang <li...@apache.org>
Authored: Thu Jul 6 17:37:55 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Jul 6 17:37:55 2017 +0800
----------------------------------------------------------------------
kylin-it/pom.xml | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d509542f/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index e82867f..e37494a 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -209,6 +209,14 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -229,6 +237,14 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
[29/50] [abbrv] kylin git commit: minor,
make calcite work with unicode
Posted by ka...@apache.org.
minor, make calcite work with unicode
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f52eb0c5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f52eb0c5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f52eb0c5
Branch: refs/heads/KYLIN-2606
Commit: f52eb0c5c667b655d9a20f6226af2a3f5cbe0f4e
Parents: 2546e00
Author: Li Yang <li...@apache.org>
Authored: Fri Jun 30 13:42:21 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 30 13:46:14 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 21 ++++++++++++++++++++
.../apache/kylin/query/ITCombinationTest.java | 3 +--
.../org/apache/kylin/query/KylinTestBase.java | 4 ++--
.../kylin/query/schema/OLAPSchemaFactory.java | 19 ------------------
4 files changed, 24 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 1298807..cc08056 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -28,6 +28,7 @@ import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringReader;
import java.net.URL;
+import java.nio.ByteOrder;
import java.nio.charset.Charset;
import java.util.Enumeration;
import java.util.Map;
@@ -58,6 +59,26 @@ public class KylinConfig extends KylinConfigBase {
// thread-local instances, will override SYS_ENV_INSTANCE
private static transient ThreadLocal<KylinConfig> THREAD_ENV_INSTANCE = new ThreadLocal<>();
+
+ static {
+ /*
+ * Make Calcite to work with Unicode.
+ *
+ * Sets default char set for string literals in SQL and row types of
+ * RelNode. This is more a label used to compare row type equality. For
+ * both SQL string and row record, they are passed to Calcite in String
+ * object and does not require additional codec.
+ *
+ * Ref SaffronProperties.defaultCharset
+ * Ref SqlUtil.translateCharacterSetName()
+ * Ref NlsString constructor()
+ */
+ // copied from org.apache.calcite.util.ConversionUtil.NATIVE_UTF16_CHARSET_NAME
+ String NATIVE_UTF16_CHARSET_NAME = (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) ? "UTF-16BE" : "UTF-16LE";
+ System.setProperty("saffron.default.charset", NATIVE_UTF16_CHARSET_NAME);
+ System.setProperty("saffron.default.nationalcharset", NATIVE_UTF16_CHARSET_NAME);
+ System.setProperty("saffron.default.collation.name", NATIVE_UTF16_CHARSET_NAME + "$en_US");
+ }
public static KylinConfig getInstanceFromEnv() {
synchronized (KylinConfig.class) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 0bed5ed..5051db1 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -18,7 +18,6 @@
package org.apache.kylin.query;
-import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
@@ -43,7 +42,7 @@ public class ITCombinationTest extends ITKylinQueryTest {
private static final Logger logger = LoggerFactory.getLogger(ITCombinationTest.class);
@BeforeClass
- public static void setUp() throws SQLException {
+ public static void setUp() {
Map<RealizationType, Integer> priorities = Maps.newHashMap();
priorities.put(RealizationType.HYBRID, 0);
priorities.put(RealizationType.CUBE, 0);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index a05d0c3..af05237 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -21,9 +21,9 @@ package org.apache.kylin.query;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -166,7 +166,7 @@ public class KylinTestBase {
}
public static String getTextFromFile(File file) throws IOException {
- BufferedReader reader = new BufferedReader(new FileReader(file));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
String line = null;
StringBuilder stringBuilder = new StringBuilder();
String ls = System.getProperty("line.separator");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
index bfe49cf..c098b23 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
@@ -31,7 +31,6 @@ import java.util.Map.Entry;
import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.SchemaFactory;
import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.util.ConversionUtil;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -50,24 +49,6 @@ import com.google.common.collect.Maps;
public class OLAPSchemaFactory implements SchemaFactory {
public static final Logger logger = LoggerFactory.getLogger(OLAPSchemaFactory.class);
- static {
- /*
- * Tricks Optiq to work with Unicode.
- *
- * Sets default char set for string literals in SQL and row types of
- * RelNode. This is more a label used to compare row type equality. For
- * both SQL string and row record, they are passed to Optiq in String
- * object and does not require additional codec.
- *
- * Ref SaffronProperties.defaultCharset
- * Ref SqlUtil.translateCharacterSetName()
- * Ref NlsString constructor()
- */
- System.setProperty("saffron.default.charset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
- System.setProperty("saffron.default.nationalcharset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
- System.setProperty("saffron.default.collation.name", ConversionUtil.NATIVE_UTF16_CHARSET_NAME + "$en_US");
- }
-
private final static String SCHEMA_PROJECT = "project";
@Override
[12/50] [abbrv] kylin git commit: KYLIN-2668 remove extra blank
Posted by ka...@apache.org.
KYLIN-2668 remove extra blank
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d851c51d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d851c51d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d851c51d
Branch: refs/heads/KYLIN-2606
Commit: d851c51d2c5b41d3d118ff0d5115b37c13dcc1f3
Parents: 69532cd
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jun 29 14:16:10 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jun 29 14:16:10 2017 +0800
----------------------------------------------------------------------
jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d851c51d/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
index 4fd9d33..7144dea 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
@@ -196,7 +196,7 @@ public class DriverTest {
Connection conn = driver.connect("jdbc:kylin:ssl=True;//test_url/test_db", null);
assertEquals("test_url", ((KylinConnection)conn).getBaseUrl());
assertEquals("test_db", ((KylinConnection)conn).getProject());
- assertTrue(Boolean.parseBoolean( (String)((KylinConnection)conn).getConnectionProperties().get("ssl")));
+ assertTrue(Boolean.parseBoolean((String)((KylinConnection)conn).getConnectionProperties().get("ssl")));
}
private void printResultSet(ResultSet rs) throws SQLException {
[04/50] [abbrv] kylin git commit: minor,
add parserTimeStampField to KafkaConfig (#1405)
Posted by ka...@apache.org.
minor, add parserTimeStampField to KafkaConfig (#1405)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b2fc2c22
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b2fc2c22
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b2fc2c22
Branch: refs/heads/KYLIN-2606
Commit: b2fc2c220ba66482447a7631f90363382bcaa422
Parents: 1acd066
Author: 成 <ch...@kyligence.io>
Authored: Wed Jun 28 11:23:51 2017 +0800
Committer: Billy(Yiming) Liu <li...@gmail.com>
Committed: Wed Jun 28 11:23:51 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/source/kafka/KafkaMRInput.java | 2 +-
.../kylin/source/kafka/config/KafkaConfig.java | 22 ++++++++++++++++++++
2 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2fc2c22/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index 3323afb..5bce4e7 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -103,7 +103,7 @@ public class KafkaMRInput implements IMRInput {
this.cubeSegment = cubeSegment;
this.conf = conf;
try {
- streamingParser = StreamingParser.getStreamingParser(kafkaConfig.getParserName(), kafkaConfig.getParserProperties(), columns);
+ streamingParser = StreamingParser.getStreamingParser(kafkaConfig.getParserName(), kafkaConfig.getAllParserProperties(), columns);
} catch (ReflectiveOperationException e) {
throw new IllegalArgumentException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2fc2c22/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
index a096344..547e738 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
@@ -58,6 +58,9 @@ public class KafkaConfig extends RootPersistentEntity {
@JsonProperty("parserName")
private String parserName;
+ @JsonProperty("parserTimeStampField")
+ private String parserTimeStampField;
+
@Deprecated
@JsonProperty("margin")
private long margin;
@@ -120,6 +123,14 @@ public class KafkaConfig extends RootPersistentEntity {
this.margin = margin;
}
+ public void setParserTimeStampField(String parserTimeStampField) {
+ this.parserTimeStampField = parserTimeStampField;
+ }
+
+ public String getParserTimeStampField() {
+ return this.parserTimeStampField;
+ }
+
public String getParserProperties() {
return parserProperties;
}
@@ -128,6 +139,17 @@ public class KafkaConfig extends RootPersistentEntity {
this.parserProperties = parserProperties;
}
+ public String getAllParserProperties() {
+ StringBuilder sb = new StringBuilder();
+ if (parserProperties != null)
+ sb.append(parserProperties);
+ if (parserTimeStampField != null) {
+ sb.append(";");
+ sb.append(parserTimeStampField);
+ }
+ return sb.toString();
+ }
+
@Override
public KafkaConfig clone() {
try {
[34/50] [abbrv] kylin git commit: minor, refine drop job situation
Posted by ka...@apache.org.
minor, refine drop job situation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03702f1e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03702f1e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03702f1e
Branch: refs/heads/KYLIN-2606
Commit: 03702f1ecd3a79228468bb7ec6ae25ad369fd65c
Parents: 6228f88
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Jun 30 19:19:35 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 19:21:45 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/controller2/JobControllerV2.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/03702f1e/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
index a86082c..5a18e66 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
@@ -282,8 +282,7 @@ public class JobControllerV2 extends BasicController {
JobInstance jobInstance = jobService.getJobInstance(jobId);
JobStatusEnum status = jobInstance.getStatus();
- if (status == JobStatusEnum.NEW || status == JobStatusEnum.PENDING || status == JobStatusEnum.RUNNING
- || status == JobStatusEnum.STOPPED) {
+ if (status != JobStatusEnum.FINISHED && status != JobStatusEnum.DISCARDED) {
throw new BadRequestException(
"Cannot drop running job " + jobInstance.getName() + ", please discard it first.");
}
[47/50] [abbrv] kylin git commit: KYLIN-1351 Add unit tests for JDBC
source
Posted by ka...@apache.org.
KYLIN-1351 Add unit tests for JDBC source
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad47dc00
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad47dc00
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad47dc00
Branch: refs/heads/KYLIN-2606
Commit: ad47dc00e672e8e967d8605d2c8d607ce28225d5
Parents: 926a17a
Author: auphyroc99 <45...@qq.com>
Authored: Wed Jul 5 11:03:00 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Jul 5 17:02:56 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 1 +
.../jdbc/ITJdbcSourceTableLoaderTest.java | 121 +++++++++++++++++++
.../source/jdbc/ITJdbcTableReaderTest.java | 116 ++++++++++++++++++
source-hive/pom.xml | 5 +
4 files changed, 243 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 179d61f..f9c3adb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -532,6 +532,7 @@ abstract public class KylinConfigBase implements Serializable {
// ref constants in ISourceAware
r.put(0, "org.apache.kylin.source.hive.HiveSource");
r.put(1, "org.apache.kylin.source.kafka.KafkaSource");
+ r.put(8, "org.apache.kylin.source.jdbc.JdbcSource");
r.putAll(convertKeyToInteger(getPropertiesByPrefix("kylin.source.provider.")));
return r;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
new file mode 100644
index 0000000..a84dca9
--- /dev/null
+++ b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
@@ -0,0 +1,121 @@
+package org.apache.kylin.source.jdbc;
+
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Types;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ISourceAware;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableExtDesc;
+import org.apache.kylin.query.H2Database;
+import org.apache.kylin.source.ISource;
+import org.apache.kylin.source.ISourceMetadataExplorer;
+import org.apache.kylin.source.SourceFactory;
+import org.apache.kylin.source.datagen.ModelDataGenerator;
+import org.dbunit.DatabaseUnitException;
+import org.dbunit.database.DatabaseConfig;
+import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.DataTypeException;
+import org.dbunit.ext.h2.H2Connection;
+import org.dbunit.ext.h2.H2DataTypeFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ITJdbcSourceTableLoaderTest extends LocalFileMetadataTestCase implements ISourceAware {
+
+ protected KylinConfig config = null;
+ protected static Connection h2Connection = null;
+
+ @Before
+ public void setup() throws Exception {
+
+ super.createTestMetadata();
+
+ System.setProperty("kylin.source.jdbc.connection-url", "jdbc:h2:mem:db");
+ System.setProperty("kylin.source.jdbc.driver", "org.h2.Driver");
+ System.setProperty("kylin.source.jdbc.user", "sa");
+ System.setProperty("kylin.source.jdbc.pass", "");
+
+ config = KylinConfig.getInstanceFromEnv();
+
+ h2Connection = DriverManager.getConnection("jdbc:h2:mem:db", "sa", "");
+
+ H2Database h2DB = new H2Database(h2Connection, config);
+
+ MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ ModelDataGenerator gen = new ModelDataGenerator(mgr.getDataModelDesc("ci_left_join_model"), 10000);
+ gen.generate();
+
+ h2DB.loadAllTables();
+
+ }
+
+ @After
+ public void after() throws Exception {
+
+ super.cleanupTestMetadata();
+
+ if (h2Connection != null) {
+ try {
+ h2Connection.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+ System.clearProperty("kylin.source.jdbc.connection-url");
+ System.clearProperty("kylin.source.jdbc.driver");
+ System.clearProperty("kylin.source.jdbc.user");
+ System.clearProperty("kylin.source.jdbc.pass");
+
+ }
+
+ @Test
+ public void test() throws Exception {
+
+ ISource source = SourceFactory.getSource(new ITJdbcSourceTableLoaderTest());
+ ISourceMetadataExplorer explr = source.getSourceMetadataExplorer();
+ Pair<TableDesc, TableExtDesc> pair;
+
+ pair = explr.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT");
+ assertTrue(pair.getFirst().getIdentity().equals("DEFAULT.TEST_KYLIN_FACT"));
+
+ pair = explr.loadTableMetadata("EDW", "TEST_CAL_DT");
+ assertTrue(pair.getFirst().getIdentity().equals("EDW.TEST_CAL_DT"));
+
+ }
+
+ @Override
+ public int getSourceType() {
+ return ISourceAware.ID_JDBC;
+ }
+
+ @SuppressWarnings("deprecation")
+ protected static H2Connection newH2Connection() throws DatabaseUnitException {
+ H2Connection h2Conn = new H2Connection(h2Connection, null);
+ h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
+ h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
+ return h2Conn;
+ }
+
+ public static class TestH2DataTypeFactory extends H2DataTypeFactory {
+ @Override
+ public DataType createDataType(int sqlType, String sqlTypeName, String tableName, String columnName)
+ throws DataTypeException {
+
+ if ((columnName.startsWith("COL") || columnName.startsWith("col")) && sqlType == Types.BIGINT) {
+ return DataType.INTEGER;
+ }
+ return super.createDataType(sqlType, sqlTypeName);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
new file mode 100644
index 0000000..7850952
--- /dev/null
+++ b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
@@ -0,0 +1,116 @@
+package org.apache.kylin.source.jdbc;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Types;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ISourceAware;
+import org.apache.kylin.query.H2Database;
+import org.apache.kylin.source.datagen.ModelDataGenerator;
+import org.dbunit.DatabaseUnitException;
+import org.dbunit.database.DatabaseConfig;
+import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.DataTypeException;
+import org.dbunit.ext.h2.H2Connection;
+import org.dbunit.ext.h2.H2DataTypeFactory;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ITJdbcTableReaderTest extends LocalFileMetadataTestCase implements ISourceAware {
+
+ protected KylinConfig config = null;
+ protected static Connection h2Connection = null;
+
+ @Before
+ public void setup() throws Exception {
+
+ super.createTestMetadata();
+
+ System.setProperty("kylin.source.jdbc.connection-url", "jdbc:h2:mem:db");
+ System.setProperty("kylin.source.jdbc.driver", "org.h2.Driver");
+ System.setProperty("kylin.source.jdbc.user", "sa");
+ System.setProperty("kylin.source.jdbc.pass", "");
+
+ config = KylinConfig.getInstanceFromEnv();
+
+ h2Connection = DriverManager.getConnection("jdbc:h2:mem:db", "sa", "");
+
+ H2Database h2DB = new H2Database(h2Connection, config);
+
+ MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ ModelDataGenerator gen = new ModelDataGenerator(mgr.getDataModelDesc("ci_left_join_model"), 10000);
+ gen.generate();
+
+ h2DB.loadAllTables();
+
+ }
+
+ @After
+ public void after() throws Exception {
+
+ super.cleanupTestMetadata();
+
+ if (h2Connection != null) {
+ try {
+ h2Connection.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+ System.clearProperty("kylin.source.jdbc.connection-url");
+ System.clearProperty("kylin.source.jdbc.driver");
+ System.clearProperty("kylin.source.jdbc.user");
+ System.clearProperty("kylin.source.jdbc.pass");
+
+ }
+
+ @Test
+ public void test() throws Exception {
+
+ JdbcTableReader reader = new JdbcTableReader("default", "test_kylin_fact");
+ int rowNumber = 0;
+ while (reader.next()) {
+ String[] row = reader.getRow();
+ Assert.assertEquals(11, row.length);
+
+ rowNumber++;
+ }
+
+ reader.close();
+ Assert.assertEquals(10000, rowNumber);
+
+ }
+
+ @Override
+ public int getSourceType() {
+ return ISourceAware.ID_JDBC;
+ }
+
+ @SuppressWarnings("deprecation")
+ protected static H2Connection newH2Connection() throws DatabaseUnitException {
+ H2Connection h2Conn = new H2Connection(h2Connection, null);
+ h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
+ h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
+ return h2Conn;
+ }
+
+ public static class TestH2DataTypeFactory extends H2DataTypeFactory {
+ @Override
+ public DataType createDataType(int sqlType, String sqlTypeName, String tableName, String columnName)
+ throws DataTypeException {
+
+ if ((columnName.startsWith("COL") || columnName.startsWith("col")) && sqlType == Types.BIGINT) {
+ return DataType.INTEGER;
+ }
+ return super.createDataType(sqlType, sqlTypeName);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index abe8af5..b58f25e 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -41,6 +41,11 @@
<!-- Env & Test -->
<dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
<type>test-jar</type>
[33/50] [abbrv] kylin git commit: KYLIN-2700 Should allow user to
override Kafka conf in cube level
Posted by ka...@apache.org.
KYLIN-2700 Should allow user to override Kafka conf in cube level
Signed-off-by: Billy Liu <bi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/40e4d627
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/40e4d627
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/40e4d627
Branch: refs/heads/KYLIN-2606
Commit: 40e4d627227f584d6212ef588eb3fd658bf159dc
Parents: 1eb32c3
Author: qiumingming <qi...@bytedance.com>
Authored: Fri Jun 30 18:31:52 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Jun 30 18:40:10 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/40e4d627/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
index 11466e5..e98b784 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
@@ -104,7 +104,7 @@ public class KafkaFlatTableJob extends AbstractHadoopJob {
job.getConfiguration().addResource(new Path(jobEngineConfig.getHadoopJobConfFilePath(null)));
KafkaConsumerProperties kafkaConsumerProperties = KafkaConsumerProperties.getInstanceFromEnv();
job.getConfiguration().addResource(new Path(kafkaConsumerProperties.getKafkaConsumerHadoopJobConf()));
- appendKafkaOverrideProperties(KylinConfig.getInstanceFromEnv(), job.getConfiguration());
+ appendKafkaOverrideProperties(cube.getConfig(), job.getConfiguration());
job.getConfiguration().set(CONFIG_KAFKA_BROKERS, brokers);
job.getConfiguration().set(CONFIG_KAFKA_TOPIC, topic);
job.getConfiguration().set(CONFIG_KAFKA_TIMEOUT, String.valueOf(kafkaConfig.getTimeout()));
[20/50] [abbrv] kylin git commit: KYLIN-2695 Should allow user to
override spark conf in cube
Posted by ka...@apache.org.
KYLIN-2695 Should allow user to override spark conf in cube
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/804a1aa3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/804a1aa3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/804a1aa3
Branch: refs/heads/KYLIN-2606
Commit: 804a1aa3809e67d8e701f147bee794eec19d57e2
Parents: a8627e8
Author: kangkaisen <ka...@live.com>
Authored: Wed Jun 28 20:21:52 2017 +0800
Committer: kangkaisen <ka...@live.com>
Committed: Thu Jun 29 21:03:20 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/spark/SparkExecutable.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/804a1aa3/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 1ed2235..1e032c6 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -23,6 +23,8 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.job.common.PatternedLogger;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -70,7 +72,10 @@ public class SparkExecutable extends AbstractExecutable {
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final KylinConfig config = context.getConfig();
+ String cubeName = this.getParam(SparkCubingByLayer.OPTION_CUBE_NAME.getOpt());
+ CubeInstance cube = CubeManager.getInstance(context.getConfig()).getCube(cubeName);
+ final KylinConfig config = cube.getConfig();
+
if (KylinConfig.getSparkHome() == null) {
throw new NullPointerException();
}
[40/50] [abbrv] kylin git commit: minor, fix acl migrate tool
Posted by ka...@apache.org.
minor, fix acl migrate tool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f4382df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f4382df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f4382df
Branch: refs/heads/KYLIN-2606
Commit: 4f4382df90ddd04a963dda36bcc1093c4b5d77b4
Parents: 7002dd8
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Jun 30 23:40:42 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 23:45:52 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/service/AclTableMigrationTool.java | 4 ++--
.../src/main/java/org/apache/kylin/rest/service/CubeService.java | 2 --
2 files changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f4382df/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
index 029efdc..bb07c22 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
@@ -224,8 +224,8 @@ public class AclTableMigrationTool {
Map<String, AceInfo> allAceInfoMap = new HashMap<>();
NavigableMap<byte[], byte[]> familyMap = result.getFamilyMap(Bytes.toBytes(AclConstant.ACL_ACES_FAMILY));
for (Map.Entry<byte[], byte[]> entry : familyMap.entrySet()) {
- String sid = String.valueOf(entry.getKey());
- AceInfo aceInfo = aceSerializer.deserialize(familyMap.get(entry.getValue()));
+ String sid = new String(entry.getKey());
+ AceInfo aceInfo = aceSerializer.deserialize(entry.getValue());
if (null != aceInfo) {
allAceInfoMap.put(sid, aceInfo);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f4382df/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index b2e44f6..aa42cb0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -659,8 +659,6 @@ public class CubeService extends BasicService {
getDraftManager().save(draft);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
- + " or hasPermission(#draft, 'ADMINISTRATION') or hasPermission(#draft, 'MANAGEMENT')")
public void deleteDraft(Draft draft) throws IOException {
getDraftManager().delete(draft.getUuid());
}
[03/50] [abbrv] kylin git commit: KYLIN-2689 Only dimension columns
can join when create a model
Posted by ka...@apache.org.
KYLIN-2689 Only dimension columns can join when create a model
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2bc90f93
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2bc90f93
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2bc90f93
Branch: refs/heads/KYLIN-2606
Commit: 2bc90f9331bcdc2e64b43cb5025640f95ec4efe1
Parents: 3f3cb39
Author: luguosheng1314 <55...@qq.com>
Authored: Wed Jun 28 00:56:39 2017 +0800
Committer: chenzhx <34...@qq.com>
Committed: Wed Jun 28 09:10:33 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/modelMeasures.js | 33 +++++++++++---------
webapp/app/js/filters/filter.js | 17 ++++++++++
.../partials/modelDesigner/model_measures.html | 2 +-
3 files changed, 37 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2bc90f93/webapp/app/js/controllers/modelMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelMeasures.js b/webapp/app/js/controllers/modelMeasures.js
index 342f341..5a3ac61 100644
--- a/webapp/app/js/controllers/modelMeasures.js
+++ b/webapp/app/js/controllers/modelMeasures.js
@@ -22,7 +22,7 @@
'use strict';
-KylinApp.controller('ModelMeasuresCtrl', function ($scope, $modal,MetaModel,modelsManager,VdmUtil) {
+KylinApp.controller('ModelMeasuresCtrl', function ($scope, $modal,MetaModel,modelsManager,VdmUtil,$filter) {
$scope.modelsManager = modelsManager;
$scope.availableFactTables = [];
$scope.selectedFactTables = {};
@@ -33,20 +33,25 @@ KylinApp.controller('ModelMeasuresCtrl', function ($scope, $modal,MetaModel,mode
$scope.availableFactTables.push(joinTable[j].alias);
}
}
+ $scope.changeColumns = function (table){
+ angular.forEach($scope.selectedFactTables[table],function(column){
+ if($scope.modelsManager.selectedModel.metrics.indexOf(column)==-1){
+ $scope.modelsManager.selectedModel.metrics.push(column);
+ }
+ });
+ angular.forEach($scope.modelsManager.selectedModel.metrics,function(metric){
+ if($scope.selectedFactTables[VdmUtil.getNameSpaceAliasName(metric)].indexOf(metric)==-1){
+ $scope.modelsManager.selectedModel.metrics.splice($scope.modelsManager.selectedModel.metrics.indexOf(metric),1);
+ }
+ });
+ }
angular.forEach($scope.modelsManager.selectedModel.metrics,function(metric){
- $scope.selectedFactTables[VdmUtil.getNameSpaceAliasName(metric)]=$scope.selectedFactTables[VdmUtil.getNameSpaceAliasName(metric)]||[];
- $scope.selectedFactTables[VdmUtil.getNameSpaceAliasName(metric)].push(metric);
+ var aliasName = VdmUtil.getNameSpaceAliasName(metric)
+ $scope.selectedFactTables[aliasName]=$scope.selectedFactTables[aliasName]||[];
+ $scope.selectedFactTables[aliasName].push(metric);
});
- $scope.changeColumns = function (table){
- angular.forEach($scope.selectedFactTables[table],function(column){
- if($scope.modelsManager.selectedModel.metrics.indexOf(column)==-1){
- $scope.modelsManager.selectedModel.metrics.push(column);
- }
- });
- angular.forEach($scope.modelsManager.selectedModel.metrics,function(metric){
- if($scope.selectedFactTables[VdmUtil.getNameSpaceAliasName(metric)].indexOf(metric)==-1){
- $scope.modelsManager.selectedModel.metrics.splice($scope.modelsManager.selectedModel.metrics.indexOf(metric),1);
- }
- });
+ for (var i in $scope.selectedFactTables) {
+ $scope.selectedFactTables[i] = $filter('notInJoin')($scope.selectedFactTables[i], i, $scope.modelsManager.selectedModel.lookups)
+ $scope.changeColumns(i)
}
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/2bc90f93/webapp/app/js/filters/filter.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index 78f6cf9..8b6cffa 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -204,6 +204,23 @@ KylinApp
});
return out;
}
+ }).filter('notInJoin', function ($filter) {
+ return function (inputArr, table, arr) {
+ var out=[];
+ angular.forEach(inputArr, function (inputItem) {
+ var isInJoin = false
+ angular.forEach(arr,function(item) {
+ var checkColumn = inputItem.name ? table + '.' + inputItem.name : inputItem;
+ if (item.join.foreign_key.indexOf(checkColumn) !== -1 || item.join.primary_key.indexOf(checkColumn) !== -1) {
+ isInJoin = true;
+ }
+ });
+ if (!isInJoin) {
+ out.push(inputItem);
+ }
+ });
+ return out;
+ }
}).filter('inMeaNotInDim', function ($filter) {
return function (inputArr, table, arr) {
var out=[];
http://git-wip-us.apache.org/repos/asf/kylin/blob/2bc90f93/webapp/app/partials/modelDesigner/model_measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/model_measures.html b/webapp/app/partials/modelDesigner/model_measures.html
index bf27914..d41f690 100644
--- a/webapp/app/partials/modelDesigner/model_measures.html
+++ b/webapp/app/partials/modelDesigner/model_measures.html
@@ -68,7 +68,7 @@
ng-model="selectedFactTables[table]" multiple>
<ui-select-match placeholder="Select Column...">{{$item.name}}</ui-select-match>
<ui-select-choices
- repeat="table+'.'+measure.name as measure in getColumnsByAlias(table) | filter:$select.search|inDimNotInMea:table:modelsManager.selectedModel.dimensions">
+ repeat="table+'.'+measure.name as measure in getColumnsByAlias(table) | filter:$select.search|inDimNotInMea:table:modelsManager.selectedModel.dimensions|notInJoin:table:modelsManager.selectedModel.lookups">
{{measure.name}}
</ui-select-choices>
</ui-select>
[23/50] [abbrv] kylin git commit: KYLIN-2652 Make kylinConfig
threadsafe in CubeVisitService
Posted by ka...@apache.org.
KYLIN-2652 Make kylinConfig threadsafe in CubeVisitService
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1e055a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1e055a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1e055a6
Branch: refs/heads/KYLIN-2606
Commit: a1e055a6b448f7e797ea29bc97e84325b3056638
Parents: 772a95f
Author: kangkaisen <ka...@live.com>
Authored: Fri May 26 12:30:17 2017 +0800
Committer: kangkaisen <ka...@live.com>
Committed: Thu Jun 29 21:20:22 2017 +0800
----------------------------------------------------------------------
.../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1e055a6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index cde127e..c66af66 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -238,9 +238,8 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
- String serverPropString = request.getKylinProperties();
- KylinConfig.setKylinConfigInEnvIfMissing(serverPropString);
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ KylinConfig kylinConfig = KylinConfig.createKylinConfig(request.getKylinProperties());
+ KylinConfig.setKylinConfigThreadLocal(kylinConfig);
debugGitTag = region.getTableDesc().getValue(IRealizationConstants.HTableGitTag);
@@ -425,8 +424,6 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
@Override
public void stop(CoprocessorEnvironment env) throws IOException {
- // destroy KylinConfig when coprocessor stop
- KylinConfig.destroyInstance();
}
@Override
[16/50] [abbrv] kylin git commit: minor, refine acl
Posted by ka...@apache.org.
minor, refine acl
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c490e6cf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c490e6cf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c490e6cf
Branch: refs/heads/KYLIN-2606
Commit: c490e6cf7c01e0e1443fa93fac681300f96680d6
Parents: 8e1a7d5
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 29 16:34:52 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 29 16:36:57 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/CubeController.java | 15 ++-
.../rest/controller2/AccessControllerV2.java | 16 ++-
.../kylin/rest/service/AccessService.java | 26 +++--
.../apache/kylin/rest/service/CubeService.java | 109 +++++++++++++++----
4 files changed, 131 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c490e6cf/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 6916dd8..a370292 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -352,7 +352,7 @@ public class CubeController extends BasicController {
@ResponseBody
public CubeInstance cloneCube(@PathVariable String cubeName, @RequestBody CubeRequest cubeRequest) {
String newCubeName = cubeRequest.getCubeName();
- String project = cubeRequest.getProject();
+ String projectName = cubeRequest.getProject();
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
if (cube == null) {
@@ -366,6 +366,11 @@ public class CubeController extends BasicController {
throw new BadRequestException("Invalid Cube name, only letters, numbers and underline supported.");
}
+ ProjectInstance project = cubeService.getProjectManager().getProject(projectName);
+ if (project == null) {
+ throw new BadRequestException("Project " + projectName + " doesn't exist");
+ }
+
CubeDesc cubeDesc = cube.getDescriptor();
CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
@@ -373,7 +378,7 @@ public class CubeController extends BasicController {
CubeInstance newCube;
try {
- newCube = cubeService.createCubeAndDesc(newCubeName, project, newCubeDesc);
+ newCube = cubeService.createCubeAndDesc(project, newCubeDesc);
//reload to avoid shallow clone
cubeService.getCubeDescManager().reloadCubeDescLocal(newCubeName);
@@ -449,7 +454,11 @@ public class CubeController extends BasicController {
try {
desc.setUuid(UUID.randomUUID().toString());
String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
- cubeService.createCubeAndDesc(name, projectName, desc);
+ ProjectInstance project = cubeService.getProjectManager().getProject(projectName);
+ if (project == null) {
+ throw new BadRequestException("Project " + projectName + " doesn't exist");
+ }
+ cubeService.createCubeAndDesc(project, desc);
} catch (Exception e) {
logger.error("Failed to deal with the request.", e);
throw new InternalErrorException(e.getLocalizedMessage(), e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c490e6cf/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
index 255e312..a61e321 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/AccessControllerV2.java
@@ -58,7 +58,6 @@ public class AccessControllerV2 extends BasicController {
* @return
* @throws IOException
*/
-
@RequestMapping(value = "/{type}/{uuid}", method = { RequestMethod.GET }, produces = {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
@@ -70,6 +69,21 @@ public class AccessControllerV2 extends BasicController {
}
/**
+ * List access entry list of a domain object including its parent
+ * @param type
+ * @param uuid
+ * @return
+ */
+ @RequestMapping(value = "all/{type}/{uuid}", method = { RequestMethod.GET }, produces = {
+ "application/vnd.apache.kylin-v2+json" })
+ @ResponseBody
+ public EnvelopeResponse listAccessEntitiesV2(@PathVariable String type, @PathVariable String uuid) {
+ AclEntity ae = accessService.getAclEntity(type, uuid);
+ Acl acl = accessService.getAcl(ae);
+ return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, accessService.generateListAceResponses(acl), "");
+ }
+
+ /**
* Grant a new access on a domain object to a user/role
*
* @param accessRequest
http://git-wip-us.apache.org/repos/asf/kylin/blob/c490e6cf/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index e5cd793..3a6d723 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -20,9 +20,7 @@ package org.apache.kylin.rest.service;
import java.util.ArrayList;
import java.util.List;
-import java.util.Set;
-import com.google.common.collect.Sets;
import org.apache.kylin.common.persistence.AclEntity;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.rest.constant.Constant;
@@ -300,16 +298,9 @@ public class AccessService {
public List<AccessEntryResponse> generateAceResponses(Acl acl) {
List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
- Set<Sid> sidSet = Sets.newHashSet();
- while (acl != null) {
- for (AccessControlEntry ace : acl.getEntries()) {
- if (!sidSet.contains(ace.getSid())) {
- result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
- sidSet.add(ace.getSid());
- }
- }
- acl = acl.getParentAcl();
+ for (AccessControlEntry ace : acl.getEntries()) {
+ result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
}
return result;
@@ -330,4 +321,17 @@ public class AccessService {
throw new ForbiddenException(msg.getREVOKE_ADMIN_PERMISSION());
}
}
+
+ public Object generateListAceResponses(Acl acl) {
+ List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
+
+ while (acl != null) {
+ for (AccessControlEntry ace : acl.getEntries()) {
+ result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
+ }
+ acl = acl.getParentAcl();
+ }
+
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c490e6cf/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a5fc36a..b2e44f6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -58,6 +58,7 @@ import org.apache.kylin.rest.request.MetricsRequest;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.security.AclPermission;
+import org.apache.kylin.rest.util.AclUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -96,6 +97,9 @@ public class CubeService extends BasicService {
@Qualifier("modelMgmtService")
private ModelService modelService;
+ @Autowired
+ private AclUtil aclUtil;
+
@PostFilter(Constant.ACCESS_POST_FILTER_READ)
public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName, boolean exactMatch) {
List<CubeInstance> cubeInstances = null;
@@ -153,9 +157,11 @@ public class CubeService extends BasicService {
return getCubeManager().updateCube(cubeBuilder);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
- public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeDesc desc) throws IOException {
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
+ public CubeInstance createCubeAndDesc(ProjectInstance project, CubeDesc desc) throws IOException {
Message msg = MsgPicker.getMsg();
+ String cubeName = desc.getName();
if (getCubeManager().getCube(cubeName) != null) {
throw new BadRequestException(String.format(msg.getCUBE_ALREADY_EXIST(), cubeName));
@@ -178,10 +184,9 @@ public class CubeService extends BasicService {
int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc, false);
logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids");
- createdCube = getCubeManager().createCube(cubeName, projectName, createdDesc, owner);
+ createdCube = getCubeManager().createCube(cubeName, project.getName(), createdDesc, owner);
accessService.init(createdCube, AclPermission.ADMINISTRATION);
- ProjectInstance project = getProjectManager().getProject(projectName);
accessService.inherit(createdCube, project);
return createdCube;
@@ -611,32 +616,69 @@ public class CubeService extends BasicService {
}
}
- public CubeDesc updateCubeToResourceStore(CubeDesc desc, String projectName) throws IOException {
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
+ public CubeDesc saveCube(CubeDesc desc, ProjectInstance project) throws IOException {
Message msg = MsgPicker.getMsg();
desc.setDraft(false);
if (desc.getUuid() == null)
desc.updateRandomUuid();
- String cubeName = desc.getName();
try {
- if (desc.getLastModified() == 0) {
- // new
- createCubeAndDesc(cubeName, projectName, desc);
- } else {
- // update
- CubeInstance cube = getCubeManager().getCube(desc.getName());
+ createCubeAndDesc(project, desc);
+ } catch (AccessDeniedException accessDeniedException) {
+ throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
+ }
- if (cube == null) {
- throw new BadRequestException(String.format(msg.getCUBE_NOT_FOUND(), desc.getName()));
- }
+ if (!desc.getError().isEmpty()) {
+ throw new BadRequestException(desc.getErrorMsg());
+ }
- if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) {
- throw new BadRequestException(String.format(msg.getINCONSISTENT_CUBE_DESC(), desc.getName()));
- }
+ return desc;
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+ public void saveDraft(ProjectInstance project, CubeInstance cube, String uuid, RootPersistentEntity... entities)
+ throws IOException {
+ Draft draft = new Draft();
+ draft.setProject(project.getName());
+ draft.setUuid(uuid);
+ draft.setEntities(entities);
+ getDraftManager().save(draft);
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
+ public void saveDraft(ProjectInstance project, String uuid, RootPersistentEntity... entities) throws IOException {
+ Draft draft = new Draft();
+ draft.setProject(project.getName());
+ draft.setUuid(uuid);
+ draft.setEntities(entities);
+ getDraftManager().save(draft);
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#draft, 'ADMINISTRATION') or hasPermission(#draft, 'MANAGEMENT')")
+ public void deleteDraft(Draft draft) throws IOException {
+ getDraftManager().delete(draft.getUuid());
+ }
- desc = updateCubeAndDesc(cube, desc, projectName, true);
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+ public CubeDesc updateCube(CubeInstance cube, CubeDesc desc, ProjectInstance project) throws IOException {
+ Message msg = MsgPicker.getMsg();
+ String projectName = project.getName();
+
+ desc.setDraft(false);
+
+ try {
+ if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) {
+ throw new BadRequestException(String.format(msg.getINCONSISTENT_CUBE_DESC(), desc.getName()));
}
+
+ desc = updateCubeAndDesc(cube, desc, projectName, true);
} catch (AccessDeniedException accessDeniedException) {
throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
}
@@ -671,6 +713,33 @@ public class CubeService extends BasicService {
}
}
- return result;
+ List<Draft> filtered = new ArrayList<>();
+
+ // if cube's there, follow cube permission. otherwise follow project permission
+ for (Draft d : result) {
+ CubeDesc desc = (CubeDesc) d.getEntity();
+ CubeInstance cube = getCubeManager().getCube(desc.getName());
+
+ if (cube == null) {
+ try {
+ project = project == null ? d.getProject() : project;
+ if (aclUtil.hasProjectReadPermission(getProjectManager().getProject(project))) {
+ filtered.add(d);
+ }
+ } catch (Exception e) {
+ // do nothing
+ }
+ } else {
+ try {
+ if (aclUtil.hasCubeReadPermission(cube)) {
+ filtered.add(d);
+ }
+ } catch (Exception e) {
+ // do nothing
+ }
+ }
+ }
+
+ return filtered;
}
}
[32/50] [abbrv] kylin git commit: minor,
validate join key should be a dimension
Posted by ka...@apache.org.
minor, validate join key should be a dimension
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6228f886
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6228f886
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6228f886
Branch: refs/heads/KYLIN-2606
Commit: 6228f886ecf697a41fd658b6eb46dde149cd0e2a
Parents: cf1ba95
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Jun 30 18:09:18 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 30 18:20:45 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 31 ++++++++++++++++++++
.../localmeta/model_desc/ssb.json | 20 ++++++-------
2 files changed, 41 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6228f886/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index bc35e2a..abee71b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -617,7 +617,38 @@ public class DataModelDesc extends RootPersistentEntity {
+ fkCols[i].getDatatype());
}
}
+
+ if (!isSpecialTestModel()) {
+ for (TblColRef t : fkCols) {
+ if (!isJoinKeyDimension(t.getIdentity())) {
+ throw new IllegalStateException(
+ "Foreigner key: " + t.getIdentity() + " should be a dimension.");
+ }
+ }
+
+ for (TblColRef t : pkCols) {
+ if (!isJoinKeyDimension(t.getIdentity())) {
+ throw new IllegalStateException("Primary key: " + t.getIdentity() + " should be a dimension.");
+ }
+ }
+ }
+ }
+ }
+
+ private boolean isJoinKeyDimension(String key) {
+ for (ModelDimensionDesc d : dimensions) {
+ for (String col : d.getColumns()) {
+ if (key.equals(d.getTable() + '.' + col))
+ return true;
+ }
}
+ return false;
+ }
+
+ private boolean isSpecialTestModel() {
+ if (config.isDevEnv() && (name.startsWith("test_kylin_") || name.startsWith("test_streaming")))
+ return true;
+ return false;
}
private void initJoinsTree() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6228f886/examples/test_case_data/localmeta/model_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ssb.json b/examples/test_case_data/localmeta/model_desc/ssb.json
index 52b330f..bbac99d 100644
--- a/examples/test_case_data/localmeta/model_desc/ssb.json
+++ b/examples/test_case_data/localmeta/model_desc/ssb.json
@@ -33,20 +33,20 @@
}
} ],
"dimensions" : [ {
- "table" : "SSB.V_LINEORDER",
- "columns" : [ "LO_DATE" ]
+ "table" : "V_LINEORDER",
+ "columns" : [ "LO_DATE", "LO_PARTKEY", "LO_CUSTKEY", "LO_ORDERDATE", "LO_SUPPKEY" ]
}, {
- "table" : "SSB.PART",
- "columns" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+ "table" : "PART",
+ "columns" : [ "P_MFGR", "P_CATEGORY", "P_BRAND", "P_PARTKEY" ]
}, {
- "table" : "SSB.CUSTOMER",
- "columns" : [ "C_CITY", "C_NATION", "C_REGION" ]
+ "table" : "CUSTOMER",
+ "columns" : [ "C_CITY", "C_NATION", "C_REGION", "C_CUSTKEY" ]
}, {
- "table" : "SSB.SUPPLIER",
- "columns" : [ "S_CITY", "S_NATION", "S_REGION" ]
+ "table" : "SUPPLIER",
+ "columns" : [ "S_CITY", "S_NATION", "S_REGION", "S_SUPPKEY" ]
}, {
- "table" : "SSB.DATES",
- "columns" : [ "D_YEAR", "D_YEARMONTHNUM", "D_YEARMONTH", "D_WEEKNUMINYEAR" ]
+ "table" : "DATES",
+ "columns" : [ "D_YEAR", "D_YEARMONTHNUM", "D_YEARMONTH", "D_WEEKNUMINYEAR", "D_DATEKEY" ]
} ],
"metrics" : [ "LO_REVENUE", "LO_SUPPLYCOST", "V_REVENUE" ],
"last_modified" : 1457444314662,
[22/50] [abbrv] kylin git commit: KYLIN-2693 Should use
overrideHiveConfig for LookupHiveViewMaterialization and
RedistributeFlatHiveTable
Posted by ka...@apache.org.
KYLIN-2693 Should use overrideHiveConfig for LookupHiveViewMaterialization and RedistributeFlatHiveTable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/772a95f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/772a95f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/772a95f3
Branch: refs/heads/KYLIN-2606
Commit: 772a95f3a0aaafca5260140c461edb053990cfe8
Parents: 547722d
Author: kangkaisen <ka...@live.com>
Authored: Tue Jun 27 20:02:04 2017 +0800
Committer: kangkaisen <ka...@live.com>
Committed: Thu Jun 29 21:17:45 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/772a95f3/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 337bafd..4747cb9 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -206,6 +206,7 @@ public class HiveMRInput implements IMRInput {
}
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride());
hiveCmdBuilder.addStatement(hiveInitStatements);
for (TableDesc lookUpTableDesc : lookupViewsTables) {
String identity = lookUpTableDesc.getIdentity();
@@ -273,6 +274,7 @@ public class HiveMRInput implements IMRInput {
private void redistributeTable(KylinConfig config, int numReducers) throws IOException {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.overwriteHiveProps(config.getHiveConfigOverride());
hiveCmdBuilder.addStatement(getInitStatement());
hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n");
hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n");
[08/50] [abbrv] kylin git commit: fix ci
Posted by ka...@apache.org.
fix ci
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f7aa456
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f7aa456
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f7aa456
Branch: refs/heads/KYLIN-2606
Commit: 2f7aa456640885dfde622966a6d487579a117931
Parents: 8603297
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 29 12:34:03 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 29 13:51:45 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/util/AdHocUtilTest.java | 45 ++++++++++----------
1 file changed, 22 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f7aa456/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
index b93e2d3..724844c 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/AdHocUtilTest.java
@@ -30,36 +30,28 @@ public class AdHocUtilTest {
@Test
public void testReplaceIdentifierInExpr() {
{
- String ret = AdHocUtil.replaceIdentifierInExpr("a.b.x * a.b.y", null, false);
- Assert.assertEquals("b.x * b.y", ret);
+ String ret = AdHocUtil.replaceIdentifierInExpr("x * y", null, false);
+ Assert.assertEquals("x * y", ret);
}
{
- String ret = AdHocUtil.replaceIdentifierInExpr("a_1.b_2.x_3 * a_1.b_2.y_3", null, false);
+ String ret = AdHocUtil.replaceIdentifierInExpr("x_3 * y_3", "b_2", false);
Assert.assertEquals("b_2.x_3 * b_2.y_3", ret);
}
{
- String ret = AdHocUtil.replaceIdentifierInExpr("a.b.x * a.b.y", "c", false);
- Assert.assertEquals("c.x * c.y", ret);
+ String ret = AdHocUtil.replaceIdentifierInExpr("substr(x,1,3)>y", "c", true);
+ Assert.assertEquals("substr(c.x,1,3)>c.y", ret);
}
{
- String ret = AdHocUtil.replaceIdentifierInExpr("a.b.x * a.b.y", "c", true);
- Assert.assertEquals("\"C\".\"X\" * \"C\".\"Y\"", ret);
+ String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", "c", true);
+ Assert.assertEquals("strcmp(substr(c.x,1,3),c.y)", ret);
}
{
- String ret = AdHocUtil.replaceIdentifierInExpr("substr(a.b.x,1,3)>a.b.y", "c", true);
- Assert.assertEquals("substr(\"C\".\"X\",1,3)>\"C\".\"Y\"", ret);
+ String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, true);
+ Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
}
{
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(a.b.x,1,3),a.b.y) > 0", "c", true);
- Assert.assertEquals("strcmp(substr(\"C\".\"X\",1,3),\"C\".\"Y\") > 0", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(a.b.x,1,3),a.b.y) > 0", null, true);
- Assert.assertEquals("strcmp(substr(\"B\".\"X\",1,3),\"B\".\"Y\") > 0", ret);
- }
- {
- String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(a.b.x, 1, 3),a.b.y) > 0", null, false);
- Assert.assertEquals("strcmp(substr(b.x, 1, 3),b.y) > 0", ret);
+ String ret = AdHocUtil.replaceIdentifierInExpr("strcmp(substr(x,1,3),y)", null, false);
+ Assert.assertEquals("strcmp(substr(x,1,3),y)", ret);
}
}
@@ -68,7 +60,7 @@ public class AdHocUtilTest {
ComputedColumnDesc computedColumnDesc = mock(ComputedColumnDesc.class);
when(computedColumnDesc.getColumnName()).thenReturn("DEAL_AMOUNT");
- when(computedColumnDesc.getExpression()).thenReturn("DB.TABLE.price * DB.TABLE.number");
+ when(computedColumnDesc.getExpression()).thenReturn("price * number");
CCInfo ccInfo = mock(CCInfo.class);
when(ccInfo.getComputedColumnDesc()).thenReturn(computedColumnDesc);
@@ -77,22 +69,29 @@ public class AdHocUtilTest {
String ret = AdHocUtil.restoreComputedColumnToExpr(
"select DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
Assert.assertEquals(
- "select (TABLE.price * TABLE.number) from DB.TABLE group by date order by (TABLE.price * TABLE.number)",
+ "select (price * number) from DB.TABLE group by date order by (price * number)",
ret);
}
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
"select DEAL_AMOUNT as DEAL_AMOUNT from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
Assert.assertEquals(
- "select (TABLE.price * TABLE.number) as DEAL_AMOUNT from DB.TABLE group by date order by (TABLE.price * TABLE.number)",
+ "select (price * number) as DEAL_AMOUNT from DB.TABLE group by date order by (price * number)",
ret);
}
{
String ret = AdHocUtil.restoreComputedColumnToExpr(
"select \"DEAL_AMOUNT\" AS deal_amount from DB.TABLE group by date order by DEAL_AMOUNT", ccInfo);
Assert.assertEquals(
- "select (\"TABLE\".\"PRICE\" * \"TABLE\".\"NUMBER\") AS deal_amount from DB.TABLE group by date order by (TABLE.price * TABLE.number)",
+ "select (price * number) AS deal_amount from DB.TABLE group by date order by (price * number)",
ret);
}
+ {
+ String ret = AdHocUtil.restoreComputedColumnToExpr(
+ "select x.DEAL_AMOUNT AS deal_amount from DB.TABLE x group by date order by x.DEAL_AMOUNT", ccInfo);
+ Assert.assertEquals(
+ "select (x.price * x.number) AS deal_amount from DB.TABLE x group by date order by (x.price * x.number)",
+ ret);
+ }
}
}