You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/01/26 11:20:27 UTC
[01/33] kylin git commit: Minor, get metadata of query columns
Repository: kylin
Updated Branches:
refs/heads/sync [created] 923a9bd9f
Minor, get metadata of query columns
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4196ddd4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4196ddd4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4196ddd4
Branch: refs/heads/sync
Commit: 4196ddd42b65809cfcfba102596224ee1f40312f
Parents: 34cbd18
Author: Yifan Zhang <ev...@gmail.com>
Authored: Thu Dec 28 21:02:16 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 16:49:34 2018 +0800
----------------------------------------------------------------------
.../kylin/source/ISourceMetadataExplorer.java | 3 +
.../kylin/source/hive/HiveMetadataExplorer.java | 74 ++++++++++++-----
.../apache/kylin/source/jdbc/JdbcExplorer.java | 83 ++++++++++++++------
.../apache/kylin/source/kafka/KafkaSource.java | 6 ++
4 files changed, 125 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4196ddd4/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java b/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
index 4072846..0a6747c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
@@ -21,6 +21,7 @@ package org.apache.kylin.source;
import java.util.List;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
@@ -33,4 +34,6 @@ public interface ISourceMetadataExplorer {
Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj) throws Exception;
List<String> getRelatedKylinResources(TableDesc table);
+
+ ColumnDesc[] evalQueryMetadata(String query);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4196ddd4/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index 55e6267..d80c546 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -23,6 +23,7 @@ import java.util.Collections;
import java.util.List;
import java.util.UUID;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.TableMetadataManager;
@@ -59,7 +60,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
}
TableDesc tableDesc = metaMgr.getTableDesc(database + "." + tableName, prj);
-
+
// make a new TableDesc instance, don't modify the one in use
if (tableDesc == null) {
tableDesc = new TableDesc();
@@ -70,28 +71,12 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
} else {
tableDesc = new TableDesc(tableDesc);
}
-
+
if (hiveTableMeta.tableType != null) {
tableDesc.setTableType(hiveTableMeta.tableType);
}
- int columnNumber = hiveTableMeta.allColumns.size();
- List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);
- for (int i = 0; i < columnNumber; i++) {
- HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);
- ColumnDesc cdesc = new ColumnDesc();
- cdesc.setName(field.name.toUpperCase());
- // use "double" in kylin for "float"
- if ("float".equalsIgnoreCase(field.dataType)) {
- cdesc.setDatatype("double");
- } else {
- cdesc.setDatatype(field.dataType);
- }
- cdesc.setId(String.valueOf(i + 1));
- cdesc.setComment(field.comment);
- columns.add(cdesc);
- }
- tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
+ tableDesc.setColumns(extractColumnFromMeta(hiveTableMeta));
StringBuffer partitionColumnString = new StringBuffer();
for (int i = 0, n = hiveTableMeta.partitionColumns.size(); i < n; i++) {
@@ -193,4 +178,55 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
return hiveDataType.toLowerCase();
}
+ @Override
+ public ColumnDesc[] evalQueryMetadata(String query) {
+ if (StringUtils.isEmpty(query)) {
+ throw new RuntimeException("Evalutate query shall not be empty.");
+ }
+
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ String tmpDatabase = config.getHiveDatabaseForIntermediateTable();
+ String tmpView = "kylin_eval_query_" + UUID.nameUUIDFromBytes(query.getBytes()).toString().replace("-", "");
+
+ String dropViewSql = "DROP VIEW IF EXISTS " + tmpDatabase + "." + tmpView;
+ String evalViewSql = "CREATE VIEW " + tmpDatabase + "." + tmpView + " as " + query;
+
+ try {
+ hiveClient.executeHQL(new String[] { dropViewSql, evalViewSql });
+ HiveTableMeta hiveTableMeta = hiveClient.getHiveTableMeta(tmpDatabase, tmpView);
+ return extractColumnFromMeta(hiveTableMeta);
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot evalutate metadata of query: " + query, e);
+ } finally {
+ try {
+ hiveClient.executeHQL(dropViewSql);
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot temp view of query: " + query, e);
+ }
+ }
+ }
+
+ private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) {
+ int columnNumber = hiveTableMeta.allColumns.size();
+ List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);
+
+ for (int i = 0; i < columnNumber; i++) {
+ HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);
+ ColumnDesc cdesc = new ColumnDesc();
+ cdesc.setName(field.name.toUpperCase());
+
+ // use "double" in kylin for "float"
+ if ("float".equalsIgnoreCase(field.dataType)) {
+ cdesc.setDatatype("double");
+ } else {
+ cdesc.setDatatype(field.dataType);
+ }
+
+ cdesc.setId(String.valueOf(i + 1));
+ cdesc.setComment(field.comment);
+ columns.add(cdesc);
+ }
+
+ return columns.toArray(new ColumnDesc[columnNumber]);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4196ddd4/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index 2827b3d..81e42bb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -28,6 +28,7 @@ import java.util.Collections;
import java.util.List;
import java.util.UUID;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.DBUtils;
@@ -99,33 +100,12 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
}
}
- List<ColumnDesc> columns = new ArrayList<>();
try (ResultSet rs = jdbcMetadataDialect.listColumns(dbmd, database, table)) {
- while (rs.next()) {
- String cname = rs.getString("COLUMN_NAME");
- int type = rs.getInt("DATA_TYPE");
- int csize = rs.getInt("COLUMN_SIZE");
- int digits = rs.getInt("DECIMAL_DIGITS");
- int pos = rs.getInt("ORDINAL_POSITION");
- String remarks = rs.getString("REMARKS");
-
- ColumnDesc cdesc = new ColumnDesc();
- cdesc.setName(cname.toUpperCase());
-
- String kylinType = SqlUtil.jdbcTypeToKylinDataType(type);
- int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
- int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;
-
- cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
- cdesc.setId(String.valueOf(pos));
- cdesc.setComment(remarks);
- columns.add(cdesc);
- }
+ tableDesc.setColumns(extractColumnFromMeta(rs));
} finally {
DBUtils.closeQuietly(con);
}
- tableDesc.setColumns(columns.toArray(new ColumnDesc[columns.size()]));
TableExtDesc tableExtDesc = new TableExtDesc();
tableExtDesc.setIdentity(tableDesc.getIdentity());
@@ -249,4 +229,63 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
return Collections.emptyList();
}
+ @Override
+ public ColumnDesc[] evalQueryMetadata(String query) {
+ if (StringUtils.isEmpty(query)) {
+ throw new RuntimeException("Evalutate query shall not be empty.");
+ }
+
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ String tmpDatabase = config.getHiveDatabaseForIntermediateTable();
+ String tmpView = tmpDatabase + ".kylin_eval_query_" + UUID.nameUUIDFromBytes(query.getBytes()).toString().replaceAll("-", "");
+
+ String dropViewSql = "DROP VIEW IF EXISTS " + tmpView;
+ String evalViewSql = "CREATE VIEW " + tmpView + " as " + query;
+
+ try {
+ executeSQL(new String[] { dropViewSql, evalViewSql });
+ Connection con = SqlUtil.getConnection(dbconf);
+ DatabaseMetaData dbmd = con.getMetaData();
+ ResultSet rs = dbmd.getColumns(null, tmpDatabase, tmpView, null);
+ ColumnDesc[] result = extractColumnFromMeta(rs);
+ DBUtils.closeQuietly(rs);
+ DBUtils.closeQuietly(con);
+ return result;
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot evalutate metadata of query: " + query, e);
+ } finally {
+ try {
+ executeSQL(dropViewSql);
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot temp view of query: " + query, e);
+ }
+ }
+ }
+
+ private ColumnDesc[] extractColumnFromMeta(ResultSet meta) throws SQLException {
+ List<ColumnDesc> columns = new ArrayList<>();
+
+ while (meta.next()) {
+ String cname = meta.getString("COLUMN_NAME");
+ int type = meta.getInt("DATA_TYPE");
+ int csize = meta.getInt("COLUMN_SIZE");
+ int digits = meta.getInt("DECIMAL_DIGITS");
+ int pos = meta.getInt("ORDINAL_POSITION");
+ String remarks = meta.getString("REMARKS");
+
+ ColumnDesc cdesc = new ColumnDesc();
+ cdesc.setName(cname.toUpperCase());
+
+ String kylinType = SqlUtil.jdbcTypeToKylinDataType(type);
+ int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
+ int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;
+
+ cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
+ cdesc.setId(String.valueOf(pos));
+ cdesc.setComment(remarks);
+ columns.add(cdesc);
+ }
+
+ return columns.toArray(new ColumnDesc[columns.size()]);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4196ddd4/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
index 396c0e1..1142243 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.IMRInput;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableDesc;
@@ -209,6 +210,11 @@ public class KafkaSource implements ISource {
dependentResources.add(StreamingConfig.concatResourcePath(table.getIdentity()));
return dependentResources;
}
+
+ @Override
+ public ColumnDesc[] evalQueryMetadata(String query) {
+ throw new UnsupportedOperationException();
+ }
};
}
[22/33] kylin git commit: minor, refine sql comment removing
Posted by li...@apache.org.
minor, refine sql comment removing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/327bd766
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/327bd766
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/327bd766
Branch: refs/heads/sync
Commit: 327bd766c85e2fd0b6a335b38f96fe3a9d753bdf
Parents: 5977985
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Jan 16 15:17:46 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:00:23 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/query/util/QueryUtil.java | 17 ++++-------------
.../org/apache/kylin/query/util/QueryUtilTest.java | 16 +++++++++++++++-
2 files changed, 19 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/327bd766/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 22658f1..da18bd7 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -142,23 +142,14 @@ public class QueryUtil {
public static String removeCommentInSql(String sql1) {
// match two patterns, one is "-- comment", the other is "/* comment */"
- final String[] commentPatterns = new String[] { "--[^\r\n]*", "/\\*[\\s\\S]*?\\*/" };
+ final String[] commentPatterns = new String[] { "--.*?[\r\n]", "/\\*.*?\\*/" };
for (int i = 0; i < commentPatterns.length; i++) {
- String commentPattern = commentPatterns[i];
- Pattern pattern = Pattern.compile(commentPattern);
- Matcher matcher = pattern.matcher(sql1);
-
- while (matcher.find()) {
- if (matcher.start() == 0) {
- sql1 = sql1.substring(matcher.end()).trim();
- } else {
- sql1 = (sql1.substring(0, matcher.start()) + sql1.substring(matcher.end())).trim();
- }
- matcher = pattern.matcher(sql1);
- }
+ sql1 = sql1.replaceAll(commentPatterns[i], "");
}
+ sql1 = sql1.trim();
+
return sql1;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/327bd766/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index e7a7ecb..e3cd819 100644
--- a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -149,9 +149,23 @@ public class QueryUtilTest extends LocalFileMetadataTestCase {
}
{
- String sqlWithComment = "/* comment \n select count(*) from kylin_sales; \n */ " + originSql;
+ String sqlWithComment = "/* comment1/comment2 */ " + originSql;
Assert.assertEquals(originSql, QueryUtil.removeCommentInSql(sqlWithComment));
}
+ {
+ String sqlWithComment = "/* comment1 * comment2 */ " + originSql;
+ Assert.assertEquals(originSql, QueryUtil.removeCommentInSql(sqlWithComment));
+ }
+
+ {
+ String sqlWithComment = "/* comment1 * comment2 */ /* comment3 / comment4 */ -- comment 5\n" + originSql;
+ Assert.assertEquals(originSql, QueryUtil.removeCommentInSql(sqlWithComment));
+ }
+
+ {
+ String sqlWithComment = "/* comment1 * comment2 */ -- comment 5\n" + originSql + "/* comment3 / comment4 */";
+ Assert.assertEquals(originSql, QueryUtil.removeCommentInSql(sqlWithComment));
+ }
}
}
[08/33] kylin git commit: Update exec-maven-plugin version to 1.6.0
Posted by li...@apache.org.
Update exec-maven-plugin version to 1.6.0
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1bf98864
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1bf98864
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1bf98864
Branch: refs/heads/sync
Commit: 1bf988640b7580f29ca11d8e0ed587512be5f76c
Parents: 6cf78bc
Author: nichunen <ch...@kyligence.io>
Authored: Thu Jan 4 21:32:15 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 17:33:12 2018 +0800
----------------------------------------------------------------------
external/pom.xml | 2 +-
pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1bf98864/external/pom.xml
----------------------------------------------------------------------
diff --git a/external/pom.xml b/external/pom.xml
index 2cc09b2..8e0a40a 100644
--- a/external/pom.xml
+++ b/external/pom.xml
@@ -47,7 +47,7 @@
<plugin>
<artifactId>exec-maven-plugin</artifactId>
<groupId>org.codehaus.mojo</groupId>
- <version>1.5.0</version>
+ <version>1.6.0</version>
<executions>
<execution>
<id>install_htrace</id>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1bf98864/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index da43c62..be2d2bf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1096,7 +1096,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.5.0</version>
+ <version>1.6.0</version>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
[26/33] kylin git commit: KYLIN-2876,
refine singleton in ExternalAclProvider.
Posted by li...@apache.org.
KYLIN-2876, refine singleton in ExternalAclProvider.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a6e6013b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a6e6013b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a6e6013b
Branch: refs/heads/sync
Commit: a6e6013b8975f55093057867bb2df5a33e9ac1e9
Parents: 1d825a1
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 8 19:35:53 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:03:57 2018 +0800
----------------------------------------------------------------------
.../rest/security/ExternalAclProvider.java | 28 +++++++++-----------
1 file changed, 12 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a6e6013b/server-base/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java b/server-base/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
index 9e977d0..ef21f00 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
@@ -19,6 +19,7 @@ package org.apache.kylin.rest.security;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Pair;
@@ -28,26 +29,21 @@ import org.springframework.security.acls.model.Permission;
*/
abstract public class ExternalAclProvider {
- private static boolean inited = false;
- private static ExternalAclProvider singleton = null;
+ private volatile static ExternalAclProvider singleton = null;
public static ExternalAclProvider getInstance() {
- if (inited)
- return singleton;
-
- synchronized (ExternalAclProvider.class) {
- if (inited)
- return singleton;
-
- String cls = KylinConfig.getInstanceFromEnv().getExternalAclProvider();
- if (cls != null && cls.length() > 0) {
- singleton = (ExternalAclProvider) ClassUtil.newInstance(cls);
- singleton.init();
+ if (singleton == null) {
+ synchronized (ExternalAclProvider.class) {
+ if (singleton == null) {
+ String cls = KylinConfig.getInstanceFromEnv().getExternalAclProvider();
+ if (!StringUtils.isBlank(cls)) {
+ singleton = (ExternalAclProvider) ClassUtil.newInstance(cls);
+ singleton.init();
+ }
+ }
}
-
- inited = true;
- return singleton;
}
+ return singleton;
}
// ============================================================================
[23/33] kylin git commit: minor,
create job work dir with Materialize Hive View in Lookup Tables job.
Posted by li...@apache.org.
minor, create job work dir with Materialize Hive View in Lookup Tables job.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/de9b5286
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/de9b5286
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/de9b5286
Branch: refs/heads/sync
Commit: de9b528697729df97bffa26112c2e761e4cd5634
Parents: 327bd76
Author: yiming.xu <10...@qq.com>
Authored: Tue Jan 16 14:04:10 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:01:08 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/hive/HiveMRInput.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/de9b5286/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index e4564d0..6f42961 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -210,6 +210,9 @@ public class HiveMRInput implements IMRInput {
if (lookupViewsTables.size() == 0) {
return null;
}
+ // Create work dir to avoid hive create it,
+ // the difference is that the owners are different.
+ checkAndCreateWorkDir(jobWorkingDir);
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride());
@@ -239,6 +242,19 @@ public class HiveMRInput implements IMRInput {
return step;
}
+ private void checkAndCreateWorkDir(String jobWorkingDir) {
+ try {
+ Path path = new Path(jobWorkingDir);
+ FileSystem fileSystem = HadoopUtil.getFileSystem(path);
+ if (!fileSystem.exists(path)) {
+ logger.info("Create jobWorkDir : " + jobWorkingDir);
+ fileSystem.mkdirs(path);
+ }
+ } catch (IOException e) {
+ logger.error("Could not create lookUp table dir : " + jobWorkingDir);
+ }
+ }
+
private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
String cubeName) {
//from hive to hive
[02/33] kylin git commit: KYLIN-2836 Fix timers counters and meters
add synchronization impl
Posted by li...@apache.org.
KYLIN-2836 Fix timers counters and meters add synchronization impl
This closes #75
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4daa8a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4daa8a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4daa8a3
Branch: refs/heads/sync
Commit: b4daa8a3198a99400629c79db9832070bbe063d0
Parents: 4196ddd
Author: mingming.ge <mi...@kyligence>
Authored: Mon Sep 25 17:01:51 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 17:22:46 2018 +0800
----------------------------------------------------------------------
.../metrics/metrics2/CodahaleMetrics.java | 24 +++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b4daa8a3/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java
index bae6cfb..e728759 100644
--- a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java
@@ -144,9 +144,27 @@ public class CodahaleMetrics implements Metrics {
for (Map.Entry<String, Metric> metric : metricRegistry.getMetrics().entrySet()) {
metricRegistry.remove(metric.getKey());
}
- timers.invalidateAll();
- counters.invalidateAll();
- meters.invalidateAll();
+
+ try {
+ timersLock.lock();
+ timers.invalidateAll();
+ } finally {
+ timersLock.unlock();
+ }
+
+ try {
+ countersLock.lock();
+ counters.invalidateAll();
+ } finally {
+ countersLock.unlock();
+ }
+
+ try {
+ metersLock.lock();
+ meters.invalidateAll();
+ } finally {
+ metersLock.unlock();
+ }
}
@Override
[30/33] kylin git commit: minor refactor: avoid exposing cubedesc's
getError public api
Posted by li...@apache.org.
minor refactor: avoid exposing cubedesc's getError public api
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2062d6e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2062d6e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2062d6e7
Branch: refs/heads/sync
Commit: 2062d6e74550102bebda7aac1abfa52afb1fa6de
Parents: e071c1c
Author: Hongbin Ma <ma...@kyligence.io>
Authored: Fri Jan 26 19:13:12 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:13:12 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeDescManager.java | 2 +-
.../org/apache/kylin/cube/CubeInstance.java | 29 ++++++++++--------
.../org/apache/kylin/cube/model/CubeDesc.java | 13 ++------
.../kylin/rest/controller/CubeController.java | 32 ++++++++++++--------
.../apache/kylin/rest/service/CubeService.java | 12 ++++----
5 files changed, 47 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2062d6e7/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index cd43f15..1db66e1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -198,7 +198,7 @@ public class CubeDescManager {
postProcessCubeDesc(cubeDesc);
// Check base validation
- if (!cubeDesc.getError().isEmpty()) {
+ if (cubeDesc.isBroken()) {
return cubeDesc;
}
// Semantic validation
http://git-wip-us.apache.org/repos/asf/kylin/blob/2062d6e7/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index a299203..6395abd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -6,15 +6,15 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
package org.apache.kylin.cube;
@@ -130,13 +130,13 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
// default constructor for jackson
public CubeInstance() {
}
-
+
public CubeInstance latestCopyForWrite() {
CubeManager mgr = CubeManager.getInstance(config);
CubeInstance latest = mgr.getCube(name); // in case this object is out-of-date
return mgr.copyForWrite(latest);
}
-
+
void init(KylinConfig config) {
CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(descName);
checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", descName, name);
@@ -144,9 +144,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
if (cubeDesc.isBroken()) {
setStatus(RealizationStatusEnum.DESCBROKEN);
logger.error("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), name);
- for (String error : cubeDesc.getError()) {
- logger.error("Error: {}", error);
- }
+ logger.error("Errors: {}", cubeDesc.getErrorsAsString());
} else if (getStatus() == RealizationStatusEnum.DESCBROKEN) {
setStatus(RealizationStatusEnum.DISABLED);
logger.info("cube {} changed from DESCBROKEN to DISABLED", name);
@@ -217,7 +215,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN)
&& segments.isEmpty();
}
-
+
@Override
public String resourceName() {
return name;
@@ -301,7 +299,12 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
public int getCost() {
int countedDimensionNum = getRowKeyColumnCount();
int c = countedDimensionNum * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;
- for (JoinTableDesc join : getModel().getJoinTables()) {
+ DataModelDesc model = getModel();
+ if (model == null) {
+ //in case broken cube
+ return 0;
+ }
+ for (JoinTableDesc join : model.getJoinTables()) {
if (join.getJoin().isInnerJoin())
c += CubeInstance.COST_WEIGHT_INNER_JOIN;
}
@@ -356,7 +359,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
return null;
}
-
+
public CubeSegment[] regetSegments(CubeSegment... segs) {
CubeSegment[] r = new CubeSegment[segs.length];
for (int i = 0; i < segs.length; i++) {
@@ -477,6 +480,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
/**
* Get cuboid level count except base cuboid
+ *
* @return
*/
public int getBuildLevel() {
@@ -562,7 +566,8 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public SegmentRange autoMergeCubeSegments() throws IOException {
- return segments.autoMergeCubeSegments(needAutoMerge(), getName(), getDescriptor().getAutoMergeTimeRanges(), getDescriptor().getVolatileRange());
+ return segments.autoMergeCubeSegments(needAutoMerge(), getName(), getDescriptor().getAutoMergeTimeRanges(),
+ getDescriptor().getVolatileRange());
}
public Segments calculateToBeSegments(CubeSegment newSegment) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/2062d6e7/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 2369629..dbd8708 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -82,6 +82,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.base.Joiner;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -1215,16 +1216,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
this.errors.add(message);
}
- public List<String> getError() {
- return this.errors;
- }
-
- public String getErrorMsg() {
- StringBuffer sb = new StringBuffer();
- for (String error : errors) {
- sb.append(error + " ");
- }
- return sb.toString();
+ public String getErrorsAsString() {
+ return Joiner.on("; ").join(errors);
}
public HBaseMappingDesc getHbaseMapping() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/2062d6e7/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f6c2abf..90a614f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -6,15 +6,15 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
package org.apache.kylin.rest.controller;
@@ -93,7 +93,6 @@ import org.springframework.web.bind.annotation.ResponseBody;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
-import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -209,7 +208,7 @@ public class CubeController extends BasicController {
/**
* Get SQL of a Cube segment
*
- * @param cubeName Cube Name
+ * @param cubeName Cube Name
* @param segmentName Segment Name
* @return
* @throws IOException
@@ -319,7 +318,9 @@ public class CubeController extends BasicController {
}
}
- /** Build/Rebuild a cube segment */
+ /**
+ * Build/Rebuild a cube segment
+ */
@RequestMapping(value = "/{cubeName}/build", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance build(@PathVariable String cubeName, @RequestBody JobBuildRequest req) {
@@ -328,7 +329,9 @@ public class CubeController extends BasicController {
/** Build/Rebuild a cube segment */
- /** Build/Rebuild a cube segment */
+ /**
+ * Build/Rebuild a cube segment
+ */
@RequestMapping(value = "/{cubeName}/rebuild", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest req) {
@@ -336,7 +339,9 @@ public class CubeController extends BasicController {
req.getBuildType(), req.isForce() || req.isForceMergeEmptySegment());
}
- /** Build/Rebuild a cube segment by source offset */
+ /**
+ * Build/Rebuild a cube segment by source offset
+ */
@RequestMapping(value = "/{cubeName}/build2", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance build2(@PathVariable String cubeName, @RequestBody JobBuildRequest2 req) {
@@ -355,7 +360,9 @@ public class CubeController extends BasicController {
return rebuild2(cubeName, req);
}
- /** Build/Rebuild a cube segment by source offset */
+ /**
+ * Build/Rebuild a cube segment by source offset
+ */
@RequestMapping(value = "/{cubeName}/rebuild2", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance rebuild2(@PathVariable String cubeName, @RequestBody JobBuildRequest2 req) {
@@ -415,7 +422,7 @@ public class CubeController extends BasicController {
/**
* Send a optimize cube segment job
*
- * @param cubeName Cube ID
+ * @param cubeName Cube ID
* @param segmentID for segment to be optimized
*/
@RequestMapping(value = "/{cubeName}/recover_segment_optimize/{segmentID}", method = { RequestMethod.PUT })
@@ -661,8 +668,8 @@ public class CubeController extends BasicController {
throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
}
- if (!desc.getError().isEmpty()) {
- updateRequest(cubeRequest, false, Joiner.on("\n").join(desc.getError()));
+ if (desc.isBroken()) {
+ updateRequest(cubeRequest, false, desc.getErrorsAsString());
return cubeRequest;
}
@@ -959,6 +966,7 @@ public class CubeController extends BasicController {
/**
* Initiate the very beginning of a streaming cube. Will seek the latest offests of each partition from streaming
* source (kafka) and record in the cube descriptor; In the first build job, it will use these offests as the start point.
+ *
* @param cubeName
* @return
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/2062d6e7/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a81cc24..5f5f9ed 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -200,8 +200,8 @@ public class CubeService extends BasicService implements InitializingBean {
createdDesc = getCubeDescManager().createCubeDesc(desc);
- if (!createdDesc.getError().isEmpty()) {
- throw new BadRequestException(createdDesc.getErrorMsg());
+ if (createdDesc.isBroken()) {
+ throw new BadRequestException(createdDesc.getErrorsAsString());
}
int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc, false);
@@ -630,8 +630,8 @@ public class CubeService extends BasicService implements InitializingBean {
throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
}
- if (!desc.getError().isEmpty()) {
- throw new BadRequestException(desc.getErrorMsg());
+ if (desc.isBroken()) {
+ throw new BadRequestException(desc.getErrorsAsString());
}
return desc;
@@ -680,8 +680,8 @@ public class CubeService extends BasicService implements InitializingBean {
throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
}
- if (!desc.getError().isEmpty()) {
- throw new BadRequestException(desc.getErrorMsg());
+ if (desc.isBroken()) {
+ throw new BadRequestException(desc.getErrorsAsString());
}
return desc;
[17/33] kylin git commit: KYLIN-3164 HBase connection must be closed
when clearing connection pool
Posted by li...@apache.org.
KYLIN-3164 HBase connection must be closed when clearing connection pool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3116abf1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3116abf1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3116abf1
Branch: refs/heads/sync
Commit: 3116abf1ded94cdc30e59f4613b22ce3dc5337dd
Parents: 120f9f3
Author: Li Yang <li...@apache.org>
Authored: Mon Jan 15 13:25:56 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 18:53:19 2018 +0800
----------------------------------------------------------------------
.../kylin/storage/hbase/HBaseConnection.java | 38 +++++++++++++++-----
1 file changed, 29 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3116abf1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 16bd20d..0c2fb04 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -20,8 +20,10 @@ package org.apache.kylin.storage.hbase;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -74,14 +76,11 @@ public class HBaseConnection {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
- closeCoprocessorPool();
-
- for (Connection conn : connPool.values()) {
- try {
- conn.close();
- } catch (IOException e) {
- logger.error("error closing hbase connection " + conn, e);
- }
+ try {
+ closeCoprocessorPool();
+ closeAndRestConnPool().join();
+ } catch (InterruptedException e) {
+ logger.error("", e);
}
}
});
@@ -128,9 +127,30 @@ public class HBaseConnection {
coprocessorPool.shutdownNow();
}
}
+
+ private static Thread closeAndRestConnPool() {
+ final List<Connection> copy = new ArrayList<>(connPool.values());
+ connPool.clear();
+
+ Thread t = new Thread() {
+ public void run() {
+ logger.info("Closing HBase connections...");
+ for (Connection conn : copy) {
+ try {
+ conn.close();
+ } catch (Exception e) {
+ logger.error("error closing hbase connection " + conn, e);
+ }
+ }
+ }
+ };
+ t.setName("close-hbase-conn");
+ t.start();
+ return t;
+ }
public static void clearConnCache() {
- connPool.clear();
+ closeAndRestConnPool();
}
public static Configuration getCurrentHBaseConfiguration() {
[11/33] kylin git commit: KYLIN-3144,
adopt Collections.emptyList() for empty list values.
Posted by li...@apache.org.
KYLIN-3144, adopt Collections.emptyList() for empty list values.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/98b1047f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/98b1047f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/98b1047f
Branch: refs/heads/sync
Commit: 98b1047fe9fd7f1c66c78fc8d4324c083ba8c9f2
Parents: 19381d3
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 8 19:18:24 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 17:46:00 2018 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/rest/service/HybridService.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/98b1047f/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
index 912923a..de20e7c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.service;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
@@ -119,7 +120,7 @@ public class HybridService extends BasicService {
if (StringUtils.isEmpty(projectName)) {
hybridsInProject = new ArrayList(getHybridManager().listHybridInstances());
} else if (project == null) {
- return new ArrayList<>();
+ return Collections.emptyList();
} else {
List<RealizationEntry> realizationEntries = project.getRealizationEntries(RealizationType.HYBRID);
if (realizationEntries != null) {
@@ -134,7 +135,7 @@ public class HybridService extends BasicService {
if (StringUtils.isEmpty(modelName)) {
return hybridsInProject;
} else if (model == null) {
- return new ArrayList<>();
+ return Collections.emptyList();
} else {
List<HybridInstance> hybridsInModel = new ArrayList<HybridInstance>();
for (HybridInstance hybridInstance : hybridsInProject) {
[14/33] kylin git commit: KYLIN-3157, refine test.
Posted by li...@apache.org.
KYLIN-3157, refine test.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b22ee0d2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b22ee0d2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b22ee0d2
Branch: refs/heads/sync
Commit: b22ee0d2d1ba3f11506d771d45efd653c499ab0b
Parents: d43abbd
Author: tttMelody <24...@qq.com>
Authored: Tue Jan 9 16:49:11 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 18:49:18 2018 +0800
----------------------------------------------------------------------
.../rest/service/KyilnQueryTimeoutTest.java | 104 ------------------
.../rest/service/KylinQueryTimeoutTest.java | 107 +++++++++++++++++++
2 files changed, 107 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b22ee0d2/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java b/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
deleted file mode 100644
index 25ff75b..0000000
--- a/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.service;
-import java.sql.SQLException;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.exceptions.KylinTimeoutException;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.TupleInfo;
-import org.apache.kylin.query.security.QueryACLTestUtil;
-import org.apache.kylin.rest.request.SQLRequest;
-import org.apache.kylin.storage.IStorage;
-import org.apache.kylin.storage.IStorageQuery;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.StorageFactory;
-import org.hamcrest.CoreMatchers;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-public class KyilnQueryTimeoutTest extends LocalFileMetadataTestCase {
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @Before
- public void setUp() {
- this.createTestMetadata();
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- config.setProperty("kylin.storage.provider.2", MockQueryTimeoutStorage.class.getName());
- config.setProperty("kylin.storage.default", "2");
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- StorageFactory.clearCache();
- }
-
- @Test
- public void testQueryTimeout() throws SQLException {
- thrown.expectCause(CoreMatchers.isA(KylinTimeoutException.class));
- thrown.expectMessage(CoreMatchers.containsString("Kylin query timeout"));
- StorageFactory.clearCache();
- BadQueryDetector detector = new BadQueryDetector(100, BadQueryDetector.getSystemAvailMB() * 2, 100, 1);
- detector.start();
- SQLRequest request = new SQLRequest();
- request.setProject("default");
- request.setSql("select count(*) from STREAMING_TABLE");
- detector.queryStart(Thread.currentThread(), request, "ADMIN");
- try {
- QueryACLTestUtil.mockQuery("default", "select * from STREAMING_TABLE");
- } finally{
- detector.queryEnd(Thread.currentThread(), "timeout");
- detector.interrupt();
- }
- }
-
- public static class MockQueryTimeoutStorage implements IStorage {
-
- @Override
- public IStorageQuery createQuery(IRealization realization) {
- return new MockQueryTimeoutQuery();
- }
-
- @Override
- public <I> I adaptToBuildEngine(Class<I> engineInterface) {
- return null;
- }
- }
-
- private static class MockQueryTimeoutQuery implements IStorageQuery {
- @Override
- public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
- try {
- Thread.sleep(5 * 1000);
- } catch (InterruptedException e) {
- throw new KylinTimeoutException("Kylin query timeout");
- }
- return null;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/b22ee0d2/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java b/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
new file mode 100644
index 0000000..6a61b2f
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+import java.sql.SQLException;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exceptions.KylinTimeoutException;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
+import org.apache.kylin.query.security.QueryACLTestUtil;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.storage.IStorage;
+import org.apache.kylin.storage.IStorageQuery;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageFactory;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class KylinQueryTimeoutTest extends LocalFileMetadataTestCase {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Before
+ public void setUp() {
+ this.createTestMetadata();
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setProperty("kylin.storage.provider.2", MockQueryTimeoutStorage.class.getName());
+ config.setProperty("kylin.storage.default", "2");
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ StorageFactory.clearCache();
+ }
+
+ @Test
+ public void testQueryTimeout() throws SQLException {
+ thrown.expectCause(CoreMatchers.isA(KylinTimeoutException.class));
+ thrown.expectMessage(CoreMatchers.containsString("Kylin query timeout"));
+ StorageFactory.clearCache();
+ BadQueryDetector detector = new BadQueryDetector(100, BadQueryDetector.getSystemAvailMB() * 2, 100, 1);
+ detector.start();
+ SQLRequest request = new SQLRequest();
+ request.setProject("default");
+ request.setSql("select count(*) from STREAMING_TABLE");
+ detector.queryStart(Thread.currentThread(), request, "ADMIN");
+ try {
+ QueryACLTestUtil.mockQuery("default", "select * from STREAMING_TABLE");
+ } finally{
+ detector.queryEnd(Thread.currentThread(), "timeout");
+ detector.interrupt();
+ }
+ // every place that thrown KylinTimeoutException should reset the interrupt.
+ Assert.assertEquals(false, Thread.currentThread().isInterrupted());
+ }
+
+ public static class MockQueryTimeoutStorage implements IStorage {
+
+ @Override
+ public IStorageQuery createQuery(IRealization realization) {
+ return new MockQueryTimeoutQuery();
+ }
+
+ @Override
+ public <I> I adaptToBuildEngine(Class<I> engineInterface) {
+ return null;
+ }
+ }
+
+ private static class MockQueryTimeoutQuery implements IStorageQuery {
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ try {
+ Thread.sleep(5 * 1000);
+ } catch (InterruptedException e) {
+ throw new KylinTimeoutException("Kylin query timeout");
+ }
+ return null;
+ }
+ }
+}
\ No newline at end of file
[32/33] kylin git commit: minor,
add switch table-dir-create-first for build cube with hive step.
Posted by li...@apache.org.
minor, add switch table-dir-create-first for build cube with hive step.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ad6acbe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ad6acbe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ad6acbe
Branch: refs/heads/sync
Commit: 4ad6acbe85d33159cc85bf23739c3c9e8f22fb33
Parents: 4559101
Author: yiming.xu <10...@qq.com>
Authored: Wed Jan 24 17:33:18 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:15:49 2018 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../main/java/org/apache/kylin/source/hive/HiveMRInput.java | 8 +++++---
.../java/org/apache/kylin/source/hive/HiveMRInputTest.java | 2 +-
3 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ad6acbe/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index edb5b29..b6d8c02 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -755,6 +755,10 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.source.hive.sparksql-beeline-params", "");
}
+ public boolean getHiveTableDirCreateFirst() {
+ return Boolean.parseBoolean(getOptional("kylin.source.hive.table-dir-create-first", "false"));
+ }
+
public String getFlatHiveTableClusterByDictColumn() {
return getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ad6acbe/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 31a99cd..f02d2ea 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -179,9 +179,11 @@ public class HiveMRInput implements IMRInput {
protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
String jobWorkingDir = JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
- // Create work dir to avoid hive create it,
- // the difference is that the owners are different.
- checkAndCreateWorkDir(jobWorkingDir);
+ if (KylinConfig.getInstanceFromEnv().getHiveTableDirCreateFirst()) {
+ // Create work dir to avoid hive create it,
+ // the difference is that the owners are different.
+ checkAndCreateWorkDir(jobWorkingDir);
+ }
return jobWorkingDir;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ad6acbe/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
index 6d0737d..f33ee74 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -41,8 +41,8 @@ public class HiveMRInputTest {
try {
KylinConfig kylinConfig = mock(KylinConfig.class);
KylinConfig.setKylinConfigThreadLocal(kylinConfig);
+ when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true);
when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
-
DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class);
defaultChainedExecutable.setId(UUID.randomUUID().toString());
[25/33] kylin git commit: KYLIN-3167, fulfil datatype's precision
Posted by li...@apache.org.
KYLIN-3167, fulfil datatype's precision
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d825a1e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d825a1e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d825a1e
Branch: refs/heads/sync
Commit: 1d825a1e1869ac18b4c494cb838da60af9a841df
Parents: 5bdf723
Author: Cheng Wang <ch...@kyligence.io>
Authored: Tue Jan 16 14:55:36 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:02:20 2018 +0800
----------------------------------------------------------------------
.../kylin/source/hive/BeelineHiveClient.java | 27 +++++++++--
.../source/hive/BeelineHIveClientTest.java | 51 ++++++++++++++++++++
2 files changed, 74 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d825a1e/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 314402c..747b1bb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -114,7 +114,7 @@ public class BeelineHiveClient implements IHiveClient {
}
return count;
}
-
+
@Override
public void executeHQL(String hql) throws IOException {
throw new UnsupportedOperationException();
@@ -132,7 +132,11 @@ public class BeelineHiveClient implements IHiveClient {
List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
while (columns.next()) {
- allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), columns.getString(12)));
+ String columnName = columns.getString(4);
+ String dataType = columns.getString(6);
+ String comment = columns.getString(12);
+ dataType = considerDataTypePrecision(dataType, columns.getString(7), columns.getString(9));
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columnName, dataType, comment));
}
builder.setAllColumns(allColumns);
DBUtils.closeQuietly(columns);
@@ -143,6 +147,19 @@ public class BeelineHiveClient implements IHiveClient {
return builder.createHiveTableMeta();
}
+ public static String considerDataTypePrecision(String dataType, String precision, String scale) {
+ if ("VARCHAR".equalsIgnoreCase(dataType) || "CHAR".equalsIgnoreCase(dataType)) {
+ if (null != precision)
+ dataType = new StringBuilder(dataType).append("(").append(precision).append(")").toString();
+ }
+ if ("DECIMAL".equalsIgnoreCase(dataType) || "NUMERIC".equalsIgnoreCase(dataType)) {
+ if (precision != null && scale != null)
+ dataType = new StringBuilder(dataType).append("(").append(precision).append(",").append(scale)
+ .append(")").toString();
+ }
+ return dataType;
+ }
+
private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException {
while (resultSet.next()) {
@@ -156,7 +173,8 @@ public class BeelineHiveClient implements IHiveClient {
if ("".equals(resultSet.getString(1).trim())) {
break;
}
- partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), resultSet.getString(2).trim(), resultSet.getString(3).trim()));
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(),
+ resultSet.getString(2).trim(), resultSet.getString(3).trim()));
}
builder.setPartitionColumns(partitionColumns);
}
@@ -213,7 +231,8 @@ public class BeelineHiveClient implements IHiveClient {
public static void main(String[] args) throws SQLException {
- BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
+ BeelineHiveClient loader = new BeelineHiveClient(
+ "-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
//BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "test_kylin_fact_part");
System.out.println(hiveTableMeta);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d825a1e/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java
new file mode 100644
index 0000000..00a25e5
--- /dev/null
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class BeelineHIveClientTest {
+ @Test
+ public void testBasics() {
+ String dataType = "varchar";
+ String precision = "60";
+ String scale = null;
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("varchar(60)", dataType);
+
+ dataType = "char";
+ precision = "50";
+ scale = null;
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("char(50)", dataType);
+
+ dataType = "decimal";
+ precision = "8";
+ scale = "4";
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("decimal(8,4)", dataType);
+
+ dataType = "numeric";
+ precision = "7";
+ scale = "3";
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("numeric(7,3)", dataType);
+ }
+}
[31/33] kylin git commit: Update decimal's max precision to 38,
be aligned with Hive
Posted by li...@apache.org.
Update decimal's max precision to 38, be aligned with Hive
(cherry picked from commit 30e09d8c8198ed074658d2664676a5243a0690dc)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4559101e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4559101e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4559101e
Branch: refs/heads/sync
Commit: 4559101ec7baa5caf1c61b43bbfcf067e5523218
Parents: 2062d6e
Author: Yifan Zhang <ev...@gmail.com>
Authored: Thu Jan 18 23:44:25 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:14:52 2018 +0800
----------------------------------------------------------------------
.../query/calcite/KylinRelDataTypeSystem.java | 11 +++
.../calcite/KylinRelDataTypeSystemTest.java | 78 ++++++++++++++++++++
2 files changed, 89 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4559101e/query/src/main/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystem.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystem.java b/query/src/main/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystem.java
index 060747a..35b4c3d 100644
--- a/query/src/main/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystem.java
+++ b/query/src/main/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystem.java
@@ -45,4 +45,15 @@ public class KylinRelDataTypeSystem extends RelDataTypeSystemImpl {
}
return argumentType;
}
+
+ /**
+ * Hive support decimal with 38 digits, kylin should align
+ *
+ * @see org.apache.calcite.rel.type.RelDataTypeSystem
+ * @see <a href="https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-DecimalsdecimalDecimals">Hive/LanguageManualTypes-Decimals</a>
+ */
+ @Override
+ public int getMaxNumericPrecision() {
+ return 38;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4559101e/query/src/test/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystemTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystemTest.java b/query/src/test/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystemTest.java
new file mode 100644
index 0000000..e9ef548
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystemTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.calcite;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.query.schema.OLAPTable;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class KylinRelDataTypeSystemTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testLegalDecimalType() {
+ RelDataTypeSystem typeSystem = new KylinRelDataTypeSystem();
+ RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(typeSystem);
+
+ DataType dataType = DataType.getType("decimal(30, 10)");
+ RelDataType relDataType = OLAPTable.createSqlType(typeFactory, dataType, true);
+
+ Assert.assertTrue(relDataType instanceof BasicSqlType);
+ Assert.assertEquals(relDataType.getSqlTypeName(), SqlTypeName.DECIMAL);
+ Assert.assertEquals(relDataType.getPrecision(), 30);
+ Assert.assertTrue(relDataType.getPrecision() <= typeSystem.getMaxNumericPrecision());
+ Assert.assertEquals(relDataType.getScale(), 10);
+ Assert.assertTrue(relDataType.getScale() <= typeSystem.getMaxNumericScale());
+ }
+
+ @Test
+ public void testIllegalDecimalType() {
+ RelDataTypeSystem typeSystem = new KylinRelDataTypeSystem();
+ RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(typeSystem);
+
+ DataType dataType = DataType.getType("decimal(40, 10)");
+ RelDataType relDataType = OLAPTable.createSqlType(typeFactory, dataType, true);
+
+ Assert.assertTrue(relDataType instanceof BasicSqlType);
+ Assert.assertEquals(relDataType.getSqlTypeName(), SqlTypeName.DECIMAL);
+ Assert.assertTrue(typeSystem.getMaxNumericPrecision() < 40);
+ Assert.assertEquals(relDataType.getPrecision(), typeSystem.getMaxNumericPrecision());
+ Assert.assertEquals(relDataType.getScale(), 10);
+ Assert.assertTrue(relDataType.getScale() <= typeSystem.getMaxNumericScale());
+ }
+}
[15/33] kylin git commit: KYLIN-3125 bug fix
Posted by li...@apache.org.
KYLIN-3125 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d9765e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d9765e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d9765e3
Branch: refs/heads/sync
Commit: 6d9765e3145d3748070a7c0d616dd574bce90b96
Parents: b22ee0d
Author: Li Yang <li...@apache.org>
Authored: Thu Jan 11 13:21:14 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 18:50:52 2018 +0800
----------------------------------------------------------------------
core-common/src/main/resources/kylin-defaults.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d9765e3/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index bbb5f55..a79728a 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -83,7 +83,7 @@ kylin.server.external-acl-provider=
kylin.source.hive.client=cli
# Absolute path to beeline shell, can be set to spark beeline instead of the default hive beeline on PATH
-#kylin.source.hive.beeline-shell=beeline
+kylin.source.hive.beeline-shell=beeline
# Parameters for beeline client, only necessary if hive client is beeline
#kylin.source.hive.beeline-params=-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u jdbc:hive2://localhost:10000
[13/33] kylin git commit: minor, refine basic tuple filter class
Posted by li...@apache.org.
minor, refine basic tuple filter class
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d43abbd6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d43abbd6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d43abbd6
Branch: refs/heads/sync
Commit: d43abbd6bb722f85035d432ae05399e55e2917e8
Parents: 13f64f4
Author: Roger Shi <ro...@hotmail.com>
Authored: Mon Jan 8 17:57:57 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 18:46:52 2018 +0800
----------------------------------------------------------------------
.../metadata/filter/LogicalTupleFilter.java | 26 ++++++++
.../kylin/metadata/filter/TupleFilter.java | 43 +++++++++++++
.../kylin/metadata/filter/TupleFilterTest.java | 65 ++++++++++++++++++++
3 files changed, 134 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d43abbd6/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 7893ed8..f0c825f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -170,4 +170,30 @@ public class LogicalTupleFilter extends TupleFilter implements IOptimizeableTupl
return transformer.visit(this);
}
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+ if (!(other instanceof LogicalTupleFilter)) {
+ return false;
+ }
+ final LogicalTupleFilter otherFilter = (LogicalTupleFilter) other;
+ if (otherFilter.operator != this.operator || otherFilter.children.size() != this.children.size()) {
+ return false;
+ }
+
+ for (int i = 0; i < otherFilter.children.size(); i++) {
+ if (!otherFilter.children.get(i).equals(this.children.get(i))) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return (operator == null ? 0 : operator.hashCode()) + 31 * this.children.hashCode();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d43abbd6/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 5ba8726..09b41f5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -99,6 +99,11 @@ public abstract class TupleFilter {
addChild(c); // subclass overrides addChild()
}
+ final public void addChildren(TupleFilter... children) {
+ for (TupleFilter c : children)
+ addChild(c); // subclass overrides addChild()
+ }
+
public List<? extends TupleFilter> getChildren() {
return children;
}
@@ -121,6 +126,44 @@ public abstract class TupleFilter {
}
/**
+ * The storage level dislike NOT logic
+ */
+ public TupleFilter removeNot() {
+ return removeNotInternal(this);
+ }
+
+ private TupleFilter removeNotInternal(TupleFilter filter) {
+ FilterOperatorEnum op = filter.getOperator();
+
+ if (!(filter instanceof LogicalTupleFilter)) {
+ return filter;
+ }
+
+ LogicalTupleFilter logicalFilter = (LogicalTupleFilter) filter;
+
+ switch (logicalFilter.operator) {
+ case NOT:
+ assert (filter.children.size() == 1);
+ TupleFilter reverse = filter.children.get(0).reverse();
+ return removeNotInternal(reverse);
+ case AND:
+ LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ for (TupleFilter child : logicalFilter.children) {
+ andFilter.addChild(removeNotInternal(child));
+ }
+ return andFilter;
+ case OR:
+ LogicalTupleFilter orFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ for (TupleFilter child : logicalFilter.children) {
+ orFilter.addChild(removeNotInternal(child));
+ }
+ return orFilter;
+ default:
+ throw new IllegalStateException("This filter is unexpected: " + filter);
+ }
+ }
+
+ /**
* flatten to OR-AND filter, (A AND B AND ..) OR (C AND D AND ..) OR ..
* flatten filter will ONLY contain AND and OR , no NOT will exist.
* This will help to decide scan ranges.
http://git-wip-us.apache.org/repos/asf/kylin/blob/d43abbd6/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
new file mode 100644
index 0000000..e17f4d1
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TupleFilterTest {
+ @Test
+ // true ==> true
+ public void removeNotTest0() {
+ TupleFilter constFilter = ConstantTupleFilter.TRUE;
+ Assert.assertEquals(constFilter, constFilter.removeNot());
+ }
+
+ @Test
+ // Not(true) ==> false
+ public void removeNotTest1() {
+ TupleFilter notFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.NOT);
+ notFilter.addChild(ConstantTupleFilter.TRUE);
+ Assert.assertEquals(ConstantTupleFilter.FALSE, notFilter.removeNot());
+ }
+
+ @Test
+ // Not(And(true, false)) ==> Or(false, true)
+ public void removeNotTest2() {
+ TupleFilter notFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.NOT);
+ TupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ andFilter.addChildren(ConstantTupleFilter.TRUE, ConstantTupleFilter.FALSE);
+ notFilter.addChild(andFilter);
+
+ TupleFilter orFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ orFilter.addChildren(ConstantTupleFilter.FALSE, ConstantTupleFilter.TRUE);
+ Assert.assertEquals(orFilter, notFilter.removeNot());
+ }
+
+ @Test
+ // And(Not(true), false) ==> And(false, false)
+ public void removeNotTest3() {
+ TupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ TupleFilter notFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.NOT);
+ notFilter.addChild(ConstantTupleFilter.TRUE);
+ andFilter.addChildren(notFilter, ConstantTupleFilter.FALSE);
+
+ TupleFilter andFilter2 = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ andFilter2.addChildren(ConstantTupleFilter.FALSE, ConstantTupleFilter.FALSE);
+ Assert.assertEquals(andFilter2, andFilter.removeNot());
+ }
+}
[28/33] kylin git commit: KYLIN-3172 No such file or directory error
with CreateLookupHiveViewMaterializationStep
Posted by li...@apache.org.
KYLIN-3172 No such file or directory error with CreateLookupHiveViewMaterializationStep
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/93a42b3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/93a42b3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/93a42b3e
Branch: refs/heads/sync
Commit: 93a42b3e327e70c754bd509d4ac59d27c707b138
Parents: 490d867
Author: yiming.xu <10...@qq.com>
Authored: Wed Jan 17 18:37:08 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:06:00 2018 +0800
----------------------------------------------------------------------
.../kylin/common/util/HiveCmdBuilder.java | 45 ++++++---------
.../kylin/common/util/HiveCmdBuilderTest.java | 33 +++++++----
.../apache/kylin/source/hive/HiveMRInput.java | 36 ++++++------
.../kylin/source/hive/HiveMRInputTest.java | 59 ++++++++++++++++++++
4 files changed, 115 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/93a42b3e/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index add53db..c33a7fb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -18,11 +18,7 @@
package org.apache.kylin.common.util;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
@@ -30,8 +26,6 @@ import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
@@ -42,9 +36,10 @@ import org.w3c.dom.NodeList;
import com.google.common.collect.Lists;
public class HiveCmdBuilder {
- private static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class);
+ public static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class);
public static final String HIVE_CONF_FILENAME = "kylin_hive_conf";
+ static final String CREATE_HQL_TMP_FILE_TEMPLATE = "cat >%s<<EOL\n%sEOL";
public enum HiveClientMode {
CLI, BEELINE
@@ -68,10 +63,11 @@ public class HiveCmdBuilder {
beelineShell = kylinConfig.getSparkSqlBeelineShell();
beelineParams = kylinConfig.getSparkSqlBeelineParams();
if (StringUtils.isBlank(beelineShell)) {
- throw new IllegalStateException("Missing config 'kylin.source.hive.sparksql-beeline-shell', please check kylin.properties");
+ throw new IllegalStateException(
+ "Missing config 'kylin.source.hive.sparksql-beeline-shell', please check kylin.properties");
}
}
-
+
StringBuffer buf = new StringBuffer();
switch (clientMode) {
@@ -84,37 +80,28 @@ public class HiveCmdBuilder {
buf.append(parseProps());
break;
case BEELINE:
- BufferedWriter bw = null;
- File tmpHql = null;
+ String tmpHqlPath = null;
+ StringBuilder hql = new StringBuilder();
try {
- tmpHql = File.createTempFile("beeline_", ".hql");
- bw = new BufferedWriter(new FileWriter(tmpHql));
+ tmpHqlPath = "/tmp/" + System.currentTimeMillis() + ".hql";
for (String statement : statements) {
- bw.write(statement);
- bw.newLine();
+ hql.append(statement);
+ hql.append("\n");
}
+ String createFileCmd = String.format(CREATE_HQL_TMP_FILE_TEMPLATE, tmpHqlPath, hql);
+ buf.append(createFileCmd);
+ buf.append("\n");
buf.append(beelineShell);
buf.append(" ");
buf.append(beelineParams);
buf.append(parseProps());
buf.append(" -f ");
- buf.append(tmpHql.getAbsolutePath());
+ buf.append(tmpHqlPath);
buf.append(";ret_code=$?;rm -f ");
- buf.append(tmpHql.getAbsolutePath());
+ buf.append(tmpHqlPath);
buf.append(";exit $ret_code");
-
- } catch (IOException e) {
- throw new RuntimeException(e);
} finally {
- IOUtils.closeQuietly(bw);
-
- if (tmpHql != null && logger.isDebugEnabled()) {
- String hql = null;
- try {
- hql = FileUtils.readFileToString(tmpHql, Charset.defaultCharset());
- } catch (IOException e) {
- // ignore
- }
+ if (tmpHqlPath != null && logger.isDebugEnabled()) {
logger.debug("The SQL to execute in beeline: \n" + hql);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/93a42b3e/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index 4262722..ecc8961 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -45,7 +45,7 @@ public class HiveCmdBuilderTest {
System.clearProperty("kylin.source.hive.client");
System.clearProperty("kylin.source.hive.beeline-shell");
System.clearProperty("kylin.source.hive.beeline-params");
-
+
System.clearProperty("kylin.source.hive.enable-sparksql-for-table-ops");
System.clearProperty("kylin.source.hive.sparksql-beeline-shell");
System.clearProperty("kylin.source.hive.sparksql-beeline-params");
@@ -65,12 +65,15 @@ public class HiveCmdBuilderTest {
hiveCmdBuilder.addStatement("SHOW\n TABLES;");
hiveCmdBuilder.setHiveConfProps(hiveProps);
hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite);
- assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
+ assertEquals(
+ "hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez",
+ hiveCmdBuilder.build());
}
@Test
public void testBeeline() throws IOException {
- String lineSeparator = java.security.AccessController.doPrivileged(new sun.security.action.GetPropertyAction("line.separator"));
+ String lineSeparator = java.security.AccessController
+ .doPrivileged(new sun.security.action.GetPropertyAction("line.separator"));
System.setProperty("kylin.source.hive.client", "beeline");
System.setProperty("kylin.source.hive.beeline-shell", "/spark-client/bin/beeline");
System.setProperty("kylin.source.hive.beeline-params", "-u jdbc_url");
@@ -78,17 +81,19 @@ public class HiveCmdBuilderTest {
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement("USE default;");
hiveCmdBuilder.addStatement("DROP TABLE test;");
- hiveCmdBuilder.addStatement("SHOW\n TABLES;");
+ hiveCmdBuilder.addStatement("SHOW TABLES;");
String cmd = hiveCmdBuilder.build();
- assertTrue(cmd.startsWith("/spark-client/bin/beeline -u jdbc_url"));
-
String hqlFile = cmd.substring(cmd.lastIndexOf("-f ") + 3).trim();
hqlFile = hqlFile.substring(0, hqlFile.length() - ";exit $ret_code".length());
-
+ String createFileCmd = cmd.substring(0, cmd.indexOf("EOL\n", cmd.indexOf("EOL\n") + 1) + 3);
+ CliCommandExecutor cliCommandExecutor = new CliCommandExecutor();
+ Pair<Integer, String> execute = cliCommandExecutor.execute(createFileCmd);
String hqlStatement = FileUtils.readFileToString(new File(hqlFile), Charset.defaultCharset());
- assertEquals("USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW\n TABLES;" + lineSeparator, hqlStatement);
-
+ assertEquals(
+ "USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW TABLES;" + lineSeparator,
+ hqlStatement);
+ assertBeelineCmd(cmd);
FileUtils.forceDelete(new File(hqlFile));
}
@@ -101,9 +106,13 @@ public class HiveCmdBuilderTest {
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement("USE default;");
hiveCmdBuilder.addStatement("DROP TABLE test;");
- hiveCmdBuilder.addStatement("SHOW\n TABLES;");
-
+ hiveCmdBuilder.addStatement("SHOW TABLES;");
String cmd = hiveCmdBuilder.build();
- assertTrue(cmd.startsWith("/spark-client/bin/beeline -u jdbc_url"));
+ assertBeelineCmd(cmd);
+ }
+
+ private void assertBeelineCmd(String cmd) {
+ String beelineCmd = cmd.substring(cmd.indexOf("EOL\n", cmd.indexOf("EOL\n") + 1) + 4);
+ assertTrue(beelineCmd.startsWith("/spark-client/bin/beeline -u jdbc_url"));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/93a42b3e/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 6f42961..31a99cd 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -177,7 +177,25 @@ public class HiveMRInput implements IMRInput {
}
protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
- return JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
+
+ String jobWorkingDir = JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
+ // Create work dir to avoid hive create it,
+ // the difference is that the owners are different.
+ checkAndCreateWorkDir(jobWorkingDir);
+ return jobWorkingDir;
+ }
+
+ private void checkAndCreateWorkDir(String jobWorkingDir) {
+ try {
+ Path path = new Path(jobWorkingDir);
+ FileSystem fileSystem = HadoopUtil.getFileSystem(path);
+ if (!fileSystem.exists(path)) {
+ logger.info("Create jobWorkDir : " + jobWorkingDir);
+ fileSystem.mkdirs(path);
+ }
+ } catch (IOException e) {
+ logger.error("Could not create lookUp table dir : " + jobWorkingDir);
+ }
}
private AbstractExecutable createRedistributeFlatHiveTableStep(String hiveInitStatements, String cubeName) {
@@ -210,9 +228,6 @@ public class HiveMRInput implements IMRInput {
if (lookupViewsTables.size() == 0) {
return null;
}
- // Create work dir to avoid hive create it,
- // the difference is that the owners are different.
- checkAndCreateWorkDir(jobWorkingDir);
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride());
@@ -242,19 +257,6 @@ public class HiveMRInput implements IMRInput {
return step;
}
- private void checkAndCreateWorkDir(String jobWorkingDir) {
- try {
- Path path = new Path(jobWorkingDir);
- FileSystem fileSystem = HadoopUtil.getFileSystem(path);
- if (!fileSystem.exists(path)) {
- logger.info("Create jobWorkDir : " + jobWorkingDir);
- fileSystem.mkdirs(path);
- }
- } catch (IOException e) {
- logger.error("Could not create lookUp table dir : " + jobWorkingDir);
- }
- }
-
private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
String cubeName) {
//from hive to hive
http://git-wip-us.apache.org/repos/asf/kylin/blob/93a42b3e/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
new file mode 100644
index 0000000..6d0737d
--- /dev/null
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class HiveMRInputTest {
+
+ @Test
+ public void TestGetJobWorkingDir() throws IOException {
+ FileSystem fileSystem = FileSystem.get(new Configuration());
+ Path jobWorkDirPath = null;
+ try {
+ KylinConfig kylinConfig = mock(KylinConfig.class);
+ KylinConfig.setKylinConfigThreadLocal(kylinConfig);
+ when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
+
+ DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class);
+ defaultChainedExecutable.setId(UUID.randomUUID().toString());
+
+ HiveMRInput.BatchCubingInputSide batchCubingInputSide = new HiveMRInput.BatchCubingInputSide(null);
+ String jobWorkingDir = batchCubingInputSide.getJobWorkingDir(defaultChainedExecutable);
+ jobWorkDirPath = new Path(jobWorkingDir);
+ Assert.assertTrue(fileSystem.exists(jobWorkDirPath));
+ } finally {
+ if (jobWorkDirPath != null)
+ fileSystem.deleteOnExit(jobWorkDirPath);
+ }
+ }
+
+}
\ No newline at end of file
[24/33] kylin git commit: minor refactor
Posted by li...@apache.org.
minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5bdf7231
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5bdf7231
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5bdf7231
Branch: refs/heads/sync
Commit: 5bdf72314827dcb9d8282566691c5bf8f6278dee
Parents: de9b528
Author: Hongbin Ma <ma...@kyligence.io>
Authored: Thu Jan 11 16:47:37 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:01:50 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/request/ModelRequest.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdf7231/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
index 5f07997..2c5b3bf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
@@ -26,6 +26,8 @@ public class ModelRequest {
private boolean successful;
private String message;
private String project;
+
+ private String ccInCheck;
private boolean isSeekingExprAdvice;
public String getUuid() {
@@ -112,4 +114,12 @@ public class ModelRequest {
public void setIsSeekingExprAdvice(boolean seekingExprAdvice) {
isSeekingExprAdvice = seekingExprAdvice;
}
+
+ public String getCcInCheck() {
+ return ccInCheck;
+ }
+
+ public void setCcInCheck(String ccInCheck) {
+ this.ccInCheck = ccInCheck;
+ }
}
[07/33] kylin git commit: Make some hbase config items configurable
at kylin.properties
Posted by li...@apache.org.
Make some hbase config items configurable at kylin.properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6cf78bcf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6cf78bcf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6cf78bcf
Branch: refs/heads/sync
Commit: 6cf78bcf10070910c7fd2a3eca888b7723e369fb
Parents: b10eb7c
Author: nichunen <ch...@kyligence.io>
Authored: Thu Jan 4 19:13:00 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 17:32:46 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/KylinConfigBase.java | 12 ++++++++++++
.../apache/kylin/storage/hbase/HBaseResourceStore.java | 6 +++---
2 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6cf78bcf/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index cfe6fe2..761bb9a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -358,6 +358,18 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.metadata.check-copy-on-write", "false"));
}
+ public String getHbaseClientScannerTimeoutPeriod() {
+ return getOptional("kylin.metadata.hbase-client-scanner-timeout-period", "10000");
+ }
+
+ public String getHbaseRpcTimeout() {
+ return getOptional("kylin.metadata.hbase-rpc-timeout", "5000");
+ }
+
+ public String getHbaseClientRetriesNumber() {
+ return getOptional("kylin.metadata.hbase-client-retries-number", "1");
+ }
+
// ============================================================================
// DICTIONARY & SNAPSHOT
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/6cf78bcf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 433a03c..1f83e3e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -99,9 +99,9 @@ public class HBaseResourceStore extends ResourceStore {
// control timeout for prompt error report
Map<String, String> newParams = new LinkedHashMap<>();
- newParams.put("hbase.client.scanner.timeout.period", "10000");
- newParams.put("hbase.rpc.timeout", "5000");
- newParams.put("hbase.client.retries.number", "1");
+ newParams.put("hbase.client.scanner.timeout.period", kylinConfig.getHbaseClientScannerTimeoutPeriod());
+ newParams.put("hbase.rpc.timeout", kylinConfig.getHbaseRpcTimeout());
+ newParams.put("hbase.client.retries.number", kylinConfig.getHbaseClientRetriesNumber());
newParams.putAll(url.getAllParameters());
return url.copy(newParams);
[18/33] kylin git commit: KYLIN-1664 Revert the incorrect change.
Posted by li...@apache.org.
KYLIN-1664 Revert the incorrect change.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b00661d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b00661d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b00661d9
Branch: refs/heads/sync
Commit: b00661d9b6888a28c63deec1b6b76109b6454e7b
Parents: 3116abf
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 15 15:05:27 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 18:54:01 2018 +0800
----------------------------------------------------------------------
.../kylin/rest/service/AdminServiceTest.java | 35 +++++++++++++-------
1 file changed, 23 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b00661d9/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
index 3ee14e0..ce46536 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
@@ -46,17 +46,28 @@ public class AdminServiceTest extends ServiceTestBase {
String path = Thread.currentThread().getContextClassLoader().getResource("kylin.properties").getPath();
KylinConfig.setKylinConfigThreadLocal(KylinConfig.createInstanceFromUri(path));
- String publicConfig = adminService.getPublicConfig();
-
- Assert.assertFalse(publicConfig.contains("kylin.metadata.data-model-manager-impl"));
- Assert.assertFalse(publicConfig.contains("kylin.dictionary.use-forest-trie"));
- Assert.assertFalse(publicConfig.contains("kylin.cube.segment-advisor"));
- Assert.assertFalse(publicConfig.contains("kylin.job.use-remote-cli"));
- Assert.assertFalse(publicConfig.contains("kylin.job.scheduler.provider"));
- Assert.assertFalse(publicConfig.contains("kylin.engine.mr.job-jar"));
- Assert.assertFalse(publicConfig.contains("kylin.engine.spark.sanity-check-enabled"));
- Assert.assertFalse(publicConfig.contains("kylin.storage.provider"));
- Assert.assertFalse(publicConfig.contains("kylin.query.convert-create-table-to-with"));
- Assert.assertFalse(publicConfig.contains("kylin.server.init-tasks"));
+ String expected = "kylin.web.link-streaming-guide=http://kylin.apache.org/\n" +
+ "kylin.web.contact-mail=\n" +
+ "kylin.query.cache-enabled=true\n" +
+ "kylin.web.link-diagnostic=\n" +
+ "kylin.web.help.length=4\n" +
+ "kylin.web.timezone=GMT+8\n" +
+ "kylin.server.external-acl-provider=\n" +
+ "kylin.storage.default=2\n" +
+ "kylin.web.help=\n" +
+ "kylin.web.export-allow-other=true\n" +
+ "kylin.web.link-hadoop=\n" +
+ "kylin.web.hide-measures=RAW\n" +
+ "kylin.htrace.show-gui-trace-toggle=false\n" +
+ "kylin.web.export-allow-admin=true\n" +
+ "kylin.env=QA\n" +
+ "kylin.web.hive-limit=20\n" +
+ "kylin.engine.default=2\n" +
+ "kylin.web.help.3=onboard|Cube Design Tutorial|\n" +
+ "kylin.web.help.2=tableau|Tableau Guide|\n" +
+ "kylin.web.help.1=odbc|ODBC Driver|\n" +
+ "kylin.web.help.0=start|Getting Started|\n" +
+ "kylin.security.profile=testing\n";
+ Assert.assertEquals(expected, adminService.getPublicConfig());
}
}
[06/33] kylin git commit: minor, add array type test
Posted by li...@apache.org.
minor, add array type test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b10eb7cb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b10eb7cb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b10eb7cb
Branch: refs/heads/sync
Commit: b10eb7cbaf61355535d67d7fc542a72e39ad976f
Parents: bd048b9
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jan 4 16:52:23 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 17:31:36 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/query/schema/OLAPTable.java | 2 +-
.../kylin/query/schema/OLAPTableTest.java | 78 ++++++++++++++++++++
2 files changed, 79 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b10eb7cb/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index ca8807e..b5313ed 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -88,7 +88,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
SQLTYPE_MAPPING.put("timestamp", SqlTypeName.TIMESTAMP);
SQLTYPE_MAPPING.put("any", SqlTypeName.ANY);
- REGEX_SQLTYPE_MAPPING.put("array\\<.*\\>", SqlTypeName.ARRAY);
+ REGEX_SQLTYPE_MAPPING.put("array\\<.+\\>", SqlTypeName.ARRAY);
}
private final boolean exposeMore;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b10eb7cb/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java b/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
new file mode 100644
index 0000000..00440de
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.schema;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.sql.type.ArraySqlType;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class OLAPTableTest extends LocalFileMetadataTestCase {
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testCreateSqlType() {
+ RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
+ DataType kylinDataType = DataType.getType("array<string>");
+
+ RelDataType relDataType = OLAPTable.createSqlType(typeFactory, kylinDataType, true);
+ Assert.assertTrue(relDataType instanceof ArraySqlType);
+ Assert.assertEquals(SqlTypeName.ARRAY, relDataType.getSqlTypeName());
+ Assert.assertTrue(relDataType.getComponentType() instanceof BasicSqlType);
+ Assert.assertEquals(SqlTypeName.VARCHAR, relDataType.getComponentType().getSqlTypeName());
+ Assert.assertTrue(relDataType.isNullable());
+
+ kylinDataType = DataType.getType("array<>");
+ boolean catchedEx = false;
+ try {
+ OLAPTable.createSqlType(typeFactory, kylinDataType, true);
+ } catch (IllegalArgumentException ex) {
+ Assert.assertEquals("Unrecognized data type array<>", ex.getMessage());
+ catchedEx = true;
+ }
+ Assert.assertTrue(catchedEx);
+
+ kylinDataType = DataType.getType("array<AAA>");
+ catchedEx = false;
+ try {
+ OLAPTable.createSqlType(typeFactory, kylinDataType, true);
+ } catch (IllegalArgumentException ex) {
+ Assert.assertTrue(ex.getMessage().startsWith("bad data type -- aaa"));
+ catchedEx = true;
+ }
+ Assert.assertTrue(catchedEx);
+ }
+}
[20/33] kylin git commit: KYLIN-2760, remove useless ACL code.
Posted by li...@apache.org.
KYLIN-2760, remove useless ACL code.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8758188
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8758188
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8758188
Branch: refs/heads/sync
Commit: a87581881ba74051e63d3f15a657d040bced8ba0
Parents: aa57e4c
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 15 17:40:47 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 18:57:41 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/AccessService.java | 1 +
.../org/apache/kylin/rest/service/CubeService.java | 12 ------------
.../org/apache/kylin/rest/service/JobService.java | 17 -----------------
.../apache/kylin/rest/service/ModelService.java | 13 -------------
4 files changed, 1 insertion(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a8758188/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 3732c15..8498ecd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -218,6 +218,7 @@ public class AccessService {
return acl;
}
+ @Deprecated
@Transactional
public void inherit(AclEntity ae, AclEntity parentAe) {
Message msg = MsgPicker.getMsg();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a8758188/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a81d189..73fb4b6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -68,7 +68,6 @@ import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.MetricsResponse;
-import org.apache.kylin.rest.security.AclPermission;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
import org.slf4j.Logger;
@@ -100,10 +99,6 @@ public class CubeService extends BasicService implements InitializingBean {
protected Cache<String, HBaseResponse> htableInfoCache = CacheBuilder.newBuilder().build();
@Autowired
- @Qualifier("accessService")
- private AccessService accessService;
-
- @Autowired
@Qualifier("projectService")
private ProjectService projectService;
@@ -213,10 +208,6 @@ public class CubeService extends BasicService implements InitializingBean {
logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids");
createdCube = getCubeManager().createCube(cubeName, project.getName(), createdDesc, owner);
- accessService.init(createdCube, AclPermission.ADMINISTRATION);
-
- accessService.inherit(createdCube, project);
-
return createdCube;
}
@@ -286,8 +277,6 @@ public class CubeService extends BasicService implements InitializingBean {
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
ProjectInstance newProject = projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(),
newProjectName, owner);
-
- accessService.inherit(cube, newProject);
}
return updatedCubeDesc;
@@ -312,7 +301,6 @@ public class CubeService extends BasicService implements InitializingBean {
int cubeNum = getCubeManager().getCubesByDesc(cube.getDescriptor().getName()).size();
getCubeManager().dropCube(cube.getName(), cubeNum == 1);//only delete cube desc when no other cube is using it
- accessService.clean(cube, true);
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/a8758188/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index bd90742..e447031 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -74,7 +74,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
@@ -97,10 +96,6 @@ public class JobService extends BasicService implements InitializingBean {
private static final Logger logger = LoggerFactory.getLogger(JobService.class);
@Autowired
- @Qualifier("accessService")
- private AccessService accessService;
-
- @Autowired
private AclEvaluate aclEvaluate;
/*
@@ -213,9 +208,6 @@ public class JobService extends BasicService implements InitializingBean {
JobInstance jobInstance = submitJobInternal(cube, tsRange, segRange, sourcePartitionOffsetStart,
sourcePartitionOffsetEnd, buildType, force, submitter);
- accessService.init(jobInstance, null);
- accessService.inherit(jobInstance, cube);
-
return jobInstance;
}
@@ -282,13 +274,6 @@ public class JobService extends BasicService implements InitializingBean {
String submitter) throws IOException, JobException {
Pair<JobInstance, List<JobInstance>> result = submitOptimizeJobInternal(cube, cuboidsRecommend, submitter);
- accessService.init(result.getFirst(), null);
- accessService.inherit(result.getFirst(), cube);
- for (JobInstance jobInstance : result.getSecond()) {
- accessService.init(jobInstance, null);
- accessService.inherit(jobInstance, cube);
- }
-
return result;
}
@@ -391,8 +376,6 @@ public class JobService extends BasicService implements InitializingBean {
getExecutableManager().addJob(optimizeJob);
JobInstance optimizeJobInstance = getSingleJobInstance(optimizeJob);
- accessService.init(optimizeJobInstance, null);
- accessService.inherit(optimizeJobInstance, cubeInstance);
/** Update the checkpoint job */
checkpointExecutable.getSubTasksForCheck().set(checkpointExecutable.getSubTasksForCheck().indexOf(toBeReplaced),
http://git-wip-us.apache.org/repos/asf/kylin/blob/a8758188/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index ee01f3e..463107b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -42,7 +42,6 @@ import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
-import org.apache.kylin.rest.security.AclPermission;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
import org.slf4j.Logger;
@@ -65,10 +64,6 @@ public class ModelService extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(ModelService.class);
@Autowired
- @Qualifier("accessService")
- private AccessService accessService;
-
- @Autowired
@Qualifier("cubeMgmtService")
private CubeService cubeService;
@@ -142,12 +137,6 @@ public class ModelService extends BasicService {
DataModelDesc createdDesc = null;
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
createdDesc = getDataModelManager().createDataModelDesc(desc, projectName, owner);
-
- if (!desc.isDraft()) {
- accessService.init(createdDesc, AclPermission.ADMINISTRATION);
- ProjectInstance project = getProjectManager().getProject(projectName);
- accessService.inherit(createdDesc, project);
- }
return createdDesc;
}
@@ -169,8 +158,6 @@ public class ModelService extends BasicService {
}
getDataModelManager().dropModel(desc);
-
- accessService.clean(desc, true);
}
public boolean isTableInAnyModel(TableDesc table) {
[21/33] kylin git commit: KYLIN-2760,
change annotation to AclEvaluate call.
Posted by li...@apache.org.
KYLIN-2760, change annotation to AclEvaluate call.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59779854
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59779854
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59779854
Branch: refs/heads/sync
Commit: 597798544f5164b215a395d35a52e3f36354e410
Parents: a875818
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 15 20:19:37 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 18:58:10 2018 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/service/CubeService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/59779854/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 73fb4b6..a81cc24 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -827,9 +827,9 @@ public class CubeService extends BasicService implements InitializingBean {
return formattedRollingUpCount;
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION')")
public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+ aclEvaluate.checkProjectAdminPermission(cube.getProject());
return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
}
}
[19/33] kylin git commit: KYLIN-3090, complete UUID if entity hasn't.
Posted by li...@apache.org.
KYLIN-3090, complete UUID if entity hasn't.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa57e4c3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa57e4c3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa57e4c3
Branch: refs/heads/sync
Commit: aa57e4c391933b4bd17d17d145be6375fd9d996b
Parents: b00661d
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 15 18:48:57 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 18:55:26 2018 +0800
----------------------------------------------------------------------
.../metadata/cachesync/CachedCrudAssist.java | 8 ++-
.../cachesync/CachedCrudAssistTest.java | 63 ++++++++++++++++++++
2 files changed, 70 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa57e4c3/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
index c7244fe..b3c200e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
@@ -173,7 +173,7 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> {
public T save(T entity) throws IOException {
Preconditions.checkArgument(entity != null);
- Preconditions.checkArgument(entity.getUuid() != null);
+ completeUuidIfNeeded(entity);
Preconditions.checkArgument(entityType.isInstance(entity));
String resName = entity.resourceName();
@@ -199,6 +199,12 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> {
return reload(resName);
}
+ private void completeUuidIfNeeded(T entity) {
+ if (entity.getUuid() == null) {
+ entity.updateRandomUuid();
+ }
+ }
+
public void delete(T entity) throws IOException {
delete(entity.resourceName());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa57e4c3/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
new file mode 100644
index 0000000..482ec09
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.cachesync;
+
+import java.io.IOException;
+
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CachedCrudAssistTest extends LocalFileMetadataTestCase {
+ @Before
+ public void setup() throws Exception {
+ staticCreateTestMetadata();
+ }
+
+ @Test
+ public void testSave() throws IOException {
+ CachedCrudAssist<TestEntity> crudAssist = new CachedCrudAssist<TestEntity>(getStore() //
+ , "/test" //
+ , TestEntity.class //
+ , new CaseInsensitiveStringCache<TestEntity>(getTestConfig(), "test")) { //
+ @Override
+ protected TestEntity initEntityAfterReload(TestEntity entity, String resourceName) {
+ return entity;
+ }
+ };
+
+ TestEntity entity = new TestEntity();
+ Assert.assertNull(entity.getUuid());
+ crudAssist.save(entity);
+ Assert.assertNotNull(entity.getUuid());
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanAfterClass();
+ }
+}
+
+class TestEntity extends RootPersistentEntity {
+ public TestEntity() {
+ }
+}
\ No newline at end of file
[05/33] kylin git commit: minor,
restore OLAPToEnumerableConverter to original
Posted by li...@apache.org.
minor, restore OLAPToEnumerableConverter to original
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bd048b96
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bd048b96
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bd048b96
Branch: refs/heads/sync
Commit: bd048b96971ed92e5e5dc1cc05173bd678ae3c24
Parents: 91ed19b
Author: Roger Shi <ro...@hotmail.com>
Authored: Wed Jan 3 22:21:28 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 17:30:46 2018 +0800
----------------------------------------------------------------------
.../apache/calcite/plan/OLAPRelMdRowCount.java | 174 ----------------
.../calcite/plan/OLAPRelMetadataProvider.java | 39 ----
.../org/apache/calcite/plan/RelOptCluster.java | 199 -------------------
.../src/test/resources/query/sql/query110.sql | 58 ++++++
.../relnode/OLAPToEnumerableConverter.java | 2 +-
5 files changed, 59 insertions(+), 413 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bd048b96/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java b/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
deleted file mode 100644
index 751561f..0000000
--- a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
+++ /dev/null
@@ -1,174 +0,0 @@
-package org.apache.calcite.plan;
-
-import org.apache.calcite.adapter.enumerable.EnumerableLimit;
-import org.apache.calcite.plan.volcano.RelSubset;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.SingleRel;
-import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.Calc;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Intersect;
-import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.core.Minus;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.SemiJoin;
-import org.apache.calcite.rel.core.Sort;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.core.Union;
-import org.apache.calcite.rel.core.Values;
-import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMdRowCount;
-import org.apache.calcite.rel.metadata.RelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.util.BuiltInMethod;
-
-public class OLAPRelMdRowCount extends RelMdRowCount {
- public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider
- .reflectiveSource(BuiltInMethod.ROW_COUNT.method, new OLAPRelMdRowCount());
-
- //avoid case like sql/query60.sql, which will generate plan:
- /*
- EnumerableLimit(fetch=[3])
- OLAPToEnumerableConverter
- OLAPAggregateRel(group=[{0}], SUM_PRICE=[SUM($1)], CNT_1=[COUNT()], TOTAL_ITEMS=[SUM($2)])
- */
- private boolean shouldIntercept(RelNode rel) {
- for (RelNode input : rel.getInputs()) {
- if (input instanceof RelSubset) {
- RelSubset relSubset = (RelSubset) input;
- if (relSubset.getBest() != null
- && relSubset.getBest().getClass().getCanonicalName().endsWith("OLAPToEnumerableConverter")) {
- return true;
- }
- }
- }
- return false;
- }
-
- @Override
- public Double getRowCount(RelNode rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(RelSubset subset, RelMetadataQuery mq) {
- if (shouldIntercept(subset))
- return 1E10;
-
- return super.getRowCount(subset, mq);
- }
-
- @Override
- public Double getRowCount(Union rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Intersect rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Minus rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Filter rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Calc rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Project rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Sort rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(EnumerableLimit rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(SingleRel rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Join rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(SemiJoin rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Aggregate rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(TableScan rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Values rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bd048b96/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMetadataProvider.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMetadataProvider.java b/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMetadataProvider.java
deleted file mode 100644
index ad944db..0000000
--- a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMetadataProvider.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.apache.calcite.plan;
-
-import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMdAllPredicates;
-import org.apache.calcite.rel.metadata.RelMdCollation;
-import org.apache.calcite.rel.metadata.RelMdColumnOrigins;
-import org.apache.calcite.rel.metadata.RelMdColumnUniqueness;
-import org.apache.calcite.rel.metadata.RelMdDistinctRowCount;
-import org.apache.calcite.rel.metadata.RelMdDistribution;
-import org.apache.calcite.rel.metadata.RelMdExplainVisibility;
-import org.apache.calcite.rel.metadata.RelMdExpressionLineage;
-import org.apache.calcite.rel.metadata.RelMdMaxRowCount;
-import org.apache.calcite.rel.metadata.RelMdMemory;
-import org.apache.calcite.rel.metadata.RelMdMinRowCount;
-import org.apache.calcite.rel.metadata.RelMdNodeTypes;
-import org.apache.calcite.rel.metadata.RelMdParallelism;
-import org.apache.calcite.rel.metadata.RelMdPercentageOriginalRows;
-import org.apache.calcite.rel.metadata.RelMdPopulationSize;
-import org.apache.calcite.rel.metadata.RelMdPredicates;
-import org.apache.calcite.rel.metadata.RelMdSelectivity;
-import org.apache.calcite.rel.metadata.RelMdSize;
-import org.apache.calcite.rel.metadata.RelMdTableReferences;
-import org.apache.calcite.rel.metadata.RelMdUniqueKeys;
-
-import com.google.common.collect.ImmutableList;
-
-public class OLAPRelMetadataProvider extends ChainedRelMetadataProvider {
- public static final OLAPRelMetadataProvider INSTANCE = new OLAPRelMetadataProvider();
-
- public OLAPRelMetadataProvider() {
- super(ImmutableList.of(RelMdPercentageOriginalRows.SOURCE, RelMdColumnOrigins.SOURCE,
- RelMdExpressionLineage.SOURCE, RelMdTableReferences.SOURCE, RelMdNodeTypes.SOURCE,
- OLAPRelMdRowCount.SOURCE, RelMdMaxRowCount.SOURCE, RelMdMinRowCount.SOURCE, RelMdUniqueKeys.SOURCE,
- RelMdColumnUniqueness.SOURCE, RelMdPopulationSize.SOURCE, RelMdSize.SOURCE, RelMdParallelism.SOURCE,
- RelMdDistribution.SOURCE, RelMdMemory.SOURCE, RelMdDistinctRowCount.SOURCE, RelMdSelectivity.SOURCE,
- RelMdExplainVisibility.SOURCE, RelMdPredicates.SOURCE, RelMdAllPredicates.SOURCE,
- RelMdCollation.SOURCE));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bd048b96/atopcalcite/src/main/java/org/apache/calcite/plan/RelOptCluster.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/plan/RelOptCluster.java b/atopcalcite/src/main/java/org/apache/calcite/plan/RelOptCluster.java
deleted file mode 100644
index 62a3ef5..0000000
--- a/atopcalcite/src/main/java/org/apache/calcite/plan/RelOptCluster.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.plan;
-
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.CorrelationId;
-import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
-import org.apache.calcite.rel.metadata.MetadataFactory;
-import org.apache.calcite.rel.metadata.MetadataFactoryImpl;
-import org.apache.calcite.rel.metadata.RelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexNode;
-
-import com.google.common.base.Preconditions;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/*
- * OVERRIDE POINT:
- * - grep OLAPRelMetadataProvider();
- */
-/**
- * An environment for related relational expressions during the
- * optimization of a query.
- */
-public class RelOptCluster {
- //~ Instance fields --------------------------------------------------------
-
- private final RelDataTypeFactory typeFactory;
- private final RelOptPlanner planner;
- private final AtomicInteger nextCorrel;
- private final Map<String, RelNode> mapCorrelToRel;
- private RexNode originalExpression;
- private final RexBuilder rexBuilder;
- private RelMetadataProvider metadataProvider;
- private MetadataFactory metadataFactory;
- private final RelTraitSet emptyTraitSet;
- private RelMetadataQuery mq;
-
- //~ Constructors -----------------------------------------------------------
-
- /**
- * Creates a cluster.
- */
- @Deprecated // to be removed before 2.0
- RelOptCluster(
- RelOptQuery query,
- RelOptPlanner planner,
- RelDataTypeFactory typeFactory,
- RexBuilder rexBuilder) {
- this(planner, typeFactory, rexBuilder, query.nextCorrel,
- query.mapCorrelToRel);
- }
-
- /**
- * Creates a cluster.
- *
- * <p>For use only from {@link #create} and {@link RelOptQuery}.
- */
- RelOptCluster(RelOptPlanner planner, RelDataTypeFactory typeFactory,
- RexBuilder rexBuilder, AtomicInteger nextCorrel,
- Map<String, RelNode> mapCorrelToRel) {
- this.nextCorrel = nextCorrel;
- this.mapCorrelToRel = mapCorrelToRel;
- this.planner = Preconditions.checkNotNull(planner);
- this.typeFactory = Preconditions.checkNotNull(typeFactory);
- this.rexBuilder = rexBuilder;
- this.originalExpression = rexBuilder.makeLiteral("?");
-
- // set up a default rel metadata provider,
- // giving the planner first crack at everything
- setMetadataProvider(OLAPRelMetadataProvider.INSTANCE);
- this.emptyTraitSet = planner.emptyTraitSet();
- assert emptyTraitSet.size() == planner.getRelTraitDefs().size();
- }
-
- /** Creates a cluster. */
- public static RelOptCluster create(RelOptPlanner planner,
- RexBuilder rexBuilder) {
- return new RelOptCluster(planner, rexBuilder.getTypeFactory(),
- rexBuilder, new AtomicInteger(0), new HashMap<String, RelNode>());
- }
-
- //~ Methods ----------------------------------------------------------------
-
- @Deprecated // to be removed before 2.0
- public RelOptQuery getQuery() {
- return new RelOptQuery(planner, nextCorrel, mapCorrelToRel);
- }
-
- @Deprecated // to be removed before 2.0
- public RexNode getOriginalExpression() {
- return originalExpression;
- }
-
- @Deprecated // to be removed before 2.0
- public void setOriginalExpression(RexNode originalExpression) {
- this.originalExpression = originalExpression;
- }
-
- public RelOptPlanner getPlanner() {
- return planner;
- }
-
- public RelDataTypeFactory getTypeFactory() {
- return typeFactory;
- }
-
- public RexBuilder getRexBuilder() {
- return rexBuilder;
- }
-
- public RelMetadataProvider getMetadataProvider() {
- return metadataProvider;
- }
-
- /**
- * Overrides the default metadata provider for this cluster.
- *
- * @param metadataProvider custom provider
- */
- public void setMetadataProvider(RelMetadataProvider metadataProvider) {
- this.metadataProvider = metadataProvider;
- this.metadataFactory = new MetadataFactoryImpl(metadataProvider);
- }
-
- public MetadataFactory getMetadataFactory() {
- return metadataFactory;
- }
-
- /** Returns the current RelMetadataQuery.
- *
- * <p>This method might be changed or moved in future.
- * If you have a {@link RelOptRuleCall} available,
- * for example if you are in a {@link RelOptRule#onMatch(RelOptRuleCall)}
- * method, then use {@link RelOptRuleCall#getMetadataQuery()} instead. */
- public RelMetadataQuery getMetadataQuery() {
- if (mq == null) {
- mq = RelMetadataQuery.instance();
- }
- return mq;
- }
-
- /**
- * Should be called whenever the current {@link RelMetadataQuery} becomes
- * invalid. Typically invoked from {@link RelOptRuleCall#transformTo}.
- */
- public void invalidateMetadataQuery() {
- mq = null;
- }
-
- /**
- * Constructs a new id for a correlating variable. It is unique within the
- * whole query.
- */
- public CorrelationId createCorrel() {
- return new CorrelationId(nextCorrel.getAndIncrement());
- }
-
- /** Returns the default trait set for this cluster. */
- public RelTraitSet traitSet() {
- return emptyTraitSet;
- }
-
- /** @deprecated For {@code traitSetOf(t1, t2)},
- * use {@link #traitSet}().replace(t1).replace(t2). */
- @Deprecated // to be removed before 2.0
- public RelTraitSet traitSetOf(RelTrait... traits) {
- RelTraitSet traitSet = emptyTraitSet;
- for (RelTrait trait : traits) {
- traitSet = traitSet.replace(trait);
- }
- return traitSet;
- }
-
- public RelTraitSet traitSetOf(RelTrait trait) {
- return emptyTraitSet.replace(trait);
- }
-}
-
-// End RelOptCluster.java
http://git-wip-us.apache.org/repos/asf/kylin/blob/bd048b96/kylin-it/src/test/resources/query/sql/query110.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query110.sql b/kylin-it/src/test/resources/query/sql/query110.sql
new file mode 100644
index 0000000..07cd3b2
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query110.sql
@@ -0,0 +1,58 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+SELECT TEST_KYLIN_FACT.CAL_DT as TEST_KYLIN_FACT_CAL_DT,
+ TEST_KYLIN_FACT.ORDER_ID as TEST_KYLIN_FACT_ORDER_ID,
+ TEST_ORDER.TEST_DATE_ENC as TEST_ORDER_TEST_DATE_ENC,
+ TEST_ORDER.TEST_TIME_ENC as TEST_ORDER_TEST_TIME_ENC,
+ TEST_KYLIN_FACT.LEAF_CATEG_ID as TEST_KYLIN_FACT_LEAF_CATEG_ID,
+ TEST_KYLIN_FACT.LSTG_SITE_ID as TEST_KYLIN_FACT_LSTG_SITE_ID,
+ TEST_CATEGORY_GROUPINGS.META_CATEG_NAME as TEST_CATEGORY_GROUPINGS_META_CATEG_NAME,
+ TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME as TEST_CATEGORY_GROUPINGS_CATEG_LVL2_NAME,
+ TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME as TEST_CATEGORY_GROUPINGS_CATEG_LVL3_NAME,
+ TEST_KYLIN_FACT.LSTG_FORMAT_NAME as TEST_KYLIN_FACT_LSTG_FORMAT_NAME,
+ TEST_KYLIN_FACT.SLR_SEGMENT_CD as TEST_KYLIN_FACT_SLR_SEGMENT_CD,
+ TEST_KYLIN_FACT.SELLER_ID as TEST_KYLIN_FACT_SELLER_ID,
+ SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL as SELLER_ACCOUNT_ACCOUNT_BUYER_LEVEL,
+ SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL as SELLER_ACCOUNT_ACCOUNT_SELLER_LEVEL,
+ SELLER_ACCOUNT.ACCOUNT_COUNTRY as SELLER_ACCOUNT_ACCOUNT_COUNTRY,
+ SELLER_COUNTRY.NAME as SELLER_COUNTRY_NAME,
+ TEST_ORDER.BUYER_ID as TEST_ORDER_BUYER_ID,
+ BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL as BUYER_ACCOUNT_ACCOUNT_BUYER_LEVEL,
+ BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL as BUYER_ACCOUNT_ACCOUNT_SELLER_LEVEL,
+ BUYER_ACCOUNT.ACCOUNT_COUNTRY as BUYER_ACCOUNT_ACCOUNT_COUNTRY,
+ year(TEST_KYLIN_FACT.CAL_DT) as TEST_KYLIN_FACT_DEAL_YEAR
+ FROM "DEFAULT".TEST_KYLIN_FACT as TEST_KYLIN_FACT
+ LEFT JOIN "DEFAULT".TEST_ORDER as TEST_ORDER
+ ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+ LEFT JOIN EDW.TEST_CAL_DT as TEST_CAL_DT
+ ON TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT
+ LEFT JOIN "DEFAULT".TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+ LEFT JOIN EDW.TEST_SITES as TEST_SITES
+ ON TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_SITES.SITE_ID
+ LEFT JOIN EDW.TEST_SELLER_TYPE_DIM as TEST_SELLER_TYPE_DIM
+ ON TEST_KYLIN_FACT.SLR_SEGMENT_CD = TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD
+ LEFT JOIN "DEFAULT".TEST_ACCOUNT as SELLER_ACCOUNT
+ ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+ LEFT JOIN "DEFAULT".TEST_ACCOUNT as BUYER_ACCOUNT
+ ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+ LEFT JOIN "DEFAULT".TEST_COUNTRY as SELLER_COUNTRY
+ ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
+ LEFT JOIN "DEFAULT".TEST_COUNTRY as BUYER_COUNTRY
+ ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+ WHERE 1=1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bd048b96/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 697827f..89c9041 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -58,7 +58,7 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
// huge cost to ensure OLAPToEnumerableConverter only appears once in rel tree
- return planner.getCostFactory().makeCost(1E10, 0, 0);
+ return planner.getCostFactory().makeCost(1E100, 0, 0);
}
@Override
[10/33] kylin git commit: KYLIN-3083 fix potential overflow in
CubeHBaseRPC#getCoprocessorTimeoutMillis
Posted by li...@apache.org.
KYLIN-3083 fix potential overflow in CubeHBaseRPC#getCoprocessorTimeoutMillis
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/19381d31
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/19381d31
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/19381d31
Branch: refs/heads/sync
Commit: 19381d31198735195647433754d3b40ec0ee42d6
Parents: d6cc4e0
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 8 19:07:54 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 17:35:43 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/19381d31/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 1e2fbd6..2e82140 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -293,7 +293,7 @@ public abstract class CubeHBaseRPC implements IGTStorage {
if (BackdoorToggles.getQueryTimeout() != -1) {
coopTimeout = BackdoorToggles.getQueryTimeout();
} else {
- coopTimeout = cubeSeg.getConfig().getQueryCoprocessorTimeoutSeconds() * 1000;
+ coopTimeout = cubeSeg.getConfig().getQueryCoprocessorTimeoutSeconds() * 1000L;
}
int rpcTimeout;
[12/33] kylin git commit: KYLIN-3157,
enhancement kylin's query timeout.
Posted by li...@apache.org.
KYLIN-3157, enhancement kylin's query timeout.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/13f64f48
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/13f64f48
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/13f64f48
Branch: refs/heads/sync
Commit: 13f64f48cc8111382840f4b0510b40df08448ace
Parents: 98b1047
Author: Li Yang <li...@apache.org>
Authored: Fri Jan 26 18:39:57 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 18:39:57 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 13 ++-
.../org/apache/kylin/common/QueryContext.java | 21 +---
.../apache/kylin/storage/StorageFactory.java | 4 +
.../gtrecord/GTCubeStorageQueryBase.java | 3 +-
.../gtrecord/SequentialCubeTupleIterator.java | 5 +-
.../kylin/rest/service/BadQueryDetector.java | 17 ++-
.../apache/kylin/rest/service/QueryService.java | 1 +
.../rest/service/BadQueryDetectorTest.java | 4 +-
.../rest/service/KyilnQueryTimeoutTest.java | 104 +++++++++++++++++++
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 7 +-
10 files changed, 144 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/13f64f48/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 761bb9a..df01842 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1266,7 +1266,11 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getBadQueryDefaultDetectIntervalSeconds() {
- return Integer.parseInt(getOptional("kylin.query.badquery-detect-interval", "60"));
+ int time = getQueryTimeoutSeconds() / 2; // half of query timeout
+ if (time == 0) {
+ time = 60; // 60 sec
+ }
+ return time;
}
public boolean getBadQueryPersistentEnabled() {
@@ -1324,7 +1328,12 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getQueryTimeoutSeconds() {
- return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
+ int time = Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
+ if (time != 0 && time <= 60) {
+ logger.warn("query timeout seconds less than 60 sec, set to 60 sec.");
+ time = 60;
+ }
+ return time;
}
public boolean isPushDownEnabled() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/13f64f48/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index cb1b09c..1aa94d3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -45,7 +45,6 @@ public class QueryContext {
}
private long queryStartMillis;
- private long deadline = Long.MAX_VALUE;
private final String queryId;
private String username;
@@ -75,26 +74,10 @@ public class QueryContext {
return queryStartMillis;
}
- public void setDeadline(long timeoutMillis) {
- if (timeoutMillis > 0) {
- deadline = queryStartMillis + timeoutMillis;
- }
- }
-
- public long getDeadline() {
- return deadline;
- }
-
- /**
- * @return millis before deadline
- * @throws KylinTimeoutException if deadline has passed
- */
- public long checkMillisBeforeDeadline() {
- long remain = deadline - System.currentTimeMillis();
- if (remain <= 0) {
+ public void checkMillisBeforeDeadline() {
+ if (Thread.interrupted()) {
throw new KylinTimeoutException("Query timeout");
}
- return remain;
}
public String getQueryId() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/13f64f48/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
index 79b93fe..3505708 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
@@ -39,6 +39,10 @@ public class StorageFactory {
return current.get(aware.getStorageType());
}
+ public static void clearCache() {
+ storages.remove();
+ }
+
public static IStorageQuery createQuery(IRealization realization) {
return storage(realization).createQuery(realization);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/13f64f48/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index ae1f64f..2f69b76 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -160,8 +160,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
sqlDigest.aggregations, context);
// set whether to aggregate results from multiple partitions
enableStreamAggregateIfBeneficial(cuboid, groupsD, context);
- // set and check query deadline
- QueryContextFacade.current().setDeadline(cubeInstance.getConfig().getQueryTimeoutSeconds() * 1000);
+ // check query deadline
QueryContextFacade.current().checkMillisBeforeDeadline();
// push down having clause filter if possible
http://git-wip-us.apache.org/repos/asf/kylin/blob/13f64f48/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index 72417bf..c067e33 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -144,10 +144,9 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
if (scanCount++ % 100 == 1) {
QueryContextFacade.current().checkMillisBeforeDeadline();
}
-
- if (++scanCountDelta >= 1000)
+ if (++scanCountDelta >= 1000) {
flushScanCountDelta();
-
+ }
return tupleIterator.next();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/13f64f48/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
index 7410c9c..4f7bccf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
@@ -44,7 +44,8 @@ public class BadQueryDetector extends Thread {
private final int alertMB;
private final int alertRunningSec;
private KylinConfig kylinConfig;
- private ArrayList<Notifier> notifiers = new ArrayList<Notifier>();
+ private ArrayList<Notifier> notifiers = new ArrayList<>();
+ private int queryTimeoutSeconds;
public BadQueryDetector() {
super("BadQueryDetector");
@@ -53,17 +54,19 @@ public class BadQueryDetector extends Thread {
this.detectionInterval = kylinConfig.getBadQueryDefaultDetectIntervalSeconds() * 1000L;
this.alertMB = 100;
this.alertRunningSec = kylinConfig.getBadQueryDefaultAlertingSeconds();
+ this.queryTimeoutSeconds = kylinConfig.getQueryTimeoutSeconds();
initNotifiers();
}
- public BadQueryDetector(long detectionInterval, int alertMB, int alertRunningSec) {
+ public BadQueryDetector(long detectionInterval, int alertMB, int alertRunningSec, int queryTimeoutSeconds) {
super("BadQueryDetector");
this.setDaemon(true);
this.detectionInterval = detectionInterval;
this.alertMB = alertMB;
this.alertRunningSec = alertRunningSec;
this.kylinConfig = KylinConfig.getInstanceFromEnv();
+ this.queryTimeoutSeconds = queryTimeoutSeconds;
initNotifiers();
}
@@ -121,6 +124,7 @@ public class BadQueryDetector extends Thread {
notify(badReason, entry);
}
+ @Override
public void run() {
while (true) {
try {
@@ -147,6 +151,8 @@ public class BadQueryDetector extends Thread {
// report if query running long
for (Entry e : entries) {
float runningSec = (float) (now - e.startTime) / 1000;
+ setQueryThreadInterrupted(e, runningSec);
+
if (runningSec >= alertRunningSec) {
notify(BadQueryEntry.ADJ_SLOW, e);
dumpStackTrace(e.thread);
@@ -161,6 +167,13 @@ public class BadQueryDetector extends Thread {
}
}
+ private void setQueryThreadInterrupted(Entry e, float runningSec) {
+ if (queryTimeoutSeconds != 0 && runningSec >= queryTimeoutSeconds) {
+ e.thread.interrupt();
+ logger.error("Trying to cancel query:" + e.thread.getName());
+ }
+ }
+
// log the stack trace of bad query thread for further analysis
private void dumpStackTrace(Thread t) {
int maxStackTraceDepth = kylinConfig.getBadQueryStackTraceDepth();
http://git-wip-us.apache.org/repos/asf/kylin/blob/13f64f48/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 7b30606..56fab34 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -194,6 +194,7 @@ public class QueryService extends BasicService {
} finally {
String badReason = (ret != null && ret.isPushDown()) ? BadQueryEntry.ADJ_PUSHDOWN : null;
badQueryDetector.queryEnd(Thread.currentThread(), badReason);
+ Thread.interrupted(); //reset if interrupted
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/13f64f48/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
index fc18d92..d61dfbe 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
@@ -50,7 +50,7 @@ public class BadQueryDetectorTest extends LocalFileMetadataTestCase {
String mockSql = "select * from just_a_test";
final ArrayList<String[]> alerts = new ArrayList<>();
- BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec);
+ BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec, 1000);
badQueryDetector.registerNotifier(new BadQueryDetector.Notifier() {
@Override
public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user, Thread t) {
@@ -72,7 +72,7 @@ public class BadQueryDetectorTest extends LocalFileMetadataTestCase {
badQueryDetector.queryEnd(Thread.currentThread(), BadQueryEntry.ADJ_PUSHDOWN);
}
- badQueryDetector.stop();
+ badQueryDetector.interrupt();
assertEquals(2, alerts.size());
// second check founds a Slow
http://git-wip-us.apache.org/repos/asf/kylin/blob/13f64f48/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java b/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
new file mode 100644
index 0000000..25ff75b
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+import java.sql.SQLException;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exceptions.KylinTimeoutException;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
+import org.apache.kylin.query.security.QueryACLTestUtil;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.storage.IStorage;
+import org.apache.kylin.storage.IStorageQuery;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageFactory;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class KyilnQueryTimeoutTest extends LocalFileMetadataTestCase {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Before
+ public void setUp() {
+ this.createTestMetadata();
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setProperty("kylin.storage.provider.2", MockQueryTimeoutStorage.class.getName());
+ config.setProperty("kylin.storage.default", "2");
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ StorageFactory.clearCache();
+ }
+
+ @Test
+ public void testQueryTimeout() throws SQLException {
+ thrown.expectCause(CoreMatchers.isA(KylinTimeoutException.class));
+ thrown.expectMessage(CoreMatchers.containsString("Kylin query timeout"));
+ StorageFactory.clearCache();
+ BadQueryDetector detector = new BadQueryDetector(100, BadQueryDetector.getSystemAvailMB() * 2, 100, 1);
+ detector.start();
+ SQLRequest request = new SQLRequest();
+ request.setProject("default");
+ request.setSql("select count(*) from STREAMING_TABLE");
+ detector.queryStart(Thread.currentThread(), request, "ADMIN");
+ try {
+ QueryACLTestUtil.mockQuery("default", "select * from STREAMING_TABLE");
+ } finally{
+ detector.queryEnd(Thread.currentThread(), "timeout");
+ detector.interrupt();
+ }
+ }
+
+ public static class MockQueryTimeoutStorage implements IStorage {
+
+ @Override
+ public IStorageQuery createQuery(IRealization realization) {
+ return new MockQueryTimeoutQuery();
+ }
+
+ @Override
+ public <I> I adaptToBuildEngine(Class<I> engineInterface) {
+ return null;
+ }
+ }
+
+ private static class MockQueryTimeoutQuery implements IStorageQuery {
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ try {
+ Thread.sleep(5 * 1000);
+ } catch (InterruptedException e) {
+ throw new KylinTimeoutException("Kylin query timeout");
+ }
+ return null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/13f64f48/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 2e82140..634a3cd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -311,11 +311,8 @@ public abstract class CubeHBaseRPC implements IGTStorage {
coopTimeout = (long) (rpcTimeout * 0.9);
}
- long millisBeforeDeadline = queryContext.checkMillisBeforeDeadline();
- coopTimeout = Math.min(coopTimeout, millisBeforeDeadline);
-
- logger.debug("{} = {} ms, {} ms before deadline, use {} ms as timeout for coprocessor",
- HConstants.HBASE_RPC_TIMEOUT_KEY, rpcTimeout, millisBeforeDeadline, coopTimeout);
+ queryContext.checkMillisBeforeDeadline();
+ logger.debug("{} = {} ms, use {} ms as timeout for coprocessor", HConstants.HBASE_RPC_TIMEOUT_KEY, rpcTimeout, coopTimeout);
return coopTimeout;
}
[33/33] kylin git commit: Deep copy segment when duplicate cube
instance
Posted by li...@apache.org.
Deep copy segment when duplicate cube instance
(cherry picked from commit 4d5cf906fbe4cd840e2daf7333be8c52c7ccfb29)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/923a9bd9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/923a9bd9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/923a9bd9
Branch: refs/heads/sync
Commit: 923a9bd9f92a1eceee9fbee50b3ba34fc9254a81
Parents: 4ad6acb
Author: Yifan Zhang <ev...@gmail.com>
Authored: Wed Jan 24 18:45:34 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:16:42 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 9 +++++++-
.../org/apache/kylin/cube/CubeInstanceTest.java | 22 +++++++++++++++++---
2 files changed, 27 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/923a9bd9/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 6395abd..27918bc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -191,6 +191,9 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public CubeDesc getDescriptor() {
+ if (config == null) {
+ return null;
+ }
return CubeDescManager.getInstance(config).getCubeDesc(descName);
}
@@ -297,6 +300,10 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
@Override
public int getCost() {
+ if (getDescriptor() == null) {
+ //in case not initialized
+ return 0;
+ }
int countedDimensionNum = getRowKeyColumnCount();
int c = countedDimensionNum * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;
DataModelDesc model = getModel();
@@ -348,7 +355,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public void setSegments(Segments segments) {
- this.segments = segments;
+ this.segments = new Segments<>(segments);
}
public CubeSegment getSegmentById(String segmentId) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/923a9bd9/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
index 9e517b9..dd3b12b 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
@@ -18,17 +18,18 @@
package org.apache.kylin.cube;
-import static org.junit.Assert.assertNotNull;
-
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -56,6 +57,21 @@ public class CubeInstanceTest {
for (Long cuboid : cuboids) {
System.out.println(cuboid + ":" + cuboidsWithRowCnt.get(cuboid));
}
- assertNotNull(cuboidsWithRowCnt.get(255L));
+ Assert.assertNotNull(cuboidsWithRowCnt.get(255L));
+ }
+
+ @Test
+ public void copyCubeSegmentTest() {
+ int origSegCount = cubeInstance.getSegments().size();
+ CubeInstance newCubeInstance = CubeInstance.getCopyOf(cubeInstance);
+
+ CubeSegment mockSeg = new CubeSegment();
+ mockSeg.setUuid(UUID.randomUUID().toString());
+ mockSeg.setStorageLocationIdentifier(UUID.randomUUID().toString());
+ mockSeg.setStatus(SegmentStatusEnum.READY);
+ newCubeInstance.getSegments().add(mockSeg);
+
+ Assert.assertEquals(origSegCount, cubeInstance.getSegments().size());
+ Assert.assertEquals(origSegCount + 1, newCubeInstance.getSegments().size());
}
}
[16/33] kylin git commit: KYLIN-3158 Retry failed the metadata sync
event on the failed node only
Posted by li...@apache.org.
KYLIN-3158 Retry failed the metadata sync event on the failed node only
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/120f9f3d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/120f9f3d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/120f9f3d
Branch: refs/heads/sync
Commit: 120f9f3d97018d2b2ec215bb2ed3eca04550cff7
Parents: 6d9765e
Author: Li Yang <li...@apache.org>
Authored: Fri Jan 12 21:46:47 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 18:52:30 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +
.../kylin/metadata/cachesync/Broadcaster.java | 127 ++++++++++++++-----
.../metadata/cachesync/SingleValueCache.java | 6 +-
.../metadata/cachesync/BroadcasterTest.java | 43 +++++++
.../kylin/rest/init/InitialTaskManager.java | 2 +-
.../apache/kylin/rest/service/CacheService.java | 4 +-
6 files changed, 151 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/120f9f3d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index df01842..edb5b29 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -303,6 +303,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.metadata.sync-retries", "3"));
}
+ public String getCacheSyncErrorHandler() {
+ return getOptional("kylin.metadata.sync-error-handler");
+ }
+
// for test only
public void setMetadataUrl(String metadataUrl) {
setProperty("kylin.metadata.url", metadataUrl);
http://git-wip-us.apache.org/repos/asf/kylin/blob/120f9f3d/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 05910ea..6462a27 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.restclient.RestClient;
+import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.DaemonThreadFactory;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
@@ -68,7 +69,7 @@ public class Broadcaster {
public static Broadcaster getInstance(KylinConfig config) {
return config.getManager(Broadcaster.class);
}
-
+
// called by reflection
static Broadcaster newInstance(KylinConfig config) {
return new Broadcaster(config);
@@ -79,13 +80,19 @@ public class Broadcaster {
static final Map<String, List<Listener>> staticListenerMap = Maps.newConcurrentMap();
private KylinConfig config;
+ private ExecutorService announceMainLoop;
+ private ExecutorService announceThreadPool;
+ private SyncErrorHandler syncErrorHandler;
private BlockingDeque<BroadcastEvent> broadcastEvents = new LinkedBlockingDeque<>();
private Map<String, List<Listener>> listenerMap = Maps.newConcurrentMap();
- private AtomicLong counter = new AtomicLong();
-
+ private AtomicLong counter = new AtomicLong(); // a counter for testing purpose
+
private Broadcaster(final KylinConfig config) {
this.config = config;
- final int retryLimitTimes = config.getCacheSyncRetrys();
+ this.syncErrorHandler = getSyncErrorHandler(config);
+ this.announceMainLoop = Executors.newSingleThreadExecutor(new DaemonThreadFactory());
+ this.announceThreadPool = new ThreadPoolExecutor(1, 10, 60L, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory());
final String[] nodes = config.getRestServers();
if (nodes == null || nodes.length < 1) {
@@ -93,21 +100,14 @@ public class Broadcaster {
}
logger.debug(nodes.length + " nodes in the cluster: " + Arrays.toString(nodes));
- Executors.newSingleThreadExecutor(new DaemonThreadFactory()).execute(new Runnable() {
+ announceMainLoop.execute(new Runnable() {
@Override
public void run() {
final Map<String, RestClient> restClientMap = Maps.newHashMap();
- final ExecutorService wipingCachePool = new ThreadPoolExecutor(1, 10, 60L, TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory());
- while (true) {
+ while (!announceThreadPool.isShutdown()) {
try {
final BroadcastEvent broadcastEvent = broadcastEvents.takeFirst();
- broadcastEvent.setRetryTime(broadcastEvent.getRetryTime() + 1);
- if (broadcastEvent.getRetryTime() > retryLimitTimes) {
- logger.info("broadcastEvent retry up to limit times, broadcastEvent:{}", broadcastEvent);
- continue;
- }
String[] restServers = config.getRestServers();
logger.debug("Servers in the cluster: " + Arrays.toString(restServers));
@@ -117,25 +117,27 @@ public class Broadcaster {
}
}
- logger.debug("Announcing new broadcast event: " + broadcastEvent);
+ String toWhere = broadcastEvent.getTargetNode();
+ if (toWhere == null)
+ toWhere = "all";
+ logger.debug("Announcing new broadcast to " + toWhere + ": " + broadcastEvent);
+
for (final String node : restServers) {
- wipingCachePool.execute(new Runnable() {
+ if (!(toWhere.equals("all") || toWhere.equals(node)))
+ continue;
+
+ announceThreadPool.execute(new Runnable() {
@Override
public void run() {
+ RestClient restClient = restClientMap.get(node);
try {
- restClientMap.get(node).wipeCache(broadcastEvent.getEntity(),
- broadcastEvent.getEvent(), broadcastEvent.getCacheKey());
+ restClient.wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(),
+ broadcastEvent.getCacheKey());
} catch (IOException e) {
- logger.warn("Thread failed during wipe cache at {}, error msg: {}",
- broadcastEvent, e);
- // when sync failed, put back to queue
- try {
- broadcastEvents.putLast(broadcastEvent);
- } catch (InterruptedException ex) {
- logger.warn(
- "error reentry failed broadcastEvent to queue, broacastEvent:{}, error: {} ",
- broadcastEvent, ex);
- }
+ logger.error(
+ "Announce broadcast event failed, targetNode {} broadcastEvent {}, error msg: {}",
+ node, broadcastEvent, e);
+ syncErrorHandler.handleAnnounceError(node, restClient, broadcastEvent);
}
}
});
@@ -148,6 +150,23 @@ public class Broadcaster {
});
}
+ private SyncErrorHandler getSyncErrorHandler(KylinConfig config) {
+ String clzName = config.getCacheSyncErrorHandler();
+ if (StringUtils.isEmpty(clzName)) {
+ clzName = DefaultSyncErrorHandler.class.getName();
+ }
+ return (SyncErrorHandler) ClassUtil.newInstance(clzName);
+ }
+
+ public KylinConfig getConfig() {
+ return config;
+ }
+
+ public void stopAnnounce() {
+ announceThreadPool.shutdown();
+ announceMainLoop.shutdown();
+ }
+
// static listener survives cache wipe and goes after normal listeners
public void registerStaticListener(Listener listener, String... entities) {
doRegisterListener(staticListenerMap, listener, entities);
@@ -263,15 +282,19 @@ public class Broadcaster {
}
/**
- * Broadcast an event out
+ * Announce an event out to peer kylin servers
*/
- public void queue(String entity, String event, String key) {
+ public void announce(String entity, String event, String key) {
+ announce(new BroadcastEvent(entity, event, key));
+ }
+
+ public void announce(BroadcastEvent event) {
if (broadcastEvents == null)
return;
try {
counter.incrementAndGet();
- broadcastEvents.putLast(new BroadcastEvent(entity, event, key));
+ broadcastEvents.putLast(event);
} catch (Exception e) {
counter.decrementAndGet();
logger.error("error putting BroadcastEvent", e);
@@ -282,6 +305,40 @@ public class Broadcaster {
return counter.getAndSet(0);
}
+ // ============================================================================
+
+ public static class DefaultSyncErrorHandler implements SyncErrorHandler {
+ Broadcaster broadcaster;
+ int maxRetryTimes;
+
+ @Override
+ public void init(Broadcaster broadcaster) {
+ this.maxRetryTimes = broadcaster.getConfig().getCacheSyncRetrys();
+ this.broadcaster = broadcaster;
+ }
+
+ @Override
+ public void handleAnnounceError(String targetNode, RestClient restClient, BroadcastEvent event) {
+ int retry = event.getRetryTime() + 1;
+
+ // when sync failed, put back to queue to retry
+ if (retry < maxRetryTimes) {
+ event.setRetryTime(retry);
+ event.setTargetNode(targetNode);
+ broadcaster.announce(event);
+ } else {
+ logger.error("Announce broadcast event exceeds retry limit, abandon targetNode {} broadcastEvent {}",
+ targetNode, event);
+ }
+ }
+ }
+
+ public interface SyncErrorHandler {
+ void init(Broadcaster broadcaster);
+
+ void handleAnnounceError(String targetNode, RestClient restClient, BroadcastEvent event);
+ }
+
public enum Event {
CREATE("create"), UPDATE("update"), DROP("drop");
@@ -326,6 +383,8 @@ public class Broadcaster {
public static class BroadcastEvent {
private int retryTime;
+ private String targetNode; // NULL means to all
+
private String entity;
private String event;
private String cacheKey;
@@ -345,6 +404,14 @@ public class Broadcaster {
this.retryTime = retryTime;
}
+ public String getTargetNode() {
+ return targetNode;
+ }
+
+ public void setTargetNode(String targetNode) {
+ this.targetNode = targetNode;
+ }
+
public String getEntity() {
return entity;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/120f9f3d/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/SingleValueCache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/SingleValueCache.java
index 4bfaeae..f803c8b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/SingleValueCache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/SingleValueCache.java
@@ -49,9 +49,9 @@ public abstract class SingleValueCache<K, V> extends AbstractCache<K, V> {
innerCache.put(key, value);
if (!exists) {
- getBroadcaster().queue(syncEntity, Broadcaster.Event.CREATE.getType(), key.toString());
+ getBroadcaster().announce(syncEntity, Broadcaster.Event.CREATE.getType(), key.toString());
} else {
- getBroadcaster().queue(syncEntity, Broadcaster.Event.UPDATE.getType(), key.toString());
+ getBroadcaster().announce(syncEntity, Broadcaster.Event.UPDATE.getType(), key.toString());
}
}
@@ -65,7 +65,7 @@ public abstract class SingleValueCache<K, V> extends AbstractCache<K, V> {
innerCache.remove(key);
if (exists) {
- getBroadcaster().queue(syncEntity, Broadcaster.Event.DROP.getType(), key.toString());
+ getBroadcaster().announce(syncEntity, Broadcaster.Event.DROP.getType(), key.toString());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/120f9f3d/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/BroadcasterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/BroadcasterTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/BroadcasterTest.java
index 80f26f9..762512f 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/BroadcasterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/BroadcasterTest.java
@@ -21,9 +21,12 @@ package org.apache.kylin.metadata.cachesync;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.cachesync.Broadcaster.BroadcastEvent;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.cachesync.Broadcaster.Listener;
+import org.apache.kylin.metadata.cachesync.Broadcaster.SyncErrorHandler;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -64,6 +67,7 @@ public class BroadcasterTest extends LocalFileMetadataTestCase {
broadcaster.notifyListener("test", Event.UPDATE, "");
+ broadcaster.stopAnnounce();
Broadcaster.staticListenerMap.clear();
}
@@ -90,6 +94,45 @@ public class BroadcasterTest extends LocalFileMetadataTestCase {
broadcaster.notifyNonStaticListener("test", Event.UPDATE, "");
+ broadcaster.stopAnnounce();
Broadcaster.staticListenerMap.clear();
}
+
+ @Test
+ public void testAnnounceErrorHandler() throws IOException, InterruptedException {
+ System.setProperty("kylin.server.cluster-servers", "localhost:717");
+ System.setProperty("kylin.metadata.sync-error-handler", MockupErrHandler.class.getName());
+ try {
+ Broadcaster broadcaster = Broadcaster.getInstance(getTestConfig());
+
+ broadcaster.announce("all", "update", "all");
+
+ for (int i = 0; i < 30 && MockupErrHandler.atom.get() == 0; i++) {
+ Thread.sleep(1000);
+ }
+
+ broadcaster.stopAnnounce();
+ Broadcaster.staticListenerMap.clear();
+ } finally {
+ System.clearProperty("kylin.server.cluster-servers");
+ System.clearProperty("kylin.metadata.sync-error-handler");
+ }
+
+ Assert.assertTrue(MockupErrHandler.atom.get() > 0);
+ }
+
+ public static class MockupErrHandler implements SyncErrorHandler {
+ static AtomicInteger atom = new AtomicInteger();
+
+ @Override
+ public void init(Broadcaster broadcaster) {
+ }
+
+ @Override
+ public void handleAnnounceError(String targetNode, RestClient restClient, BroadcastEvent event) {
+ Assert.assertEquals("localhost:717", targetNode);
+ atom.incrementAndGet();
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/120f9f3d/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
index 14052ce..467ef82 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
@@ -53,7 +53,7 @@ public class InitialTaskManager implements InitializingBean {
for (String taskClass : taskClasses) {
try {
InitialTask task = (InitialTask) Class.forName(taskClass).newInstance();
- logger.info("Running task: " + taskClass);
+ logger.info("Running initial task: " + taskClass);
task.execute();
} catch (Throwable e) {
logger.error("Initial task failed: " + taskClass, e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/120f9f3d/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 98e06e0..b61309e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
@@ -49,6 +50,7 @@ public class CacheService extends BasicService implements InitializingBean {
@Override
public void onClearAll(Broadcaster broadcaster) throws IOException {
cleanAllDataCache();
+ HBaseConnection.clearConnCache(); // take the chance to clear HBase connection cache as well
}
@Override
@@ -104,7 +106,7 @@ public class CacheService extends BasicService implements InitializingBean {
public void annouceWipeCache(String entity, String event, String cacheKey) {
Broadcaster broadcaster = Broadcaster.getInstance(getConfig());
- broadcaster.queue(entity, event, cacheKey);
+ broadcaster.announce(entity, event, cacheKey);
}
public void notifyMetadataChange(String entity, Event event, String cacheKey) throws IOException {
[27/33] kylin git commit: KYLIN-3173,
reset DefaultScheduler's initialized/hasStarted when shutdown.
Posted by li...@apache.org.
KYLIN-3173, reset DefaultScheduler's initialized/hasStarted when shutdown.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/490d8675
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/490d8675
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/490d8675
Branch: refs/heads/sync
Commit: 490d867577f867048bd5ba361e66d8f9589bbb44
Parents: a6e6013
Author: tttMelody <24...@qq.com>
Authored: Wed Jan 17 14:48:18 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:04:22 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/impl/threadpool/DefaultScheduler.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/490d8675/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 6af0192..0b01284 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -366,6 +366,8 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
public void shutdown() throws SchedulerException {
logger.info("Shutting down DefaultScheduler ....");
jobLock.unlockJobEngine();
+ initialized = false;
+ hasStarted = false;
try {
fetcherPool.shutdownNow();//interrupt
fetcherPool.awaitTermination(1, TimeUnit.MINUTES);
[09/33] kylin git commit: Minor, fix model draft not listed
Posted by li...@apache.org.
Minor, fix model draft not listed
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d6cc4e0a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d6cc4e0a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d6cc4e0a
Branch: refs/heads/sync
Commit: d6cc4e0a98b85b9792624d8f5449dc3ae55e1bff
Parents: 1bf9886
Author: nichunen <ch...@kyligence.io>
Authored: Fri Jan 5 17:05:41 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 17:33:50 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/ModelService.java | 2 +-
.../kylin/rest/service/ModelServiceTest.java | 18 ++++++++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6cc4e0a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 43ce0b1..ee01f3e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -413,7 +413,7 @@ public class ModelService extends BasicService {
RootPersistentEntity e = d.getEntity();
if (e instanceof DataModelDesc) {
DataModelDesc m = (DataModelDesc) e;
- if (modelName == null || modelName.equals(m.getName()))
+ if (StringUtils.isEmpty(modelName) || modelName.equals(m.getName()))
result.add(d);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6cc4e0a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index fff4ed9..aeb6d79 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -27,6 +27,8 @@ import java.util.List;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.job.exception.JobException;
+import org.apache.kylin.metadata.draft.Draft;
+import org.apache.kylin.metadata.draft.DraftManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.ModelDimensionDesc;
import org.junit.Assert;
@@ -132,6 +134,22 @@ public class ModelServiceTest extends ServiceTestBase {
}
}
+ @Test
+ public void testModelDraft() throws IOException {
+ DraftManager mgr = DraftManager.getInstance(getTestConfig());
+ // Create a draft of model
+ Draft d = new Draft();
+ d.setProject("default");
+ d.updateRandomUuid();
+ DataModelDesc modelDesc = modelService.getModel("ci_left_join_model", "default");
+ d.setEntity(modelDesc);
+ mgr.save(d);
+
+ // Check list draft
+ List<Draft> draftList = modelService.listModelDrafts("", "default");
+ Assert.assertEquals(draftList.size(), 1);
+ }
+
private String[] cutItems(String[] origin, int count) {
if (origin == null)
[04/33] kylin git commit: KYLIN-3150 PercentileCounter maxlength is
too small
Posted by li...@apache.org.
KYLIN-3150 PercentileCounter maxlength is too small
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/91ed19bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/91ed19bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/91ed19bc
Branch: refs/heads/sync
Commit: 91ed19bc4bf03979d2765d588a5232c76f953d00
Parents: ca8428d
Author: yiming.xu <10...@qq.com>
Authored: Tue Jan 2 21:02:47 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 17:30:06 2018 +0800
----------------------------------------------------------------------
.../kylin/measure/percentile/PercentileCounter.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/91ed19bc/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
index 2b5903c..33433dc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
@@ -106,8 +106,16 @@ public class PercentileCounter implements Serializable {
}
public int maxLength() {
- // 10KB for max length
- return 10 * 1024;
+ switch ((int) compression) {
+ case 100:
+ return 16 * 1024;
+ case 1000:
+ return 128 * 1024;
+ case 10000:
+ return 1024 * 1024;
+ default:
+ return 16 * 1024;
+ }
}
public int peekLength(ByteBuffer in) {
[03/33] kylin git commit: Minor,
fix NotClassFound issue in org.apache.hadoop.hive.ql.Driver
Posted by li...@apache.org.
Minor, fix NotClassFound issue in org.apache.hadoop.hive.ql.Driver
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca8428d3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca8428d3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca8428d3
Branch: refs/heads/sync
Commit: ca8428d388cb3b0716efc92fb5cec729d636204f
Parents: b4daa8a
Author: Yifan Zhang <ev...@gmail.com>
Authored: Mon Jan 1 16:33:26 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 17:24:08 2018 +0800
----------------------------------------------------------------------
.../kylin/source/hive/BeelineHiveClient.java | 5 +--
.../apache/kylin/source/hive/CLIHiveClient.java | 39 +++++++-------------
.../kylin/source/hive/HiveMetadataExplorer.java | 29 ++++++++++-----
.../apache/kylin/source/hive/IHiveClient.java | 6 +--
.../apache/kylin/source/jdbc/JdbcExplorer.java | 9 ++++-
5 files changed, 43 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index ee693c5..314402c 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -28,7 +28,6 @@ import java.sql.Statement;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.kylin.common.util.DBUtils;
import com.google.common.base.Preconditions;
@@ -117,12 +116,12 @@ public class BeelineHiveClient implements IHiveClient {
}
@Override
- public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
+ public void executeHQL(String hql) throws IOException {
throw new UnsupportedOperationException();
}
@Override
- public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
+ public void executeHQL(String[] hqls) throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index e8a93bd..bc9f17e 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -22,17 +22,15 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HiveCmdBuilder;
+import org.apache.kylin.common.util.Pair;
import com.google.common.collect.Lists;
@@ -43,7 +41,6 @@ import com.google.common.collect.Lists;
*/
public class CLIHiveClient implements IHiveClient {
protected HiveConf hiveConf = null;
- protected Driver driver = null;
protected HiveMetaStoreClient metaStoreClient = null;
public CLIHiveClient() {
@@ -52,22 +49,25 @@ public class CLIHiveClient implements IHiveClient {
/**
* only used by Deploy Util
+ * @throws IOException
*/
@Override
- public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
- CommandProcessorResponse response = getDriver().run(hql);
- int retCode = response.getResponseCode();
- if (retCode != 0) {
- String err = response.getErrorMessage();
- throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err);
+ public void executeHQL(String hql) throws IOException {
+ final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.addStatement(hql);
+ Pair<Integer, String> response = KylinConfig.getInstanceFromEnv().getCliCommandExecutor()
+ .execute(hiveCmdBuilder.toString());
+ if (response.getFirst() != 0) {
+ throw new IllegalArgumentException("Failed to execute hql [" + hql + "], error message is: " + response.getSecond());
}
+
}
/**
* only used by Deploy Util
*/
@Override
- public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
+ public void executeHQL(String[] hqls) throws IOException {
for (String sql : hqls)
executeHQL(sql);
}
@@ -159,17 +159,4 @@ public class CLIHiveClient implements IHiveClient {
}
return result;
}
-
- /**
- * Get the hive ql driver to execute ddl or dml
- * @return
- */
- private Driver getDriver() {
- if (driver == null) {
- driver = new Driver(hiveConf);
- SessionState.start(new CliSessionState(hiveConf));
- }
-
- return driver;
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index d80c546..cb3eb02 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -32,11 +32,15 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.source.ISampleDataDeployer;
import org.apache.kylin.source.ISourceMetadataExplorer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDataDeployer {
+ private static final Logger logger = LoggerFactory.getLogger(HiveClientFactory.class);
+
IHiveClient hiveClient = HiveClientFactory.getHiveClient();
-
+
@Override
public List<String> listDatabases() throws Exception {
return hiveClient.getHiveDbNames();
@@ -90,7 +94,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
tableExtDesc.setUuid(UUID.randomUUID().toString());
tableExtDesc.setLastModified(0);
tableExtDesc.init(prj);
-
+
tableExtDesc.addDataSourceProp("location", hiveTableMeta.sdLocation);
tableExtDesc.addDataSourceProp("owner", hiveTableMeta.owner);
tableExtDesc.addDataSourceProp("last_access_time", String.valueOf(hiveTableMeta.lastAccessTime));
@@ -114,10 +118,10 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
hiveClient.executeHQL(generateCreateSchemaSql(database));
}
- private String generateCreateSchemaSql(String schemaName){
+ private String generateCreateSchemaSql(String schemaName) {
return String.format("CREATE DATABASE IF NOT EXISTS %s", schemaName);
}
-
+
@Override
public void createSampleTable(TableDesc table) throws Exception {
hiveClient.executeHQL(generateCreateTableSql(table));
@@ -146,7 +150,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
return new String[] { dropsql, dropsql2, ddl.toString() };
}
-
+
@Override
public void loadSampleData(String tableName, String tmpDataDir) throws Exception {
hiveClient.executeHQL(generateLoadDataSql(tableName, tmpDataDir));
@@ -155,12 +159,12 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
private String generateLoadDataSql(String tableName, String tableFileDir) {
return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName + ".csv' OVERWRITE INTO TABLE " + tableName;
}
-
+
@Override
public void createWrapperView(String origTableName, String viewName) throws Exception {
hiveClient.executeHQL(generateCreateViewSql(viewName, origTableName));
}
-
+
private String[] generateCreateViewSql(String viewName, String tableName) {
String dropView = "DROP VIEW IF EXISTS " + viewName;
@@ -170,7 +174,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
return new String[] { dropView, dropTable, createSql };
}
-
+
private static String getHiveDataType(String javaDataType) {
String hiveDataType = javaDataType.toLowerCase().startsWith("varchar") ? "string" : javaDataType;
hiveDataType = javaDataType.toLowerCase().startsWith("integer") ? "int" : hiveDataType;
@@ -192,16 +196,21 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
String evalViewSql = "CREATE VIEW " + tmpDatabase + "." + tmpView + " as " + query;
try {
- hiveClient.executeHQL(new String[] { dropViewSql, evalViewSql });
+ logger.debug("Removing duplicate view {}", tmpView);
+ hiveClient.executeHQL(dropViewSql);
+ logger.debug("Creating view {} for query: {}", tmpView, query);
+ hiveClient.executeHQL(evalViewSql);
+ logger.debug("Evaluating query columns' metadata");
HiveTableMeta hiveTableMeta = hiveClient.getHiveTableMeta(tmpDatabase, tmpView);
return extractColumnFromMeta(hiveTableMeta);
} catch (Exception e) {
throw new RuntimeException("Cannot evalutate metadata of query: " + query, e);
} finally {
try {
+ logger.debug("Cleaning up.");
hiveClient.executeHQL(dropViewSql);
} catch (Exception e) {
- throw new RuntimeException("Cannot temp view of query: " + query, e);
+ logger.warn("Cannot drop temp view of query: {}", query, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
index ca5312d..ec2bf7f 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
@@ -21,13 +21,11 @@ package org.apache.kylin.source.hive;
import java.io.IOException;
import java.util.List;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-
public interface IHiveClient {
- void executeHQL(String hql) throws CommandNeedRetryException, IOException;
+ void executeHQL(String hql) throws IOException;
- void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException;
+ void executeHQL(String[] hqls) throws IOException;
HiveTableMeta getHiveTableMeta(String database, String tableName) throws Exception;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index 81e42bb..d96a68d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -243,7 +243,11 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
String evalViewSql = "CREATE VIEW " + tmpView + " as " + query;
try {
- executeSQL(new String[] { dropViewSql, evalViewSql });
+ logger.debug("Removing duplicate view {}", tmpView);
+ executeSQL(dropViewSql);
+ logger.debug("Creating view {} for query: {}", tmpView, query);
+ executeSQL(evalViewSql);
+ logger.debug("Evaluating query columns' metadata");
Connection con = SqlUtil.getConnection(dbconf);
DatabaseMetaData dbmd = con.getMetaData();
ResultSet rs = dbmd.getColumns(null, tmpDatabase, tmpView, null);
@@ -255,9 +259,10 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
throw new RuntimeException("Cannot evalutate metadata of query: " + query, e);
} finally {
try {
+ logger.debug("Cleaning up.");
executeSQL(dropViewSql);
} catch (Exception e) {
- throw new RuntimeException("Cannot temp view of query: " + query, e);
+ logger.warn("Cannot drop temp view of query: {}", query, e);
}
}
}
[29/33] kylin git commit: KYLIN-3175,
update streaming segment's tsrange after merge
Posted by li...@apache.org.
KYLIN-3175, update streaming segment's tsrange after merge
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e071c1cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e071c1cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e071c1cc
Branch: refs/heads/sync
Commit: e071c1ccd74b2ca5aaf6449d71e022d530b66f26
Parents: 93a42b3
Author: Cheng Wang <ch...@kyligence.io>
Authored: Wed Jan 17 15:55:16 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:07:39 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 33 ++++++++++++--------
.../kylin/provision/BuildCubeWithStream.java | 2 +-
2 files changed, 21 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e071c1cc/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index df1d95e..8bdb5aa 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -276,17 +276,17 @@ public class CubeManager implements IRealizationProvider {
return updateCube(update);
}
}
-
+
public CubeInstance updateCubeSegStatus(CubeSegment seg, SegmentStatusEnum status) throws IOException {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
CubeInstance cube = seg.getCubeInstance().latestCopyForWrite();
seg = cube.getSegmentById(seg.getUuid());
-
+
CubeUpdate update = new CubeUpdate(cube);
seg.setStatus(status);
update.setToUpdateSegs(seg);
return updateCube(update);
- }
+ }
}
private CubeInstance updateCubeWithRetry(CubeUpdate update, int retry) throws IOException {
@@ -523,7 +523,7 @@ public class CubeManager implements IRealizationProvider {
return segAssist.optimizeSegments(cube, cuboidsRecommend);
}
}
-
+
public CubeSegment mergeSegments(CubeInstance cube, TSRange tsRange, SegmentRange segRange, boolean force)
throws IOException {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
@@ -547,9 +547,9 @@ public class CubeManager implements IRealizationProvider {
CubeSegment... optimizedSegments) throws IOException {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
segAssist.promoteCheckpointOptimizeSegments(cube, recommendCuboids, optimizedSegments);
- }
+ }
}
-
+
public List<CubeSegment> calculateHoles(String cubeName) {
return segAssist.calculateHoles(cubeName);
}
@@ -683,8 +683,13 @@ public class CubeManager implements IRealizationProvider {
tsRange = null;
Preconditions.checkArgument(segRange != null);
} else {
- if(tsRange == null) {
- tsRange = new TSRange((Long)segRange.start.v, (Long)segRange.end.v);
+ /**In case of non-streaming segment,
+ * tsRange is the same as segRange,
+ * either could fulfill the merge job,
+ * so it needs to convert segRange to tsRange if tsRange is null.
+ **/
+ if (tsRange == null) {
+ tsRange = new TSRange((Long) segRange.start.v, (Long) segRange.end.v);
}
segRange = null;
}
@@ -832,10 +837,11 @@ public class CubeManager implements IRealizationProvider {
updateCube(update);
}
- public void promoteNewlyOptimizeSegments(CubeInstance cube, CubeSegment... optimizedSegments) throws IOException {
+ public void promoteNewlyOptimizeSegments(CubeInstance cube, CubeSegment... optimizedSegments)
+ throws IOException {
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeSegment[] segCopy = cube.regetSegments(optimizedSegments);
-
+
for (CubeSegment seg : segCopy) {
seg.setStatus(SegmentStatusEnum.READY_PENDING);
}
@@ -849,12 +855,12 @@ public class CubeManager implements IRealizationProvider {
CubeSegment... optimizedSegments) throws IOException {
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeSegment[] optSegCopy = cubeCopy.regetSegments(optimizedSegments);
-
+
if (cubeCopy.getSegments().size() != optSegCopy.length * 2) {
throw new IllegalStateException("For cube " + cubeCopy
+ ", every READY segment should be optimized and all segments should be READY before optimizing");
}
-
+
CubeSegment[] originalSegments = new CubeSegment[optSegCopy.length];
int i = 0;
for (CubeSegment seg : optSegCopy) {
@@ -865,7 +871,8 @@ public class CubeManager implements IRealizationProvider {
"For cube " + cubeCopy + ", segment " + seg + " missing StorageLocationIdentifier");
if (StringUtils.isBlank(seg.getLastBuildJobID()))
- throw new IllegalStateException("For cube " + cubeCopy + ", segment " + seg + " missing LastBuildJobID");
+ throw new IllegalStateException(
+ "For cube " + cubeCopy + ", segment " + seg + " missing LastBuildJobID");
seg.setStatus(SegmentStatusEnum.READY);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e071c1cc/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index f7b8275..181e8b9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -370,7 +370,7 @@ public class BuildCubeWithStream {
} catch (Throwable e) {
logger.error("error", e);
exitCode = 1;
- } finally{
+ } finally {
if (buildCubeWithStream != null) {
buildCubeWithStream.after();
buildCubeWithStream.cleanup();