You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/12/30 02:30:53 UTC
[01/42] kylin git commit: KYLIN-2314 more CI fixes [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/sparkcubing-rebase 61bf8a12c -> cf6bba8b0 (forced update)
KYLIN-2314 more CI fixes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed8b1683
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed8b1683
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed8b1683
Branch: refs/heads/sparkcubing-rebase
Commit: ed8b1683a35488ffb8dd222c7bbe0fa84600aa5d
Parents: 73fbbb9
Author: Yang Li <li...@apache.org>
Authored: Sat Dec 24 21:09:21 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Dec 26 07:25:54 2016 +0800
----------------------------------------------------------------------
examples/test_case_data/localmeta/project/default.json | 9 ++++++++-
.../main/java/org/apache/kylin/query/schema/OLAPTable.java | 2 +-
.../kylin/source/kafka/config/KafkaConsumerProperties.java | 8 +++-----
3 files changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ed8b1683/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 1ecc7cc..75d7ec9 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -31,11 +31,18 @@
"tables": [
"DEFAULT.TEST_KYLIN_FACT",
"DEFAULT.TEST_ORDER",
+ "DEFAULT.TEST_ACCOUNT",
+ "DEFAULT.TEST_COUNTRY",
"EDW.TEST_CAL_DT",
"DEFAULT.TEST_CATEGORY_GROUPINGS",
"EDW.TEST_SITES",
"EDW.TEST_SELLER_TYPE_DIM",
- "DEFAULT.STREAMING_TABLE"
+ "DEFAULT.STREAMING_TABLE",
+ "SSB.CUSTOMER",
+ "SSB.DATES",
+ "SSB.PART",
+ "SSB.SUPPLIER",
+ "SSB.V_LINEORDER"
],
"models": [
"ssb",
http://git-wip-us.apache.org/repos/asf/kylin/blob/ed8b1683/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index c33c4ed..85996f4 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -203,7 +203,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
DataModelDesc model = realization.getModel();
for (String metricColumn : model.getMetrics()) {
TblColRef col = model.findColumn(metricColumn);
- if (col.getType().isIntegerFamily() && !col.getType().isBigInt())
+ if (col.getTable().equals(sourceTable.getIdentity()) && col.getType().isIntegerFamily() && !col.getType().isBigInt())
updateColumns.add(col.getColumnDesc());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ed8b1683/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
index 8f0dd42..5b3dd87 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
@@ -101,8 +101,8 @@ public class KafkaConsumerProperties {
private Properties loadKafkaConsumerProperties() {
File propFile = getKafkaConsumerFile();
if (propFile == null || !propFile.exists()) {
- logger.error("fail to locate " + KAFKA_CONSUMER_FILE);
- throw new RuntimeException("fail to locate " + KAFKA_CONSUMER_FILE);
+ logger.warn("fail to locate " + KAFKA_CONSUMER_FILE + ", use empty kafka consumer properties");
+ return new Properties();
}
Properties properties = new Properties();
try {
@@ -115,7 +115,6 @@ public class KafkaConsumerProperties {
File propOverrideFile = new File(propFile.getParentFile(), propFile.getName() + ".override");
if (propOverrideFile.exists()) {
FileInputStream ois = new FileInputStream(propOverrideFile);
- Properties propOverride = new Properties();
Configuration oconf = new Configuration();
oconf.addResource(ois);
properties.putAll(extractKafkaConfigToProperties(oconf));
@@ -134,7 +133,6 @@ public class KafkaConsumerProperties {
}
private File getKafkaConsumerFile() {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
String kylinConfHome = System.getProperty(KylinConfig.KYLIN_CONF);
if (!StringUtils.isEmpty(kylinConfHome)) {
logger.info("Use KYLIN_CONF=" + kylinConfHome);
@@ -143,7 +141,7 @@ public class KafkaConsumerProperties {
logger.warn("KYLIN_CONF property was not set, will seek KYLIN_HOME env variable");
- String kylinHome = kylinConfig.getKylinHome();
+ String kylinHome = KylinConfig.getKylinHome();
if (StringUtils.isEmpty(kylinHome))
throw new KylinConfigCannotInitException("Didn't find KYLIN_CONF or KYLIN_HOME, please set one of them");
[30/42] kylin git commit: KYLIN-2333 Kylin doesn't need 0-D cuboid,
can remove that step
Posted by sh...@apache.org.
KYLIN-2333 Kylin doesn't need 0-D cuboid, can remove that step
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/687d5fd1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/687d5fd1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/687d5fd1
Branch: refs/heads/sparkcubing-rebase
Commit: 687d5fd150b6019e71db0f600307bbd4f7c749f9
Parents: 1a190ec
Author: shaofengshi <sh...@apache.org>
Authored: Thu Dec 29 13:23:14 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Dec 29 13:23:14 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/687d5fd1/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 6eba3c2..966bb1b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -84,7 +84,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
// base cuboid step
result.addTask(createBaseCuboidStep(cuboidOutputTempPath, jobId));
// n dim cuboid steps
- for (int i = 1; i <= groupRowkeyColumnsCount; i++) {
+ for (int i = 1; i < groupRowkeyColumnsCount; i++) {
int dimNum = totalRowkeyColumnsCount - i;
result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount, jobId));
}
[37/42] kylin git commit: minor,
fix wrong prop name in kylin.properties
Posted by sh...@apache.org.
minor, fix wrong prop name in kylin.properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75c8d13c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75c8d13c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75c8d13c
Branch: refs/heads/sparkcubing-rebase
Commit: 75c8d13c708e66c995cdea494f8656b72636a4d6
Parents: 020cf5f
Author: lidongsjtu <li...@apache.org>
Authored: Thu Dec 29 22:53:45 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Dec 29 22:57:54 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 2 +-
examples/test_case_data/sandbox/kylin.properties | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/75c8d13c/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 131a725..5e7d235 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -30,7 +30,7 @@ kylin.env=QA
### SERVER | WEB ###
# Kylin server mode, valid value [all, query, job]
-kyin.server.mode=all
+kylin.server.mode=all
# List of web servers in use, this enables one web server instance to sync up with other servers.
kylin.server.cluster-servers=localhost:7070
http://git-wip-us.apache.org/repos/asf/kylin/blob/75c8d13c/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 20bc427..1724619 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -18,7 +18,7 @@
### SERVICE ###
# Kylin server mode, valid value [all, query, job]
-kyin.server.mode=all
+kylin.server.mode=all
# Optional information for the owner of kylin platform, it can be your team's email
# Currently it will be attached to each kylin's htable attribute
[20/42] kylin git commit: minor,
invoke toString of translated filter may be expensive
Posted by sh...@apache.org.
minor, invoke toString of translated filter may be expensive
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7aabe4cb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7aabe4cb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7aabe4cb
Branch: refs/heads/sparkcubing-rebase
Commit: 7aabe4cb1d1480f99b4724b20598ba216f44586a
Parents: 9898a07
Author: gaodayue <ga...@meituan.com>
Authored: Wed Dec 28 12:20:51 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Wed Dec 28 12:20:51 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/dict/BuiltInFunctionTransformer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7aabe4cb/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index f451c2c..6ef2f96 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -57,13 +57,13 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
//normal case
translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
if (translated != null) {
- logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+ logger.debug("Translated {{}} to IN clause: {{}}", tupleFilter, translated);
}
} else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
//like case
translated = translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
if (translated != null) {
- logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+ logger.debug("Translated {{}} to IN clause: {{}}", tupleFilter, translated);
}
} else if (tupleFilter instanceof LogicalTupleFilter) {
@SuppressWarnings("unchecked")
[09/42] kylin git commit: KYLIN-2318 query cache is not working
Posted by sh...@apache.org.
KYLIN-2318 query cache is not working
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/182c565d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/182c565d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/182c565d
Branch: refs/heads/sparkcubing-rebase
Commit: 182c565df6b2134ef1302c88988ed27b60fefa42
Parents: fa4a5ee
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:15:44 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/QueryContext.java | 61 ++++++++++++++++++++
.../kylin/common/debug/BackdoorToggles.java | 11 ++--
.../apache/kylin/rest/request/SQLRequest.java | 23 +++++---
.../apache/kylin/rest/service/QueryService.java | 23 ++++----
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +-
5 files changed, 93 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 0000000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for comparision
+ */
+public class QueryContext {
+ private static final ThreadLocal<Map<String, String>> _queryContext = new ThreadLocal<Map<String, String>>();
+
+ public final static String KEY_QUERY_ID = "QUERY_ID";
+
+ public static String getQueryId() {
+ return getString(KEY_QUERY_ID);
+ }
+
+ public static void setQueryId(String uuid) {
+ setString(KEY_QUERY_ID, uuid);
+ }
+
+ private static void setString(String key, String value) {
+ Map<String, String> context = _queryContext.get();
+ if (context == null) {
+ Map<String, String> newMap = Maps.newHashMap();
+ newMap.put(key, value);
+ _queryContext.set(newMap);
+ } else {
+ context.put(key, value);
+ }
+ }
+
+ private static String getString(String key) {
+ Map<String, String> context = _queryContext.get();
+ if (context == null) {
+ return null;
+ } else {
+ return context.get(key);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
/**
+ * BackdoorToggles and QueryContext are similar because they're both hosting per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ *
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to SQLRequest
*/
public class BackdoorToggles {
@@ -65,10 +70,6 @@ public class BackdoorToggles {
return Integer.valueOf(v);
}
- public static String getQueryId() {
- return getString(KEY_QUERY_ID);
- }
-
public static Pair<Short, Short> getShardAssignment() {
String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
if (v == null) {
@@ -104,8 +105,6 @@ public class BackdoorToggles {
_backdoorToggles.remove();
}
- public final static String KEY_QUERY_ID = "QUERY_ID";
-
/**
* set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index bd8b7e2..1896f4f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -25,6 +25,7 @@ public class SQLRequest implements Serializable {
protected static final long serialVersionUID = 1L;
private String sql;
+
private String project;
private Integer offset = 0;
private Integer limit = 0;
@@ -83,19 +84,25 @@ public class SQLRequest implements Serializable {
this.acceptPartial = acceptPartial;
}
-
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
SQLRequest that = (SQLRequest) o;
- if (acceptPartial != that.acceptPartial) return false;
- if (sql != null ? !sql.equals(that.sql) : that.sql != null) return false;
- if (project != null ? !project.equals(that.project) : that.project != null) return false;
- if (offset != null ? !offset.equals(that.offset) : that.offset != null) return false;
- if (limit != null ? !limit.equals(that.limit) : that.limit != null) return false;
+ if (acceptPartial != that.acceptPartial)
+ return false;
+ if (sql != null ? !sql.equals(that.sql) : that.sql != null)
+ return false;
+ if (project != null ? !project.equals(that.project) : that.project != null)
+ return false;
+ if (offset != null ? !offset.equals(that.offset) : that.offset != null)
+ return false;
+ if (limit != null ? !limit.equals(that.limit) : that.limit != null)
+ return false;
return backdoorToggles != null ? backdoorToggles.equals(that.backdoorToggles) : that.backdoorToggles == null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 82e4a87..0dd5c5f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.DBUtils;
@@ -266,7 +267,7 @@ public class QueryService extends BasicService {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(newLine);
stringBuilder.append("==========================[QUERY]===============================").append(newLine);
- stringBuilder.append("Query Id: ").append(BackdoorToggles.getQueryId()).append(newLine);
+ stringBuilder.append("Query Id: ").append(QueryContext.getQueryId()).append(newLine);
stringBuilder.append("SQL: ").append(request.getSql()).append(newLine);
stringBuilder.append("User: ").append(user).append(newLine);
stringBuilder.append("Success: ").append((null == response.getExceptionMessage())).append(newLine);
@@ -324,13 +325,7 @@ public class QueryService extends BasicService {
}
final String queryId = UUID.randomUUID().toString();
- Map<String, String> toggles = new HashMap<>();
- toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
- if (sqlRequest.getBackdoorToggles() != null) {
- toggles.putAll(sqlRequest.getBackdoorToggles());
- }
- sqlRequest.setBackdoorToggles(toggles);
- BackdoorToggles.setToggles(toggles);
+ QueryContext.setQueryId(queryId);
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
String sql = sqlRequest.getSql();
@@ -442,7 +437,9 @@ public class QueryService extends BasicService {
String correctedSql = QueryUtil.massageSql(sqlRequest);
if (!correctedSql.equals(sqlRequest.getSql())) {
logger.info("The corrected query: " + correctedSql);
- sqlRequest.setSql(correctedSql);
+
+ //CAUTION: should not change sqlRequest content!
+ //sqlRequest.setSql(correctedSql);
}
// add extra parameters into olap context, like acceptPartial
@@ -521,12 +518,12 @@ public class QueryService extends BasicService {
}
/**
- * @param sql
+ * @param correctedSql
* @param sqlRequest
* @return
* @throws Exception
*/
- private SQLResponse execute(String sql, SQLRequest sqlRequest) throws Exception {
+ private SQLResponse execute(String correctedSql, SQLRequest sqlRequest) throws Exception {
Connection conn = null;
Statement stat = null;
ResultSet resultSet = null;
@@ -538,7 +535,7 @@ public class QueryService extends BasicService {
conn = cacheService.getOLAPDataSource(sqlRequest.getProject()).getConnection();
if (sqlRequest instanceof PrepareSqlRequest) {
- PreparedStatement preparedState = conn.prepareStatement(sql);
+ PreparedStatement preparedState = conn.prepareStatement(correctedSql);
processStatementAttr(preparedState, sqlRequest);
for (int i = 0; i < ((PrepareSqlRequest) sqlRequest).getParams().length; i++) {
@@ -549,7 +546,7 @@ public class QueryService extends BasicService {
} else {
stat = conn.createStatement();
processStatementAttr(stat, sqlRequest);
- resultSet = stat.executeQuery(sql);
+ resultSet = stat.executeQuery(correctedSql);
}
ResultSetMetaData metaData = resultSet.getMetaData();
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index ebacb26..df1817e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
@@ -158,7 +158,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize());
builder.setKylinProperties(kylinConfig.getConfigAsString());
- final String queryId = BackdoorToggles.getQueryId();
+ final String queryId = QueryContext.getQueryId();
if (queryId != null) {
builder.setQueryId(queryId);
}
[38/42] kylin git commit: KYLIN-2328 minor review,
open protected dumpKylinPropsAndMetadata for sub-class
Posted by sh...@apache.org.
KYLIN-2328 minor review, open protected dumpKylinPropsAndMetadata for sub-class
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5cbef32f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5cbef32f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5cbef32f
Branch: refs/heads/sparkcubing-rebase
Commit: 5cbef32fcd51bffe31f92a985bb5d02405fb7b7c
Parents: 75c8d13
Author: Yang Li <li...@apache.org>
Authored: Fri Dec 30 06:57:42 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Dec 30 06:57:42 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cbef32f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 4693ac3..e4e5e82 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -451,11 +451,11 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected void attachTableMetadata(TableDesc table, Configuration conf) throws IOException {
Set<String> dumpList = new LinkedHashSet<>();
dumpList.add(table.getResourcePath());
- attachKylinPropsAndMetadata(dumpList, KylinConfig.getInstanceFromEnv(), conf);
+ dumpKylinPropsAndMetadata(dumpList, KylinConfig.getInstanceFromEnv(), conf);
}
protected void attachCubeMetadata(CubeInstance cube, Configuration conf) throws IOException {
- attachKylinPropsAndMetadata(collectCubeMetadata(cube), cube.getConfig(), conf);
+ dumpKylinPropsAndMetadata(collectCubeMetadata(cube), cube.getConfig(), conf);
}
protected void attachCubeMetadataWithDict(CubeInstance cube, Configuration conf) throws IOException {
@@ -464,14 +464,14 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
for (CubeSegment segment : cube.getSegments()) {
dumpList.addAll(segment.getDictionaryPaths());
}
- attachKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
+ dumpKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
}
protected void attachSegmentMetadataWithDict(CubeSegment segment, Configuration conf) throws IOException {
Set<String> dumpList = new LinkedHashSet<>();
dumpList.addAll(collectCubeMetadata(segment.getCubeInstance()));
dumpList.addAll(segment.getDictionaryPaths());
- attachKylinPropsAndMetadata(dumpList, segment.getConfig(), conf);
+ dumpKylinPropsAndMetadata(dumpList, segment.getConfig(), conf);
}
private Set<String> collectCubeMetadata(CubeInstance cube) {
@@ -490,7 +490,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
return dumpList;
}
- private void attachKylinPropsAndMetadata(Set<String> dumpList, KylinConfig kylinConfig, Configuration conf) throws IOException {
+ protected void dumpKylinPropsAndMetadata(Set<String> dumpList, KylinConfig kylinConfig, Configuration conf) throws IOException {
File tmp = File.createTempFile("kylin_job_meta", "");
FileUtils.forceDelete(tmp); // we need a directory, so delete the file first
[28/42] kylin git commit: minor,
get config by prefix in KylinConfigCLI
Posted by sh...@apache.org.
minor, get config by prefix in KylinConfigCLI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdecf093
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdecf093
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdecf093
Branch: refs/heads/sparkcubing-rebase
Commit: fdecf0937802a2d17418595e6c0543c9668ac8a4
Parents: 08eda7b
Author: lidongsjtu <li...@apache.org>
Authored: Wed Dec 28 22:17:35 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Dec 28 22:45:57 2016 +0800
----------------------------------------------------------------------
.../test_case_data/localmeta/kylin.properties | 3 +
.../org/apache/kylin/tool/KylinConfigCLI.java | 30 ++++++--
.../apache/kylin/tool/KylinConfigCLITest.java | 73 ++++++++++++++++++++
3 files changed, 102 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdecf093/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 1dfac32..f4c6772 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -131,3 +131,6 @@ kylin.test.bcc.new.key=some-value
kylin.engine.mr.config-override.test1=test1
kylin.engine.mr.config-override.test2=test2
kylin.job.lock=org.apache.kylin.job.lock.MockJobLock
+
+kylin.engine.provider.0=org.apache.kylin.engine.mr.MRBatchCubingEngine
+kylin.cube.engine.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdecf093/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
index e1a5b99..b740beb 100644
--- a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
@@ -18,11 +18,14 @@
package org.apache.kylin.tool;
+import java.util.Map;
import java.util.Properties;
import org.apache.kylin.common.BackwardCompatibilityConfig;
import org.apache.kylin.common.KylinConfig;
+import com.google.common.collect.Maps;
+
public class KylinConfigCLI {
public static void main(String[] args) {
if (args.length != 1) {
@@ -33,10 +36,29 @@ public class KylinConfigCLI {
Properties config = KylinConfig.getKylinProperties();
BackwardCompatibilityConfig bcc = new BackwardCompatibilityConfig();
- String value = config.getProperty(bcc.check(args[0]));
- if (value == null) {
- value = "";
+ String key = bcc.check(args[0].trim());
+ if (!key.endsWith(".")) {
+ String value = config.getProperty(key);
+ if (value == null) {
+ value = "";
+ }
+ System.out.println(value);
+ } else {
+ Map<String, String> props = getPropertiesByPrefix(config, key);
+ for (Map.Entry<String, String> prop : props.entrySet()) {
+ System.out.println(prop.getKey() + "=" + prop.getValue());
+ }
+ }
+ }
+
+ static private Map<String, String> getPropertiesByPrefix(Properties props, String prefix) {
+ Map<String, String> result = Maps.newLinkedHashMap();
+ for (Map.Entry<Object, Object> entry : props.entrySet()) {
+ String entryKey = (String) entry.getKey();
+ if (entryKey.startsWith(prefix)) {
+ result.put(entryKey.substring(prefix.length()), (String) entry.getValue());
+ }
}
- System.out.println(value);
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdecf093/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
----------------------------------------------------------------------
diff --git a/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
new file mode 100644
index 0000000..7d1e248
--- /dev/null
+++ b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.tool;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.charset.Charset;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class KylinConfigCLITest extends LocalFileMetadataTestCase {
+ @Test
+ public void testGetProperty() throws IOException {
+ PrintStream o = System.out;
+ File f = File.createTempFile("cfg", ".tmp");
+ System.setOut(new PrintStream(new FileOutputStream(f)));
+ KylinConfigCLI.main(new String[] { "kylin.storage.url" });
+
+ String val = FileUtils.readFileToString(f, Charset.defaultCharset()).trim();
+ assertEquals("hbase", val);
+
+ FileUtils.forceDelete(f);
+ System.setOut(o);
+ }
+
+ @Test
+ public void testGetPrefix() throws IOException {
+ PrintStream o = System.out;
+ File f = File.createTempFile("cfg", ".tmp");
+ System.setOut(new PrintStream(new FileOutputStream(f)));
+ KylinConfigCLI.main(new String[] { "kylin.cube.engine." });
+
+ String val = FileUtils.readFileToString(f, Charset.defaultCharset()).trim();
+ assertEquals("2=org.apache.kylin.engine.mr.MRBatchCubingEngine2\n0=org.apache.kylin.engine.mr.MRBatchCubingEngine", val);
+
+ FileUtils.forceDelete(f);
+ System.setOut(o);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+}
[41/42] kylin git commit: KYLIN-2331 by layer spark cubing
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
index 2b28302..78d4f7b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
@@ -87,6 +87,24 @@ public class MeasureAggregators implements Serializable {
}
}
+ public void aggregate(Object[] values1, Object[] values2, Object[] result) {
+ assert values1.length == values2.length && values2.length == descLength && values1.length == result.length;
+
+ for (int i = 0; i < descLength; i++) {
+ result[i] = aggs[i].aggregate(values1[i], values2[i]);
+ }
+
+ }
+
+ public void aggregate(Object[] values1, Object[] values2, Object[] result, boolean[] aggrMask) {
+ assert values1.length == values2.length && values2.length == descLength && values1.length == result.length && result.length == aggrMask.length;
+ for (int i = 0; i < descLength; i++) {
+ if (aggrMask[i]) {
+ result[i] = aggs[i].aggregate(values1[i], values2[i]);
+ }
+ }
+ }
+
public void collectStates(Object[] states) {
for (int i = 0; i < descLength; i++) {
states[i] = aggs[i].getState();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
index edaf806..2d73e59 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
@@ -18,19 +18,19 @@
package org.apache.kylin.measure;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.apache.kylin.metadata.model.MeasureDesc;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
/**
* @author yangli9
*
*/
@SuppressWarnings({ "rawtypes" })
-public class MeasureCodec {
+public class MeasureCodec implements java.io.Serializable {
private int nMeasures;
private DataTypeSerializer[] serializers;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
index 0076252..26b7298 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
@@ -18,14 +18,14 @@
package org.apache.kylin.measure;
-import java.util.Collection;
-import java.util.Map;
-
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-abstract public class MeasureIngester<V> {
+import java.util.Collection;
+import java.util.Map;
+
+abstract public class MeasureIngester<V> implements java.io.Serializable {
public static MeasureIngester<?> create(MeasureDesc measure) {
return measure.getFunction().getMeasureType().newIngester();
@@ -42,6 +42,10 @@ abstract public class MeasureIngester<V> {
abstract public V valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap);
+ public void reset() {
+
+ }
+
public V reEncodeDictionary(V value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 89ff382..3338c8c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -18,11 +18,6 @@
package org.apache.kylin.measure;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -32,13 +27,18 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.metadata.tuple.TupleInfo;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
/**
* MeasureType captures how a kind of aggregation is defined, how it is calculated
* during cube build, and how it is involved in query and storage scan.
*
* @param <T> the Java type of aggregation data object, e.g. HLLCounter
*/
-abstract public class MeasureType<T> {
+abstract public class MeasureType<T> implements java.io.Serializable {
/* ============================================================================
* Define
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
index aa42476..7a57965 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
@@ -43,6 +43,20 @@ public class BigDecimalMaxAggregator extends MeasureAggregator<BigDecimal> {
}
@Override
+ public BigDecimal aggregate(BigDecimal value1, BigDecimal value2) {
+ if (value1 == null) {
+ return value2;
+ } else if (value2 == null) {
+ return value1;
+ }
+
+ if (value1.compareTo(value2) > 0)
+ return value1;
+ else
+ return value2;
+ }
+
+ @Override
public BigDecimal getState() {
return max;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
index 81193ad..71ba7fb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
@@ -27,24 +27,38 @@ import org.apache.kylin.measure.MeasureAggregator;
@SuppressWarnings("serial")
public class BigDecimalMinAggregator extends MeasureAggregator<BigDecimal> {
- BigDecimal max = null;
+ BigDecimal min = null;
@Override
public void reset() {
- max = null;
+ min = null;
}
@Override
public void aggregate(BigDecimal value) {
- if (max == null)
- max = value;
- else if (max.compareTo(value) > 0)
- max = value;
+ if (min == null)
+ min = value;
+ else if (min.compareTo(value) > 0)
+ min = value;
+ }
+
+ @Override
+ public BigDecimal aggregate(BigDecimal value1, BigDecimal value2) {
+ if (value1 == null) {
+ return value2;
+ } else if (value2 == null) {
+ return value1;
+ }
+
+ if (value1.compareTo(value2) > 0)
+ return value2;
+ else
+ return value1;
}
@Override
public BigDecimal getState() {
- return max;
+ return min;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
index 5e00c63..9f6ffc2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
@@ -40,6 +40,16 @@ public class BigDecimalSumAggregator extends MeasureAggregator<BigDecimal> {
}
@Override
+ public BigDecimal aggregate(BigDecimal value1, BigDecimal value2) {
+ if (value1 == null) {
+ return value2;
+ } else if (value2 == null) {
+ return value1;
+ }
+ return value1.add(value2);
+ }
+
+ @Override
public BigDecimal getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
index a2f3980..90ecb0d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
@@ -22,25 +22,24 @@ import java.util.Map;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-public class DoubleIngester extends MeasureIngester<DoubleMutable> {
-
- // avoid repeated object creation
- private DoubleMutable current = new DoubleMutable();
+public class DoubleIngester extends MeasureIngester<Double> {
@Override
- public DoubleMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ public Double valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length > 1)
throw new IllegalArgumentException();
- DoubleMutable l = current;
if (values[0] == null || values[0].length() == 0)
- l.set(0L);
+ return new Double(0);
else
- l.set(Double.parseDouble(values[0]));
- return l;
+ return Double.parseDouble(values[0]);
+ }
+
+ @Override
+ public void reset() {
+
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
index 25911e8..f33555e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
/**
*/
@SuppressWarnings("serial")
-public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
+public class DoubleMaxAggregator extends MeasureAggregator<Double> {
- DoubleMutable max = null;
+ Double max = null;
@Override
public void reset() {
@@ -34,15 +33,20 @@ public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
}
@Override
- public void aggregate(DoubleMutable value) {
+ public void aggregate(Double value) {
if (max == null)
- max = new DoubleMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
+ max = value;
+ else if (max < value)
+ max = value;
}
@Override
- public DoubleMutable getState() {
+ public Double aggregate(Double value1, Double value2) {
+ return Math.max(value1, value2);
+ }
+
+ @Override
+ public Double getState() {
return max;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
index be97deb..8e69f21 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
/**
*/
@SuppressWarnings("serial")
-public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
+public class DoubleMinAggregator extends MeasureAggregator<Double> {
- DoubleMutable min = null;
+ Double min = null;
@Override
public void reset() {
@@ -34,15 +33,20 @@ public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
}
@Override
- public void aggregate(DoubleMutable value) {
+ public void aggregate(Double value) {
if (min == null)
- min = new DoubleMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
+ min = value;
+ else if (min > value)
+ min = value;
}
@Override
- public DoubleMutable getState() {
+ public Double aggregate(Double value1, Double value2) {
+ return Math.min(value1, value2);
+ }
+
+ @Override
+ public Double getState() {
return min;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
index f276817..df0ba52 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
@@ -19,27 +19,31 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
/**
*/
@SuppressWarnings("serial")
-public class DoubleSumAggregator extends MeasureAggregator<DoubleMutable> {
+public class DoubleSumAggregator extends MeasureAggregator<Double> {
- DoubleMutable sum = new DoubleMutable();
+ Double sum = new Double(0);
@Override
public void reset() {
- sum.set(0.0);
+ sum = new Double(0);
}
@Override
- public void aggregate(DoubleMutable value) {
- sum.set(sum.get() + value.get());
+ public void aggregate(Double value) {
+ sum = sum + value;
}
@Override
- public DoubleMutable getState() {
+ public Double aggregate(Double value1, Double value2) {
+ return Double.valueOf(value1 + value2);
+ }
+
+ @Override
+ public Double getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
index 45a1634..793acf2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
@@ -22,25 +22,24 @@ import java.util.Map;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-public class LongIngester extends MeasureIngester<LongMutable> {
-
- // avoid repeated object creation
- private LongMutable current = new LongMutable();
+public class LongIngester extends MeasureIngester<Long> {
@Override
- public LongMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ public Long valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length > 1)
throw new IllegalArgumentException();
- LongMutable l = current;
if (values[0] == null || values[0].length() == 0)
- l.set(0L);
+ return new Long(0L);
else
- l.set(Long.parseLong(values[0]));
- return l;
+ return Long.valueOf(values[0]);
+ }
+
+ @Override
+ public void reset() {
+
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
index ca44f15..b9a2b94 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.LongMutable;
/**
*/
@SuppressWarnings("serial")
-public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
+public class LongMaxAggregator extends MeasureAggregator<Long> {
- LongMutable max = null;
+ Long max = null;
@Override
public void reset() {
@@ -34,15 +33,20 @@ public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
}
@Override
- public void aggregate(LongMutable value) {
+ public void aggregate(Long value) {
if (max == null)
- max = new LongMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
+ max = value;
+ else if (max < value)
+ max = value;
}
@Override
- public LongMutable getState() {
+ public Long aggregate(Long value1, Long value2) {
+ return Math.max(value1, value2);
+ }
+
+ @Override
+ public Long getState() {
return max;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
index dadc64e..9185142 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.LongMutable;
/**
*/
@SuppressWarnings("serial")
-public class LongMinAggregator extends MeasureAggregator<LongMutable> {
+public class LongMinAggregator extends MeasureAggregator<Long> {
- LongMutable min = null;
+ Long min = null;
@Override
public void reset() {
@@ -34,15 +33,20 @@ public class LongMinAggregator extends MeasureAggregator<LongMutable> {
}
@Override
- public void aggregate(LongMutable value) {
+ public void aggregate(Long value) {
if (min == null)
- min = new LongMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
+ min = value;
+ else if (min > value)
+ min = value;
}
@Override
- public LongMutable getState() {
+ public Long aggregate(Long value1, Long value2) {
+ return Math.min(value1, value2);
+ }
+
+ @Override
+ public Long getState() {
return min;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
index e7fdc9d..1f9c0d7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
@@ -19,27 +19,31 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.LongMutable;
/**
*/
@SuppressWarnings("serial")
-public class LongSumAggregator extends MeasureAggregator<LongMutable> {
+public class LongSumAggregator extends MeasureAggregator<Long> {
- LongMutable sum = new LongMutable();
+ Long sum = new Long(0L);
@Override
public void reset() {
- sum.set(0);
+ sum = new Long(0L);
}
@Override
- public void aggregate(LongMutable value) {
- sum.set(sum.get() + value.get());
+ public void aggregate(Long value) {
+ sum += value;
}
@Override
- public LongMutable getState() {
+ public Long aggregate(Long value1, Long value2) {
+ return Long.valueOf(value1 + value2);
+ }
+
+ @Override
+ public Long getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
index be72090..cd0b4bb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
@@ -42,6 +42,19 @@ public class BitmapAggregator extends MeasureAggregator<BitmapCounter> {
}
@Override
+ public BitmapCounter aggregate(BitmapCounter value1, BitmapCounter value2) {
+ if (value1 == null) {
+ return new BitmapCounter(value2);
+ } else if (value2 == null) {
+ return new BitmapCounter(value1);
+ }
+
+ BitmapCounter merged = new BitmapCounter(value1);
+ merged.merge(value2);
+ return merged;
+ }
+
+ @Override
public BitmapCounter getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index 827390d..caab094 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -18,6 +18,9 @@
package org.apache.kylin.measure.bitmap;
+import org.apache.commons.io.IOUtils;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
@@ -26,13 +29,10 @@ import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Iterator;
-import org.apache.commons.io.IOUtils;
-import org.roaringbitmap.buffer.MutableRoaringBitmap;
-
/**
* Created by sunyerui on 15/12/1.
*/
-public class BitmapCounter implements Comparable<BitmapCounter> {
+public class BitmapCounter implements Comparable<BitmapCounter>, java.io.Serializable {
private MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index 8e2b2f7..6ad82a1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -138,6 +138,11 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
}
return retValue;
}
+
+ @Override
+ public void reset() {
+ current = new BitmapCounter();
+ }
};
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
index 089d18c..c40f71b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
@@ -29,7 +29,7 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
*/
public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
- private ThreadLocal<BitmapCounter> current = new ThreadLocal<>();
+ private transient ThreadLocal<BitmapCounter> current = new ThreadLocal<>();
public BitmapSerializer(DataType type) {
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
index 6fa8788..1b2cda3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
@@ -254,6 +254,21 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> {
}
@Override
+ public ByteArray aggregate(ByteArray value1, ByteArray value2) {
+ if (value1 == null) {
+ return value2;
+ } else if (value2 == null) {
+ return value1;
+ } else if (!value1.equals(value2)) {
+ if (!warned) {
+ logger.warn("Extended column must be unique given same host column");
+ warned = true;
+ }
+ }
+ return value1;
+ }
+
+ @Override
public ByteArray getState() {
return byteArray;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
index 5b929b2..4be9f71 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
@@ -24,7 +24,7 @@ import java.util.Map;
/**
* Created by xiefan on 16-12-9.
*/
-public class DenseRegister implements Register {
+public class DenseRegister implements Register, java.io.Serializable {
private int m;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
index 5966c04..7d5b216 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
@@ -46,6 +46,13 @@ public class HLLCAggregator extends MeasureAggregator<HLLCounter> {
}
@Override
+ public HLLCounter aggregate(HLLCounter value1, HLLCounter value2) {
+ HLLCounter result = new HLLCounter(value1);
+ result.merge(value2);
+ return result;
+ }
+
+ @Override
public HLLCounter getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index 9601653..de36b08 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -105,6 +105,11 @@ public class HLLCMeasureType extends MeasureType<HLLCounter> {
}
return hllc;
}
+
+ @Override
+ public void reset() {
+ current = new HLLCounter(dataType.getPrecision());
+ }
};
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
index e0992c7..df0cfaf 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
@@ -18,21 +18,18 @@
package org.apache.kylin.measure.hllc;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
/**
* @author yangli9
*
*/
public class HLLCSerializer extends DataTypeSerializer<HLLCounter> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<HLLCounter> current = new ThreadLocal<HLLCounter>();
-
private int precision;
public HLLCSerializer(DataType type) {
@@ -49,7 +46,7 @@ public class HLLCSerializer extends DataTypeSerializer<HLLCounter> {
}
private HLLCounter current() {
- HLLCounter hllc = current.get();
+ HLLCounter hllc = (HLLCounter) current.get();
if (hllc == null) {
hllc = new HLLCounter(precision);
current.set(hllc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusTable.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusTable.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusTable.java
index 9d8de07..5d7bfeb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusTable.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusTable.java
@@ -25,7 +25,7 @@ import java.util.TreeMap;
* @author xjiang
*
*/
-public class HyperLogLogPlusTable {
+public class HyperLogLogPlusTable implements java.io.Serializable {
// threshold and bias data taken from google's bias correction data set:
// https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen#
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SingleValueRegister.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SingleValueRegister.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SingleValueRegister.java
index 7f612e2..ad6306a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SingleValueRegister.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SingleValueRegister.java
@@ -20,7 +20,7 @@ package org.apache.kylin.measure.hllc;
/**
* Created by xiefan on 16-12-20.
*/
-public class SingleValueRegister implements Register {
+public class SingleValueRegister implements Register, java.io.Serializable {
private int singleValuePos;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
index dd7d7c8..d47a05b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
@@ -25,7 +25,7 @@ import java.util.TreeMap;
/**
* Created by xiefan on 16-12-9.
*/
-public class SparseRegister implements Register {
+public class SparseRegister implements Register, java.io.Serializable {
private Map<Integer, Byte> sparseRegister = new TreeMap<>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
index 6f66269..c3ecc74 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
@@ -47,6 +47,20 @@ public class RawAggregator extends MeasureAggregator<List<ByteArray>> {
}
@Override
+ public List<ByteArray> aggregate(List<ByteArray> value1, List<ByteArray> value2) {
+ if (value1 == null) {
+ return value2;
+ } else if (value2 == null) {
+ return value1;
+ }
+
+ List<ByteArray> result = new ArrayList<>(value1.size() + value2.size());
+ result.addAll(value1);
+ result.addAll(value2);
+ return result;
+ }
+
+ @Override
public List<ByteArray> getState() {
return list;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
index 021c146..68a0273 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
@@ -34,13 +34,11 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> {
//FIXME to config this and RowConstants.ROWVALUE_BUFFER_SIZE in properties file
public static final int RAW_BUFFER_SIZE = 1024 * 1024;//1M
- private ThreadLocal<List<ByteArray>> current = new ThreadLocal<>();
-
public RawSerializer(DataType dataType) {
}
private List<ByteArray> current() {
- List<ByteArray> l = current.get();
+ List<ByteArray> l = (List<ByteArray>) current.get();
if (l == null) {
l = new ArrayList<ByteArray>();
current.set(l);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
index 38d5b20..ac925e2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
@@ -24,7 +24,7 @@ import java.nio.ByteBuffer;
*
* http://bitcharmer.blogspot.co.uk/2013/12/how-to-serialise-array-of-doubles-with.html
*/
-public class DoubleDeltaSerializer {
+public class DoubleDeltaSerializer implements java.io.Serializable {
// first 32 bits stores meta info
static final int PRECISION_BITS = 3;
@@ -55,7 +55,6 @@ public class DoubleDeltaSerializer {
this.precision = precision;
this.multiplier = (int) Math.pow(10, precision);
- this.deltasThreadLocal = new ThreadLocal<long[]>();
}
public void serialize(double[] values, ByteBuffer buf) {
@@ -112,6 +111,10 @@ public class DoubleDeltaSerializer {
int len = values.length - 1;
len = Math.max(0, len);
+ if (deltasThreadLocal == null) {
+ deltasThreadLocal = new ThreadLocal<>();
+ }
+
long[] deltas = deltasThreadLocal.get();
if (deltas == null || deltas.length < len) {
deltas = new long[len];
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
index ef997eb..b5e316f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
@@ -45,6 +45,15 @@ public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
}
@Override
+ public TopNCounter<ByteArray> aggregate(TopNCounter<ByteArray> value1, TopNCounter<ByteArray> value2) {
+ TopNCounter<ByteArray> aggregated = new TopNCounter<>(capacity * 2);
+ aggregated.merge(value1);
+ aggregated.merge(value2);
+ aggregated.retain(capacity);
+ return aggregated;
+ }
+
+ @Override
public TopNCounter<ByteArray> getState() {
sum.retain(capacity);
return sum;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index caf7961..5e4b91e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -40,7 +40,7 @@ import com.google.common.collect.Maps;
*
* @param <T> type of data in the stream to be summarized
*/
-public class TopNCounter<T> implements Iterable<Counter<T>> {
+public class TopNCounter<T> implements Iterable<Counter<T>>, java.io.Serializable {
public static final int EXTRA_SPACE_RATE = 50;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index c29af6c..8c8b5a6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Dictionary;
@@ -46,8 +47,6 @@ import org.apache.kylin.metadata.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-
public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
private static final Logger logger = LoggerFactory.getLogger(TopNMeasureType.class);
@@ -156,6 +155,7 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
return topNCounter;
}
+
@Override
public TopNCounter<ByteArray> reEncodeDictionary(TopNCounter<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
TopNCounter<ByteArray> topNCounter = value;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
index 64968b8..b5043f5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
@@ -35,7 +35,7 @@ public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
private static final Logger logger = LoggerFactory.getLogger(BigDecimalSerializer.class);
final DataType type;
- transient final int maxLength;
+ final int maxLength;
transient int avoidVerbose = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
index acb6de1..3d485d2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
@@ -18,40 +18,26 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.BooleanUtils;
-public class BooleanSerializer extends DataTypeSerializer<LongMutable> {
+import java.nio.ByteBuffer;
- public final static String[] TRUE_VALUE_SET = { "true", "t", "on", "yes" };
+public class BooleanSerializer extends DataTypeSerializer<Long> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+ public final static String[] TRUE_VALUE_SET = { "true", "t", "on", "yes" };
public BooleanSerializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- out.putLong(value.get());
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Long value, ByteBuffer out) {
+ out.putLong(value);
}
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(in.getLong());
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return in.getLong();
}
@Override
@@ -70,12 +56,10 @@ public class BooleanSerializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- LongMutable l = current();
+ public Long valueOf(String str) {
if (str == null)
- l.set(0L);
+ return Long.valueOf(0L);
else
- l.set(BooleanUtils.toInteger(ArrayUtils.contains(TRUE_VALUE_SET, str.toLowerCase())));
- return l;
+ return Long.valueOf(BooleanUtils.toInteger(ArrayUtils.contains(TRUE_VALUE_SET, str.toLowerCase())));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
index a739377..a4a35a4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
@@ -18,19 +18,21 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-import java.util.Map;
-
+import com.google.common.collect.Maps;
import org.apache.kylin.common.util.BytesSerializer;
-import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.nio.ByteBuffer;
+import java.util.Map;
/**
* Note: the implementations MUST be thread-safe.
*/
-abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
+abstract public class DataTypeSerializer<T> implements BytesSerializer<T>, java.io.Serializable {
final static Map<String, Class<?>> implementations = Maps.newHashMap();
+ protected transient ThreadLocal current = new ThreadLocal();
static {
implementations.put("char", StringSerializer.class);
implementations.put("varchar", StringSerializer.class);
@@ -94,4 +96,9 @@ abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
else
return value.toString();
}
+
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ current = new ThreadLocal();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
index 07f98b3..5101766 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
@@ -18,37 +18,23 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.common.util.DateFormat;
-public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
+import java.nio.ByteBuffer;
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+public class DateTimeSerializer extends DataTypeSerializer<Long> {
public DateTimeSerializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- out.putLong(value.get());
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Long value, ByteBuffer out) {
+ out.putLong(value);
}
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(in.getLong());
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return in.getLong();
}
@Override
@@ -67,8 +53,8 @@ public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- return new LongMutable(DateFormat.stringToMillis(str));
+ public Long valueOf(String str) {
+ return Long.valueOf(DateFormat.stringToMillis(str));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
index 976dc51..20cfff3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
@@ -22,33 +22,19 @@ import java.nio.ByteBuffer;
/**
*/
-public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<DoubleMutable> current = new ThreadLocal<DoubleMutable>();
+public class DoubleSerializer extends DataTypeSerializer<Double> {
public DoubleSerializer(DataType type) {
}
@Override
- public void serialize(DoubleMutable value, ByteBuffer out) {
- out.putDouble(value.get());
- }
-
- private DoubleMutable current() {
- DoubleMutable d = current.get();
- if (d == null) {
- d = new DoubleMutable();
- current.set(d);
- }
- return d;
+ public void serialize(Double value, ByteBuffer out) {
+ out.putDouble(value);
}
@Override
- public DoubleMutable deserialize(ByteBuffer in) {
- DoubleMutable d = current();
- d.set(in.getDouble());
- return d;
+ public Double deserialize(ByteBuffer in) {
+ return in.getDouble();
}
@Override
@@ -67,7 +53,7 @@ public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
}
@Override
- public DoubleMutable valueOf(String str) {
- return new DoubleMutable(Double.parseDouble(str));
+ public Double valueOf(String str) {
+ return Double.parseDouble(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
index 7b95505..0e82e11 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
@@ -18,39 +18,25 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.common.util.BytesUtil;
+import java.nio.ByteBuffer;
+
/**
*/
-public class Int4Serializer extends DataTypeSerializer<IntMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<IntMutable> current = new ThreadLocal<IntMutable>();
+public class Int4Serializer extends DataTypeSerializer<Integer> {
public Int4Serializer(DataType type) {
}
@Override
- public void serialize(IntMutable value, ByteBuffer out) {
- BytesUtil.writeUnsigned(value.get(), 4, out);
- }
-
- private IntMutable current() {
- IntMutable l = current.get();
- if (l == null) {
- l = new IntMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Integer value, ByteBuffer out) {
+ BytesUtil.writeUnsigned(value, 4, out);
}
@Override
- public IntMutable deserialize(ByteBuffer in) {
- IntMutable l = current();
- l.set(BytesUtil.readUnsigned(in, 4));
- return l;
+ public Integer deserialize(ByteBuffer in) {
+ return BytesUtil.readUnsigned(in, 4);
}
@Override
@@ -69,7 +55,7 @@ public class Int4Serializer extends DataTypeSerializer<IntMutable> {
}
@Override
- public IntMutable valueOf(String str) {
- return new IntMutable(Integer.parseInt(str));
+ public Integer valueOf(String str) {
+ return Integer.parseInt(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
index fa333b2..7dd5aa7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
@@ -18,39 +18,26 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.common.util.BytesUtil;
+import java.nio.ByteBuffer;
+
/**
*/
-public class Long8Serializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+public class Long8Serializer extends DataTypeSerializer<Long> {
public Long8Serializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- BytesUtil.writeLong(value.get(), out);
+ public void serialize(Long value, ByteBuffer out) {
+ BytesUtil.writeLong(value, out);
}
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(BytesUtil.readLong(in));
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return BytesUtil.readLong(in);
}
@Override
@@ -69,7 +56,7 @@ public class Long8Serializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- return new LongMutable(Long.parseLong(str));
+ public Long valueOf(String str) {
+ return Long.parseLong(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
index 9306a70..605dcd7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
@@ -18,39 +18,25 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.common.util.BytesUtil;
+import java.nio.ByteBuffer;
+
/**
*/
-public class LongSerializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+public class LongSerializer extends DataTypeSerializer<Long> {
public LongSerializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- BytesUtil.writeVLong(value.get(), out);
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Long value, ByteBuffer out) {
+ BytesUtil.writeVLong(value, out);
}
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(BytesUtil.readVLong(in));
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return BytesUtil.readVLong(in);
}
@Override
@@ -75,7 +61,7 @@ public class LongSerializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- return new LongMutable(Long.parseLong(str));
+ public Long valueOf(String str) {
+ return Long.parseLong(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 4d89e1a..ce5888d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -18,27 +18,27 @@
package org.apache.kylin.metadata.model;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.measure.MeasureTypeFactory;
-import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.datatype.DataType;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.datatype.DataType;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class FunctionDesc {
+public class FunctionDesc implements Serializable {
public static FunctionDesc newInstance(String expression, ParameterDesc param, String returnType) {
FunctionDesc r = new FunctionDesc();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 6489244..dd1500b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -18,17 +18,18 @@
package org.apache.kylin.metadata.model;
-import java.util.Arrays;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
+import java.io.Serializable;
+import java.util.Arrays;
+
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class JoinDesc {
+public class JoinDesc implements Serializable {
// inner, left, right, outer...
@JsonProperty("type")
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
index 5d0409a..51e5787 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
@@ -25,8 +25,10 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.Serializable;
+
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class JoinTableDesc {
+public class JoinTableDesc implements Serializable {
@JsonProperty("table")
private String table;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index a0b267d..c132d0e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -18,15 +18,16 @@
package org.apache.kylin.metadata.model;
+import com.google.common.base.Preconditions;
+
+import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import com.google.common.base.Preconditions;
-
-public class JoinsTree {
+public class JoinsTree implements Serializable {
final Map<String, Chain> tableChains = new LinkedHashMap<>();
@@ -111,7 +112,7 @@ public class JoinsTree {
return chain.join;
}
- static class Chain {
+ static class Chain implements java.io.Serializable {
TableRef table; // pk side
JoinDesc join;
Chain fkSide;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index 253b06b..c0719d2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -18,18 +18,19 @@
package org.apache.kylin.metadata.model;
-import java.util.Objects;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.Serializable;
+import java.util.Objects;
+
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class MeasureDesc {
+public class MeasureDesc implements Serializable {
@JsonProperty("name")
private String name;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
index 6460f71..3c5c5f1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
@@ -18,17 +18,17 @@
package org.apache.kylin.metadata.model;
-import java.util.List;
-
-import org.apache.kylin.common.util.StringUtil;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.common.util.StringUtil;
+
+import java.io.Serializable;
+import java.util.List;
/**
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class ModelDimensionDesc {
+public class ModelDimensionDesc implements Serializable {
@JsonProperty("table")
private String table;
@JsonProperty("columns")
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index c14d061..8ad20a8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -18,21 +18,22 @@
package org.apache.kylin.metadata.model;
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-import java.util.List;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.List;
+
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class ParameterDesc {
+public class ParameterDesc implements Serializable {
public static ParameterDesc newInstance(Object... objs) {
if (objs.length == 0)
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 9925990..c6e6425 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -27,10 +27,12 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.Serializable;
+
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class PartitionDesc {
+public class PartitionDesc implements Serializable {
public static enum PartitionType {
APPEND, //
@@ -175,7 +177,7 @@ public class PartitionDesc {
String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive);
}
- public static class DefaultPartitionConditionBuilder implements IPartitionConditionBuilder {
+ public static class DefaultPartitionConditionBuilder implements IPartitionConditionBuilder, java.io.Serializable {
@Override
public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
index 7089eba..0d9b442 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
@@ -18,25 +18,28 @@
package org.apache.kylin.metadata.model;
+import com.google.common.collect.Maps;
+
+import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
-import com.google.common.collect.Maps;
+public class TableRef implements Serializable {
-public class TableRef {
-
- final private DataModelDesc model;
+ final transient private DataModelDesc model;
final private String alias;
final private TableDesc table;
final private Map<String, TblColRef> columns;
+ final private String modelName;
TableRef(DataModelDesc model, String alias, TableDesc table) {
this.model = model;
+ this.modelName = model.getName();
this.alias = alias;
this.table = table;
this.columns = Maps.newLinkedHashMap();
-
+
for (ColumnDesc col : table.getColumns()) {
columns.put(col.getName(), new TblColRef(this, col));
}
@@ -94,7 +97,7 @@ public class TableRef {
TableRef t = (TableRef) o;
- if ((model == null ? t.model == null : model.getName().equals(t.model.getName())) == false)
+ if ((modelName == null ? t.modelName != null : modelName.equals(t.modelName)) == false)
return false;
if ((alias == null ? t.alias == null : alias.equals(t.alias)) == false)
return false;
@@ -107,7 +110,7 @@ public class TableRef {
@Override
public int hashCode() {
int result = 0;
- result = 31 * result + model.getName().hashCode();
+ result = 31 * result + modelName.hashCode();
result = 31 * result + alias.hashCode();
result = 31 * result + table.getIdentity().hashCode();
return result;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
index 40e5b29..3d48ac2 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -38,8 +38,6 @@ import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.measure.hllc.HLLCAggregator;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.github.jamm.MemoryMeter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -61,7 +59,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
}
private List<? extends MeasureAggregator> basicAggregators() {
- LongMutable longVal = new LongMutable(1000);
+ Long longVal = new Long(1000);
LongMinAggregator longMin = new LongMinAggregator();
LongMaxAggregator longMax = new LongMaxAggregator();
LongSumAggregator longSum = new LongSumAggregator();
@@ -69,7 +67,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
longMax.aggregate(longVal);
longSum.aggregate(longVal);
- DoubleMutable doubleVal = new DoubleMutable(1.0);
+ Double doubleVal = new Double(1.0);
DoubleMinAggregator doubleMin = new DoubleMinAggregator();
DoubleMaxAggregator doubleMax = new DoubleMaxAggregator();
DoubleSumAggregator doubleSum = new DoubleSumAggregator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
index dde0481..fcd0182 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -52,7 +52,6 @@ import org.apache.kylin.gridtable.GridTable;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
@@ -499,16 +498,16 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
GTRecord r = new GTRecord(table.getInfo());
GTBuilder builder = table.rebuild();
- builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5")));
builder.close();
return table;
@@ -524,34 +523,34 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
for (int i = 0; i < 100000; i++) {
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5")));
}
builder.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 966bb1b..f43d8d2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -32,6 +32,7 @@ import org.apache.kylin.engine.mr.steps.NDCuboidJob;
import org.apache.kylin.engine.mr.steps.SaveStatisticsStep;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,7 +77,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
return result;
}
- private void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
+ protected void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
RowKeyDesc rowKeyDesc = ((CubeSegment) seg).getCubeDesc().getRowkey();
final int groupRowkeyColumnsCount = ((CubeSegment) seg).getCubeDesc().getBuildLevel();
final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length;
@@ -100,7 +101,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
return result;
}
- private MapReduceExecutable createInMemCubingStep(String jobId, String cuboidRootPath) {
+ protected AbstractExecutable createInMemCubingStep(String jobId, String cuboidRootPath) {
// base cuboid job
MapReduceExecutable cubeStep = new MapReduceExecutable();
@@ -117,7 +118,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
cubeStep.setMapReduceParams(cmd.toString());
cubeStep.setMapReduceJobClass(getInMemCuboidJob());
- cubeStep.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES + "," + CubingJob.CUBE_SIZE_BYTES);
+// cubeStep.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES + "," + CubingJob.CUBE_SIZE_BYTES);
return cubeStep;
}
@@ -144,7 +145,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
baseCuboidStep.setMapReduceParams(cmd.toString());
baseCuboidStep.setMapReduceJobClass(getBaseCuboidJob());
- baseCuboidStep.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES);
+// baseCuboidStep.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES);
return baseCuboidStep;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 88692a0..60bf5d9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -36,7 +36,7 @@ import org.slf4j.LoggerFactory;
public class HadoopUtil {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(HadoopUtil.class);
- private static final ThreadLocal<Configuration> hadoopConfig = new ThreadLocal<>();
+ private static final transient ThreadLocal<Configuration> hadoopConfig = new ThreadLocal<>();
public static void setCurrentConfiguration(Configuration conf) {
hadoopConfig.set(conf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 5f5814b..47695b8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -75,6 +75,7 @@ public class JobBuilderSupport {
appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Fact_Distinct_Columns_" + seg.getRealization().getName() + "_Step");
appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
result.setMapReduceParams(cmd.toString());
+ result.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES);
return result;
}
@@ -192,7 +193,7 @@ public class JobBuilderSupport {
return buf.append(" -").append(paraName).append(" ").append(paraValue);
}
- public String[] getCuboidOutputPaths(String cuboidRootPath, int totalRowkeyColumnCount, int groupRowkeyColumnsCount) {
+ public static String[] getCuboidOutputPaths(String cuboidRootPath, int totalRowkeyColumnCount, int groupRowkeyColumnsCount) {
String[] paths = new String[groupRowkeyColumnsCount + 1];
for (int i = 0; i <= groupRowkeyColumnsCount; i++) {
int dimNum = totalRowkeyColumnCount - i;
@@ -205,4 +206,13 @@ public class JobBuilderSupport {
return paths;
}
+ public static String getCuboidOutputPathsByLevel(String cuboidRootPath, int level) {
+ if (level == 0) {
+ return cuboidRootPath + "base_cuboid";
+ } else {
+ return cuboidRootPath + level + "level_cuboid";
+ }
+ }
+
+
}
[25/42] kylin git commit: minor,cube measures add dimensions
Posted by sh...@apache.org.
minor,cube measures add dimensions
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73a7c25d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73a7c25d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73a7c25d
Branch: refs/heads/sparkcubing-rebase
Commit: 73a7c25dfeb28764658af3e73065aa04cc1b92cd
Parents: 76e321b
Author: chenzhx <34...@qq.com>
Authored: Wed Dec 28 16:18:01 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Wed Dec 28 16:31:30 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeMeasures.js | 11 ++++++++++-
webapp/app/partials/cubeDesigner/measures.html | 6 ++++--
2 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a7c25d/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 15c330a..c8548ce 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -84,6 +84,15 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
if(!!measure && measure.function.parameter.next_parameter){
$scope.nextPara.value = measure.function.parameter.next_parameter.value;
}
+ if($scope.newMeasure.function.parameter.value){
+ if($scope.metaModel.model.metrics&&$scope.metaModel.model.metrics.indexOf($scope.newMeasure.function.parameter.value)!=-1){
+ $scope.newMeasure.showDim=false;
+ }else{
+ $scope.newMeasure.showDim=true;
+ }
+ }else{
+ $scope.newMeasure.showDim=false;
+ }
if($scope.newMeasure.function.expression=="TOP_N"){
$scope.convertedColumns=[];
if($scope.newMeasure.function.configuration==null){
@@ -353,7 +362,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
if($scope.newMeasure.function.parameter.type=="column"&&$scope.newMeasure.function.expression!=="COUNT_DISTINCT"){
var column = $scope.newMeasure.function.parameter.value;
- if(column){
+ if(column&&(typeof column=="string")){
var colType = $scope.getColumnType(VdmUtil.removeNameSpace(column), VdmUtil.getNameSpaceAliasName(column)); // $scope.getColumnType defined in cubeEdit.js
}
if(colType==""||!colType){
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a7c25d/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 9eb04c3..4d2d8ab 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -150,14 +150,15 @@
<!--!COUNT_DISTINCT-->
<select class="form-control" chosen
- ng-if="newMeasure.function.parameter.type == 'column' && (newMeasure.function.expression == 'COUNT_DISTINCT'||newMeasure.function.expression == 'RAW')"
+ ng-if="newMeasure.function.parameter.type == 'column' && newMeasure.function.expression !== 'EXTENDED_COLUMN' && newMeasure.showDim==true "
ng-model="newMeasure.function.parameter.value" required
ng-change="measureReturnTypeUpdate();"
ng-options="column as column for column in getAllModelDimMeasureColumns(newMeasure)" >
<option value="">-- Select a Column --</option>
</select>
+
<select class="form-control" chosen
- ng-if="newMeasure.function.parameter.type == 'column' && newMeasure.function.expression !== 'EXTENDED_COLUMN' && newMeasure.function.expression !== 'COUNT_DISTINCT' && newMeasure.function.expression !== 'RAW'"
+ ng-if="newMeasure.function.parameter.type == 'column' && newMeasure.function.expression !== 'EXTENDED_COLUMN' && newMeasure.showDim==false "
ng-model="newMeasure.function.parameter.value" required
ng-change="measureReturnTypeUpdate();"
ng-options="column as column for column in getCommonMetricColumns(newMeasure)" >
@@ -171,6 +172,7 @@
<option value="">-- Select a Column --</option>
</select>
</div>
+ <label ng-if="newMeasure.function.parameter.type == 'column'&& newMeasure.function.expression !== 'EXTENDED_COLUMN'"><input type="checkbox" ng-model="newMeasure.showDim" /> Also Show Dimensions</label>
</div>
</div>
[14/42] kylin git commit: add a toggle for dumpping storage partitions
Posted by sh...@apache.org.
add a toggle for dumpping storage partitions
add more log
temp for dumpped partitions
minor fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bfe8b1a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bfe8b1a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bfe8b1a3
Branch: refs/heads/sparkcubing-rebase
Commit: bfe8b1a35c6b2c2f377eeb72295622ecb4d71284
Parents: 596ab51
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Dec 23 13:56:01 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:17:33 2016 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 30 ++++++++++++++++++++
.../gtrecord/StorageResponseGTScatter.java | 2 ++
2 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index ca4a19c..8109cf2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -62,6 +62,14 @@ public class BackdoorToggles {
return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
}
+ public static String getPartitionDumpDir() {
+ return getString(DEBUG_TOGGLE_PARTITION_DUMP_DIR);
+ }
+
+ public static String getDumpedPartitionDir() {
+ return getString(DEBUG_TOGGLE_DUMPED_PARTITION_DIR);
+ }
+
public static int getQueryTimeout() {
String v = getString(DEBUG_TOGGLE_QUERY_TIMEOUT);
if (v == null)
@@ -190,6 +198,28 @@ public class BackdoorToggles {
*/
public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = "DEBUG_TOGGLE_SHARD_ASSIGNMENT";
+ /**
+ * set DEBUG_TOGGLE_PARTITION_DUMP_DIR="dir" to dump the partitions from storage.
+ * The dumped partitions are used for performance profiling, for example.
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_PARTITION_DUMP_DIR": "/tmp/dumping"
+ }
+ */
+ public final static String DEBUG_TOGGLE_PARTITION_DUMP_DIR = "DEBUG_TOGGLE_PARTITION_DUMP_DIR";
+
+ /**
+ * set DEBUG_TOGGLE_DUMPED_PARTITION_DIR="dir" to specify the dir to retrieve previously dumped partitions
+ * it's a companion toggle with DEBUG_TOGGLE_PARTITION_DUMP_DIR
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DUMPED_PARTITION_DIR": "/tmp/dumped"
+ }
+ */
+ public final static String DEBUG_TOGGLE_DUMPED_PARTITION_DIR = "DEBUG_TOGGLE_DUMPED_PARTITION_DIR";
+
// properties on statement may go with this "channel" too
/**
* set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property
http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index 6283340..dc8746f 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -77,8 +77,10 @@ public class StorageResponseGTScatter implements IGTScanner {
public Iterator<GTRecord> iterator() {
Iterator<Iterator<GTRecord>> shardSubsets = Iterators.transform(blocks, new EndpointResponseGTScatterFunc());
if (storagePushDownLimit != Integer.MAX_VALUE) {
+ logger.info("Using SortedIteratorMergerWithLimit to merge partitions");
return new SortedIteratorMergerWithLimit<GTRecord>(shardSubsets, storagePushDownLimit, GTRecord.getPrimaryKeyComparator()).getIterator();
} else {
+ logger.info("Using Iterators.concat to merge partitions");
return Iterators.concat(shardSubsets);
}
}
[17/42] kylin git commit: KYLIN-2327 fix UT
Posted by sh...@apache.org.
KYLIN-2327 fix UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/84ec908b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/84ec908b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/84ec908b
Branch: refs/heads/sparkcubing-rebase
Commit: 84ec908b57cff0d3167cc35a9e4ea6d4b7bf94ce
Parents: 3e0edc8
Author: Billy Liu <bi...@apache.org>
Authored: Wed Dec 28 08:42:33 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Dec 28 08:42:33 2016 +0800
----------------------------------------------------------------------
.../src/test/java/org/apache/kylin/common/util/BasicTest.java | 1 -
kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java | 1 -
2 files changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/84ec908b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index bbaab09..5512147b 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,7 +34,6 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import com.google.common.collect.Iterators;
import org.apache.commons.lang3.time.FastDateFormat;
import org.junit.Ignore;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/84ec908b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 3ae7521..4f1f591 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -48,7 +48,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
[23/42] kylin git commit: minor,update model designer
Posted by sh...@apache.org.
minor,update model designer
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bc1a49ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bc1a49ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bc1a49ec
Branch: refs/heads/sparkcubing-rebase
Commit: bc1a49ec57987cf52d2596ac4d7a439b7596eb0d
Parents: 94acbc7
Author: chenzhx <34...@qq.com>
Authored: Wed Dec 28 11:11:08 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Wed Dec 28 15:15:36 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/modelDataModel.js | 5 ++++
webapp/app/js/controllers/modelSchema.js | 29 ++------------------
webapp/app/js/services/tree.js | 20 --------------
webapp/app/partials/cubes/cube_clone.html | 23 ++++------------
.../partials/modelDesigner/model_measures.html | 1 +
webapp/app/partials/models/model_clone.html | 10 ++++---
6 files changed, 20 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/js/controllers/modelDataModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelDataModel.js b/webapp/app/js/controllers/modelDataModel.js
index 95c42c4..07009a2 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -203,12 +203,17 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
modelsManager.selectedModel.dimensions.splice(i, 1);
}
}
+ delete $scope.aliasTableMap[lookup.alias];
lookupList.splice(lookupList.indexOf(lookup), 1);
+ $scope.aliasName.splice($scope.aliasName.indexOf(lookup.alias),1);
}
});
}else{
+ delete $scope.aliasTableMap[lookup.alias];
lookupList.splice(lookupList.indexOf(lookup), 1);
+ $scope.aliasName.splice($scope.aliasName.indexOf(lookup.alias),1);
}
+ console.log($scope.aliasName);
};
$scope.changeKey = function(index){
http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/js/controllers/modelSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelSchema.js b/webapp/app/js/controllers/modelSchema.js
index e309873..f917f1e 100644
--- a/webapp/app/js/controllers/modelSchema.js
+++ b/webapp/app/js/controllers/modelSchema.js
@@ -249,35 +249,10 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope, QueryService, UserServi
* 1.metric can't be null
*/
$scope.check_model_measure = function () {
-
- var errors = [];
- if (!modelsManager.selectedModel.metrics || !modelsManager.selectedModel.metrics.length) {
- errors.push("Please define your metrics.");
- }
- var errorInfo = "";
- angular.forEach(errors, function (item) {
- errorInfo += "\n" + item;
- });
- if (errors.length) {
- SweetAlert.swal('', errorInfo, 'warning');
- return false;
- } else {
- return true;
- }
-
+ return true;
};
$scope.check_model_setting = function () {
- var errors = [];
- var errorInfo = "";
- angular.forEach(errors, function (item) {
- errorInfo += "\n" + item;
- });
- if (errors.length) {
- SweetAlert.swal('', errorInfo, 'warning');
- return false;
- } else {
- return true;
- }
+ return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/js/services/tree.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/tree.js b/webapp/app/js/services/tree.js
index 20de19b..921aefe 100755
--- a/webapp/app/js/services/tree.js
+++ b/webapp/app/js/services/tree.js
@@ -72,27 +72,7 @@ KylinApp.service('ModelGraphService', function (VdmUtil) {
}
});
-/* angular.forEach(model.dimensions, function (dimension, index) {
- // for dimension on lookup table
- if(model.fact_table!==dimension.table){
- var lookup = _.find(graphData.children,function(item){
- return item.name === dimension.table;
- });
- angular.forEach(lookup.join.primary_key, function(pk, index){
- for (var i = 0; i < lookup._children.length; i++) {
- if(lookup._children[i].name == pk)
- break;
- }
- if(i == lookup._children.length) {
- lookup._children.push({
- "type": "column",
- "name": pk
- });
- }
- });
- };
- });*/
model.graph.columnsCount = 0;
model.graph.tree = tree;
model.graph.root = graphData;
http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/partials/cubes/cube_clone.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cube_clone.html b/webapp/app/partials/cubes/cube_clone.html
index f54bcd3..f5a332c 100644
--- a/webapp/app/partials/cubes/cube_clone.html
+++ b/webapp/app/partials/cubes/cube_clone.html
@@ -20,26 +20,13 @@
<div class="modal-header">
<h4 tooltip="submit">CUBE CLONE</h4>
</div>
- <div class="modal-body" style="background-color: white">
+ <ng-form name="forms.cube_clone_form" novalidate="novalidate" class="modal-body" style="background-color: white">
<div class="row">
<div class="col-md-2"></div>
<div class="col-md-8">
<div class="row">
<p class="text-info"> Cross project clone is not allowed now, cube will be cloned into current project.</p>
- <!--<div class="form-group">-->
- <!--<b>Target Project is:</b>-->
- <!--<br/>-->
- <!--<select ng-required="projectModel.projects.length" chosen ng-model="targetObj.targetProject"-->
- <!--ng-init="newAccess.permission=permissions.READ.value;"-->
- <!--ng-options="project.name as project.name for project in projectModel.projects "-->
- <!--ng-disabled="true"-->
- <!--style="width: 100% !important;"-->
- <!--data-placeholder="select a project"-->
- <!--class="chosen-select">-->
- <!--<option value=""></option>-->
- <!--</select>-->
- <!--</div>-->
</div>
</div>
<div class="col-md-2"></div>
@@ -51,15 +38,17 @@
<div class="form-group">
<b>New Cube Name:</b>
<br/>
- <input type="text" class="form-control" ng-model="targetObj.cubeName"/>
+ <input type="text" class="form-control" name="cube_name" ng-model="targetObj.cubeName" ng-pattern="/^\w+$/" required />
+ <small class="help-block red" ng-show="forms.cube_clone_form.cube_name.$error.required && (forms.cube_clone_form.cube_name.$dirty||forms.cube_clone_form.$submitted)">Cube name is required.</small>
+ <small class="help-block red" ng-show="!forms.cube_clone_form.cube_name.$error.required&&forms.cube_clone_form.cube_name.$invalid && (forms.cube_clone_form.cube_name.$dirty||forms.cube_clone_form.$submitted)">Cube name is invalid.</small>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
- </div>
+ </ng-form>
<div class="modal-footer">
- <button class="btn btn-success" ng-click="cloneCube()">Submit</button>
<button class="btn btn-primary" ng-click="cancel()">Close</button>
+ <button class="btn btn-success" ng-click="cloneCube()" ng-disabled="forms.cube_clone_form.$invalid">Submit</button>
</div>
</script>
http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/partials/modelDesigner/model_measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/model_measures.html b/webapp/app/partials/modelDesigner/model_measures.html
index 271c04f..1bb71b5 100644
--- a/webapp/app/partials/modelDesigner/model_measures.html
+++ b/webapp/app/partials/modelDesigner/model_measures.html
@@ -20,6 +20,7 @@
<div ng-controller="ModelMeasuresCtrl">
<ng-form name="forms.model_measure_form" novalidate>
<div class="dataTables_wrapper form-inline no-footer">
+ <b class="col-xs-6" ng-if="state.mode=='view'&&modelsManager.selectedModel.metrics.length==0">No Measures</b>
<table ng-if="state.mode=='view'&&modelsManager.selectedModel.metrics.length > 0" class="table table-striped table-hover">
<thead>
<tr>
http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/partials/models/model_clone.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/models/model_clone.html b/webapp/app/partials/models/model_clone.html
index de1b562..484821a 100644
--- a/webapp/app/partials/models/model_clone.html
+++ b/webapp/app/partials/models/model_clone.html
@@ -20,7 +20,7 @@
<div class="modal-header">
<h4 tooltip="submit">MODEL CLONE</h4>
</div>
- <div class="modal-body" style="background-color: white">
+ <ng-form name="forms.model_clone_form" novalidate="novalidate" class="modal-body" style="background-color: white">
<div class="row">
<div class="col-md-2"></div>
@@ -49,15 +49,17 @@
<div class="form-group">
<b>New Model Name:</b>
<br/>
- <input type="text" class="form-control" ng-model="targetObj.modelName"/>
+ <input type="text" name="model_name" class="form-control" ng-model="targetObj.modelName" ng-pattern="/^\w+$/" required />
+ <small class="help-block red" ng-show="forms.model_clone_form.model_name.$error.required && (forms.model_clone_form.model_name.$dirty||forms.model_clone_form.$submitted)">Model name is required.</small>
+ <small class="help-block red" ng-show="!forms.model_clone_form.model_name.$error.required&&forms.model_clone_form.model_name.$invalid && (forms.model_clone_form.model_name.$dirty||forms.model_clone_form.$submitted)">Model name is invalid.</small>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
- </div>
+ </ng-form>
<div class="modal-footer">
- <button class="btn btn-success" ng-click="cloneModel()">Submit</button>
<button class="btn btn-primary" ng-click="cancel()">Close</button>
+ <button class="btn btn-success" ng-click="cloneModel()" ng-disabled="forms.model_clone_form.$invalid">Submit</button>
</div>
</script>
[13/42] kylin git commit: KYLIN-2318 bug fix
Posted by sh...@apache.org.
KYLIN-2318 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1563eb94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1563eb94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1563eb94
Branch: refs/heads/sparkcubing-rebase
Commit: 1563eb945d0f3c68874934beca18579c96acc100
Parents: bfe8b1a
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Dec 27 12:47:08 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:17:33 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/debug/BackdoorToggles.java | 10 ++++++++++
.../java/org/apache/kylin/rest/service/QueryService.java | 1 +
2 files changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1563eb94/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 8109cf2..95d5d62 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -23,6 +23,8 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
+import com.google.common.collect.Maps;
+
/**
* BackdoorToggles and QueryContext are similar because they're both hosting per-query thread local variables.
* The difference is that BackdoorToggles are specified by user input and work for debug purpose. QueryContext
@@ -38,6 +40,14 @@ public class BackdoorToggles {
_backdoorToggles.set(toggles);
}
+ public static void addToggles(Map<String, String> toggles) {
+ Map<String, String> map = _backdoorToggles.get();
+ if (map == null) {
+ setToggles(Maps.<String, String> newHashMap());
+ }
+ _backdoorToggles.get().putAll(toggles);
+ }
+
public static String getCoprocessorBehavior() {
return getString(DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1563eb94/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index bb0342f..de28b17 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,6 +327,7 @@ public class QueryService extends BasicService {
}
final String queryId = UUID.randomUUID().toString();
+ BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
QueryContext.setQueryId(queryId);
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
[29/42] kylin git commit: KYLIN-2330 fix CubeDesc returning redundant
DeriveInfo
Posted by sh...@apache.org.
KYLIN-2330 fix CubeDesc returning redundant DeriveInfo
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a190ecf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a190ecf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a190ecf
Branch: refs/heads/sparkcubing-rebase
Commit: 1a190ecfcaad88e2551c5aff3bd4938719ba46a6
Parents: fdecf09
Author: Yang Li <li...@apache.org>
Authored: Thu Dec 29 07:13:12 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Thu Dec 29 07:31:17 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/QueryContext.java | 2 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 46 +++++++++++++++-----
.../org/apache/kylin/cube/CubeDescTest.java | 45 ++++++++++++++++++-
.../test_case_data/localmeta/cube_desc/ssb.json | 12 ++++-
4 files changed, 91 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index ef0cb14..93b8556 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -23,7 +23,7 @@ import java.util.Map;
import com.google.common.collect.Maps;
/**
- * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for comparision
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for comparison
*/
public class QueryContext {
private static final ThreadLocal<Map<String, String>> _queryContext = new ThreadLocal<Map<String, String>>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 3b8d034..4686a15 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -807,25 +807,49 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
for (int i = 0; i < derivedCols.length; i++) {
if (ArrayUtils.contains(hostCols, derivedCols[i])) {
derivedCols = (TblColRef[]) ArrayUtils.remove(derivedCols, i);
- extra = (String[]) ArrayUtils.remove(extra, i);
+ if (extra != null)
+ extra = (String[]) ArrayUtils.remove(extra, i);
i--;
}
}
+
+ if (derivedCols.length == 0)
+ return;
- Map<TblColRef, DeriveInfo> toHostMap = derivedToHostMap;
- Map<Array<TblColRef>, List<DeriveInfo>> hostToMap = hostToDerivedMap;
+ for (int i = 0; i < derivedCols.length; i++) {
+ TblColRef derivedCol = derivedCols[i];
+ boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1"));
+ derivedToHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne));
+ }
Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols);
- List<DeriveInfo> infoList = hostToMap.get(hostColArray);
+ List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray);
if (infoList == null) {
- hostToMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
+ hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
}
- infoList.add(new DeriveInfo(type, join, derivedCols, false));
-
- for (int i = 0; i < derivedCols.length; i++) {
- TblColRef derivedCol = derivedCols[i];
- boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1"));
- toHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne));
+
+ // Merged duplicated derived column
+ List<TblColRef> whatsLeft = new ArrayList<>();
+ for (TblColRef derCol : derivedCols) {
+ boolean merged = false;
+ for (DeriveInfo existing : infoList) {
+ if (existing.type == type && existing.join.getPKSide().equals(join.getPKSide())) {
+ if (ArrayUtils.contains(existing.columns, derCol)) {
+ merged = true;
+ break;
+ }
+ if (type == DeriveType.LOOKUP) {
+ existing.columns = (TblColRef[]) ArrayUtils.add(existing.columns, derCol);
+ merged = true;
+ break;
+ }
+ }
+ }
+ if (!merged)
+ whatsLeft.add(derCol);
+ }
+ if (whatsLeft.size() > 0) {
+ infoList.add(new DeriveInfo(type, join, (TblColRef[]) whatsLeft.toArray(new TblColRef[whatsLeft.size()]), false));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 0a7adf1..86ea1df 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -23,20 +23,28 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.io.File;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.model.AggregationGroup;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
+import org.apache.kylin.cube.model.CubeDesc.DeriveType;
import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.cube.model.SelectRule;
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -331,8 +339,43 @@ public class CubeDescTest extends LocalFileMetadataTestCase {
Map<?, ?> map2 = JsonUtil.readValue(mapStr, HashMap.class);
- assertEquals(map, map2);
+ Assert.assertEquals(map, map2);
+ }
+ @Test
+ public void testDerivedInfo() {
+ {
+ CubeDesc cube = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
+ List<TblColRef> givenCols = new ArrayList<>();
+ givenCols.add(cube.findColumnRef("TEST_KYLIN_FACT", "LSTG_SITE_ID"));
+ givenCols.add(cube.findColumnRef("TEST_KYLIN_FACT", "LEAF_CATEG_ID"));
+ Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cube.getHostToDerivedInfo(givenCols, null);
+ assertEquals(3, hostToDerivedInfo.size());
+ assertEquals(Pair.newPair(3, 2), countDerivedInfo(hostToDerivedInfo));
+ }
+
+ {
+ CubeDesc cube = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ssb");
+ List<TblColRef> givenCols = new ArrayList<>();
+ givenCols.add(cube.findColumnRef("V_LINEORDER", "LO_PARTKEY"));
+ Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cube.getHostToDerivedInfo(givenCols, null);
+ assertEquals(1, hostToDerivedInfo.size());
+ assertEquals(Pair.newPair(1, 1), countDerivedInfo(hostToDerivedInfo));
+ }
+ }
+
+ private Pair<Integer, Integer> countDerivedInfo(Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo) {
+ int pkfkCount = 0;
+ int lookupCount = 0;
+ for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
+ for (DeriveInfo deriveInfo : entry.getValue()) {
+ if (deriveInfo.type == DeriveType.PK_FK)
+ pkfkCount++;
+ if (deriveInfo.type == DeriveType.LOOKUP)
+ lookupCount++;
+ }
+ }
+ return Pair.newPair(pkfkCount, lookupCount);
}
private Collection<String> sortStrs(String[] strs) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/examples/test_case_data/localmeta/cube_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json
index d3ea10b..a13ac53 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb.json
@@ -6,7 +6,17 @@
"name" : "SSB.PART_DERIVED",
"table" : "SSB.PART",
"column" : null,
- "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+ "derived" : [ "P_MFGR" ]
+ }, {
+ "name" : "SSB.PART_DERIVED",
+ "table" : "SSB.PART",
+ "column" : null,
+ "derived" : [ "P_CATEGORY" ]
+ }, {
+ "name" : "SSB.PART_DERIVED",
+ "table" : "SSB.PART",
+ "column" : null,
+ "derived" : [ "P_BRAND" ]
}, {
"name" : "C_CITY",
"table" : "SSB.CUSTOMER",
[04/42] kylin git commit: KYLIN-2322 make TrieDictionaryForest
configurable, off by default
Posted by sh...@apache.org.
KYLIN-2322 make TrieDictionaryForest configurable, off by default
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a78b6509
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a78b6509
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a78b6509
Branch: refs/heads/sparkcubing-rebase
Commit: a78b6509f8727a53e26c57b2e5a14adc0ff88b73
Parents: 4f466a0
Author: Yang Li <li...@apache.org>
Authored: Mon Dec 26 21:12:20 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Dec 26 21:14:01 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 ++
.../apache/kylin/dict/DictionaryGenerator.java | 64 ++++++++++++++++++--
2 files changed, 63 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a78b6509/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index da93388..8080577 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -216,6 +216,10 @@ abstract public class KylinConfigBase implements Serializable {
// DICTIONARY & SNAPSHOT
// ============================================================================
+ public boolean isUseForestTrieDictionary() {
+ return Boolean.parseBoolean(getOptional("kylin.dictionary.use-forest-trie", "false"));
+ }
+
public int getTrieDictionaryForestMaxTrieSizeMB() {
return Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb", "500"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a78b6509/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index cd13d59..6f4f2c4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.datatype.DataType;
@@ -49,10 +50,12 @@ public class DictionaryGenerator {
builder = new DateDictBuilder();
else
builder = new TimeDictBuilder();
- } else if (dataType.isNumberFamily()) {
- builder = new NumberDictBuilder();
} else {
- builder = new StringDictBuilder();
+ boolean useForest = KylinConfig.getInstanceFromEnv().isUseForestTrieDictionary();
+ if (dataType.isNumberFamily())
+ builder = useForest ? new NumberTrieDictForestBuilder() : new NumberTrieDictBuilder();
+ else
+ builder = useForest ? new StringTrieDictForestBuilder() : new StringTrieDictBuilder();
}
return builder;
}
@@ -168,7 +171,32 @@ public class DictionaryGenerator {
}
}
- private static class StringDictBuilder implements IDictionaryBuilder {
+ private static class StringTrieDictBuilder implements IDictionaryBuilder {
+ int baseId;
+ TrieDictionaryBuilder builder;
+
+ @Override
+ public void init(DictionaryInfo info, int baseId) throws IOException {
+ this.baseId = baseId;
+ this.builder = new TrieDictionaryBuilder(new StringBytesConverter());
+ }
+
+ @Override
+ public boolean addValue(String value) {
+ if (value == null)
+ return false;
+
+ builder.addValue(value);
+ return true;
+ }
+
+ @Override
+ public Dictionary<String> build() throws IOException {
+ return builder.build(baseId);
+ }
+ }
+
+ private static class StringTrieDictForestBuilder implements IDictionaryBuilder {
TrieDictionaryForestBuilder builder;
@Override
@@ -191,7 +219,33 @@ public class DictionaryGenerator {
}
}
- private static class NumberDictBuilder implements IDictionaryBuilder {
+ @SuppressWarnings("deprecation")
+ private static class NumberTrieDictBuilder implements IDictionaryBuilder {
+ int baseId;
+ NumberDictionaryBuilder builder;
+
+ @Override
+ public void init(DictionaryInfo info, int baseId) throws IOException {
+ this.baseId = baseId;
+ this.builder = new NumberDictionaryBuilder(new StringBytesConverter());
+ }
+
+ @Override
+ public boolean addValue(String value) {
+ if (StringUtils.isBlank(value)) // empty string is treated as null
+ return false;
+
+ builder.addValue(value);
+ return true;
+ }
+
+ @Override
+ public Dictionary<String> build() throws IOException {
+ return builder.build(baseId);
+ }
+ }
+
+ private static class NumberTrieDictForestBuilder implements IDictionaryBuilder {
NumberDictionaryForestBuilder builder;
@Override
[26/42] kylin git commit: minor, fix NPE when filter Non-Cubing Job
Posted by sh...@apache.org.
minor, fix NPE when filter Non-Cubing Job
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fd745c88
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fd745c88
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fd745c88
Branch: refs/heads/sparkcubing-rebase
Commit: fd745c8804848a60ac96eff1bd9b30bbe38bdafc
Parents: 73a7c25
Author: Billy Liu <bi...@apache.org>
Authored: Wed Dec 28 18:47:58 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Dec 28 22:00:25 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/rest/service/JobService.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd745c88/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index ca8659c..4709a91 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -536,10 +536,13 @@ public class JobService extends BasicService implements InitializingBean {
@Override
public boolean apply(AbstractExecutable executable) {
if (executable instanceof CubingJob) {
- if (cubeName == null) {
+ if (StringUtils.isEmpty(cubeName)) {
return true;
}
- return CubingExecutableUtil.getCubeName(executable.getParams()).equalsIgnoreCase(cubeName);
+ String executableCubeName = CubingExecutableUtil.getCubeName(executable.getParams());
+ if (executableCubeName == null)
+ return true;
+ return executableCubeName.contains(cubeName);
} else {
return false;
}
[22/42] kylin git commit: minor,
refine rootfact select show width database
Posted by sh...@apache.org.
minor,refine rootfact select show width database
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/94acbc71
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/94acbc71
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/94acbc71
Branch: refs/heads/sparkcubing-rebase
Commit: 94acbc7130bb9084af9b5ac6def2527c5d238b15
Parents: 8747e24
Author: luguosheng <55...@qq.com>
Authored: Wed Dec 28 11:39:05 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Wed Dec 28 15:09:17 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/modelDesigner/data_model.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/94acbc71/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index 6e66e91..4d72266 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -27,7 +27,7 @@
</label>
<div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.data_model_form.table_name.$invalid && (forms.data_model_form.table_name.$dirty||forms.data_model_form.$submitted)}">
<select chosen ng-model="FactTable.root" ng-if="state.mode=='edit'"
- ng-options="table.name as VdmUtil.removeNameSpace(table.name) for table in tableModel.selectProjectTables"
+ ng-options="table.name as table.name for table in tableModel.selectProjectTables"
style="width:100%;" ng-change="changeFactTable()"
name="table_name" ng-disabled="cubesLength>0"
ng-required="true"
[32/42] kylin git commit: upgrade pom to 2.0.0
Posted by sh...@apache.org.
upgrade pom to 2.0.0
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8cd416a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8cd416a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8cd416a3
Branch: refs/heads/sparkcubing-rebase
Commit: 8cd416a3b18c43a63a8437a8a35728d2ba25076e
Parents: 13c3407
Author: Li Yang <li...@apache.org>
Authored: Thu Dec 29 17:03:23 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 29 17:03:23 2016 +0800
----------------------------------------------------------------------
assembly/pom.xml | 2 +-
atopcalcite/pom.xml | 2 +-
core-common/pom.xml | 2 +-
.../src/main/java/org/apache/kylin/common/KylinVersion.java | 2 +-
core-cube/pom.xml | 2 +-
core-dictionary/pom.xml | 2 +-
core-job/pom.xml | 2 +-
core-metadata/pom.xml | 2 +-
core-storage/pom.xml | 2 +-
engine-mr/pom.xml | 2 +-
engine-spark/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
kylin-it/pom.xml | 2 +-
pom.xml | 2 +-
query/pom.xml | 2 +-
server-base/pom.xml | 2 +-
server/pom.xml | 2 +-
source-hive/pom.xml | 2 +-
source-kafka/pom.xml | 2 +-
storage-hbase/pom.xml | 2 +-
tomcat-ext/pom.xml | 2 +-
tool/pom.xml | 2 +-
22 files changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 73f9e12..65e3a8c 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index c1dba66..b1bc171 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 1bc5836..2265651 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 622c6af..465ca5a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -69,7 +69,7 @@ public class KylinVersion {
/**
* Require MANUAL updating kylin version per ANY upgrading.
*/
- private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("1.6.1");
+ private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.0.0");
private static final Set<KylinVersion> SIGNATURE_INCOMPATIBLE_REVISIONS = new HashSet<KylinVersion>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index b013493..b7db363 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 636986f..a74eccb 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 22d61f3..dad0428 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 829c9fd..f5c8f8a 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 458a328..8987608 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/engine-mr/pom.xml
----------------------------------------------------------------------
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 4856feb..f31e650 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index a7cffdd..504a966 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 22d8eca..45c212a 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 86f2544..9bc1885 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 48c075f..0b323cf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -31,7 +31,7 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<packaging>pom</packaging>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<name>Apache Kylin</name>
<url>http://kylin.apache.org</url>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index c51812d..51f9370 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/server-base/pom.xml
----------------------------------------------------------------------
diff --git a/server-base/pom.xml b/server-base/pom.xml
index e4ebeb9..e75e683 100644
--- a/server-base/pom.xml
+++ b/server-base/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index cf92fb1..981d64c 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index ea87b73..c7a413e 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index 76734a8..d59c257 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 7724810..b0738f6 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/tomcat-ext/pom.xml
----------------------------------------------------------------------
diff --git a/tomcat-ext/pom.xml b/tomcat-ext/pom.xml
index 845f44e..5387eb6 100644
--- a/tomcat-ext/pom.xml
+++ b/tomcat-ext/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>kylin-tomcat-ext</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 8eddec4..213f05c 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
[06/42] kylin git commit: minor, more stable DeployUtil
Posted by sh...@apache.org.
minor, more stable DeployUtil
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74c2d6bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74c2d6bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74c2d6bc
Branch: refs/heads/sparkcubing-rebase
Commit: 74c2d6bcf0ea96939c63c3432961d666c77e609c
Parents: f9a8402
Author: Yang Li <li...@apache.org>
Authored: Tue Dec 27 07:02:32 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Tue Dec 27 07:02:32 2016 +0800
----------------------------------------------------------------------
assembly/src/test/java/org/apache/kylin/job/DeployUtil.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/74c2d6bc/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 089c3ed..a42b125 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -255,8 +255,9 @@ public class DeployUtil {
private static String[] generateCreateTableHql(TableDesc tableDesc) {
String dropsql = "DROP TABLE IF EXISTS " + tableDesc.getIdentity();
+ String dropsql2 = "DROP VIEW IF EXISTS " + tableDesc.getIdentity();
+
StringBuilder ddl = new StringBuilder();
-
ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n");
ddl.append("(" + "\n");
@@ -272,7 +273,7 @@ public class DeployUtil {
ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
ddl.append("STORED AS TEXTFILE");
- return new String[] { dropsql, ddl.toString() };
+ return new String[] { dropsql, dropsql2, ddl.toString() };
}
private static String[] generateCreateViewHql(String viewName, String tableName) {
[10/42] kylin git commit: strenthen storage visit deadline calculation
Posted by sh...@apache.org.
strenthen storage visit deadline calculation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/53b6c8c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/53b6c8c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/53b6c8c1
Branch: refs/heads/sparkcubing-rebase
Commit: 53b6c8c18033f92feadd2295456fa4f31011b5bb
Parents: 2436140
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:15:44 2016 +0800
----------------------------------------------------------------------
.../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/53b6c8c1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
- final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+ final long deadline = scanReq.getStartTime() + scanReq.getTimeout();
logger.info("deadline is " + deadline);
final long storagePushDownLimit = scanReq.getStoragePushDownLimit();
[27/42] kylin git commit: minor,
fix could not parse issue for scheduler
Posted by sh...@apache.org.
minor, fix could not parse issue for scheduler
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/08eda7b7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/08eda7b7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/08eda7b7
Branch: refs/heads/sparkcubing-rebase
Commit: 08eda7b714a30c665bdd777c296d15c7650c9ea9
Parents: fd745c8
Author: Billy Liu <bi...@apache.org>
Authored: Wed Dec 28 19:00:00 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Dec 28 22:01:39 2016 +0800
----------------------------------------------------------------------
.../kylin/job/impl/threadpool/DefaultScheduler.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/08eda7b7/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index be8e7fe..ddb3a21 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -105,15 +105,18 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
continue;
}
nReady++;
- AbstractExecutable executable = executableManager.getJob(id);
- String jobDesc = executable.toString();
- logger.info(jobDesc + " prepare to schedule");
+ AbstractExecutable executable = null;
+ String jobDesc = null;
try {
+ executable = executableManager.getJob(id);
+ jobDesc = executable.toString();
+ logger.info(jobDesc + " prepare to schedule");
context.addRunningJob(executable);
jobPool.execute(new JobRunner(executable));
logger.info(jobDesc + " scheduled");
} catch (Exception ex) {
- context.removeRunningJob(executable);
+ if (executable != null)
+ context.removeRunningJob(executable);
logger.warn(jobDesc + " fail to schedule", ex);
}
}
[36/42] kylin git commit: minor improvement on logging for
shard/reducer sizing
Posted by sh...@apache.org.
minor improvement on logging for shard/reducer sizing
temp
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/020cf5f0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/020cf5f0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/020cf5f0
Branch: refs/heads/sparkcubing-rebase
Commit: 020cf5f00ebeb3514493652629e408906ac1786e
Parents: e2e2a81
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Dec 29 14:13:36 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Dec 29 18:51:50 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/steps/CuboidJob.java | 2 +-
.../kylin/engine/mr/steps/CuboidReducer.java | 2 +-
.../engine/mr/steps/InMemCuboidReducer.java | 2 +-
.../engine/mr/steps/LayerReducerNumSizing.java | 84 ++++++++++++++++++++
.../engine/mr/steps/LayerReduerNumSizing.java | 82 -------------------
.../kylin/engine/mr/steps/MergeCuboidJob.java | 2 +-
6 files changed, 88 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/020cf5f0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index bd305c1..ef25b55 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -131,7 +131,7 @@ public class CuboidJob extends AbstractHadoopJob {
// add metadata to distributed cache
attachSegmentMetadataWithDict(segment, job.getConfiguration());
- LayerReduerNumSizing.setReduceTaskNum(job, segment, getTotalMapInputMB(), nCuboidLevel);
+ LayerReducerNumSizing.setReduceTaskNum(job, segment, getTotalMapInputMB(), nCuboidLevel);
this.deletePath(job.getConfiguration(), output);
http://git-wip-us.apache.org/repos/asf/kylin/blob/020cf5f0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index b1d4aaa..afd29e3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -91,7 +91,7 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text> {
for (Text value : values) {
if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
- logger.info("Handling value with ordinal: " + vcounter);
+ logger.info("Handling value with ordinal (This is not KV number!): " + vcounter);
}
codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), input);
if (cuboidLevel > 0) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/020cf5f0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
index 04c9e90..244889f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
@@ -80,7 +80,7 @@ public class InMemCuboidReducer extends KylinReducer<ByteArrayWritable, ByteArra
for (ByteArrayWritable value : values) {
if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
- logger.info("Handling value with ordinal: " + vcounter);
+ logger.info("Handling value with ordinal (This is not KV number!): " + vcounter);
}
codec.decode(value.asBuffer(), input);
aggs.aggregate(input);
http://git-wip-us.apache.org/repos/asf/kylin/blob/020cf5f0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
new file mode 100644
index 0000000..713a95c
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.engine.mr.common.CubeStatsReader;
+import org.apache.kylin.job.exception.JobException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LayerReducerNumSizing {
+
+ private static final Logger logger = LoggerFactory.getLogger(LayerReducerNumSizing.class);
+
+ public static void setReduceTaskNum(Job job, CubeSegment cubeSegment, double totalMapInputMB, int level) throws ClassNotFoundException, IOException, InterruptedException, JobException {
+ CubeDesc cubeDesc = cubeSegment.getCubeDesc();
+ KylinConfig kylinConfig = cubeDesc.getConfig();
+
+ double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB();
+ double reduceCountRatio = kylinConfig.getDefaultHadoopJobReducerCountRatio();
+ logger.info("Having per reduce MB " + perReduceInputMB + ", reduce count ratio " + reduceCountRatio + ", level " + level);
+
+ CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, kylinConfig);
+
+ double parentLayerSizeEst, currentLayerSizeEst, adjustedCurrentLayerSizeEst;
+
+ if (level == -1) {
+ //merge case
+ double estimatedSize = cubeStatsReader.estimateCubeSize();
+ adjustedCurrentLayerSizeEst = estimatedSize > totalMapInputMB ? totalMapInputMB : estimatedSize;
+ logger.info("estimated size {}, input size {}, adjustedCurrentLayerSizeEst: {}", estimatedSize, totalMapInputMB, adjustedCurrentLayerSizeEst);
+ } else if (level == 0) {
+ //base cuboid case TODO: the estimation could be very WRONG because it has no correction
+ adjustedCurrentLayerSizeEst = cubeStatsReader.estimateLayerSize(0);
+ logger.info("adjustedCurrentLayerSizeEst: {}", adjustedCurrentLayerSizeEst);
+ } else {
+ parentLayerSizeEst = cubeStatsReader.estimateLayerSize(level - 1);
+ currentLayerSizeEst = cubeStatsReader.estimateLayerSize(level);
+ adjustedCurrentLayerSizeEst = totalMapInputMB / parentLayerSizeEst * currentLayerSizeEst;
+ logger.info("totalMapInputMB: {}, parentLayerSizeEst: {}, currentLayerSizeEst: {}, adjustedCurrentLayerSizeEst: {}", totalMapInputMB, parentLayerSizeEst, currentLayerSizeEst, adjustedCurrentLayerSizeEst);
+ }
+
+ // number of reduce tasks
+ int numReduceTasks = (int) Math.round(adjustedCurrentLayerSizeEst / perReduceInputMB * reduceCountRatio + 0.99);
+
+ // adjust reducer number for cube which has DISTINCT_COUNT measures for better performance
+ if (cubeDesc.hasMemoryHungryMeasures()) {
+ logger.info("Multiply reducer num by 4 to boost performance for memory hungry measures");
+ numReduceTasks = numReduceTasks * 4;
+ }
+
+ // at least 1 reducer by default
+ numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks);
+ // no more than 500 reducer by default
+ numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
+
+ job.setNumReduceTasks(numReduceTasks);
+
+ logger.info("Setting " + Reducer.Context.NUM_REDUCES + "=" + numReduceTasks);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/020cf5f0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReduerNumSizing.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReduerNumSizing.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReduerNumSizing.java
deleted file mode 100644
index 6bddcbd..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReduerNumSizing.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.engine.mr.steps;
-
-import java.io.IOException;
-
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.engine.mr.common.CubeStatsReader;
-import org.apache.kylin.job.exception.JobException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LayerReduerNumSizing {
-
- private static final Logger logger = LoggerFactory.getLogger(LayerReduerNumSizing.class);
-
- public static void setReduceTaskNum(Job job, CubeSegment cubeSegment, double totalMapInputMB, int level) throws ClassNotFoundException, IOException, InterruptedException, JobException {
- CubeDesc cubeDesc = cubeSegment.getCubeDesc();
- KylinConfig kylinConfig = cubeDesc.getConfig();
-
- double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB();
- double reduceCountRatio = kylinConfig.getDefaultHadoopJobReducerCountRatio();
- logger.info("Having per reduce MB " + perReduceInputMB + ", reduce count ratio " + reduceCountRatio + ", level " + level);
-
- CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, kylinConfig);
-
- double parentLayerSizeEst, currentLayerSizeEst, adjustedCurrentLayerSizeEst;
-
- if (level == -1) {
- //merge case
- adjustedCurrentLayerSizeEst = cubeStatsReader.estimateCubeSize();
- logger.info("adjustedCurrentLayerSizeEst: {}", adjustedCurrentLayerSizeEst);
- } else if (level == 0) {
- //base cuboid case
- adjustedCurrentLayerSizeEst = cubeStatsReader.estimateLayerSize(0);
- logger.info("adjustedCurrentLayerSizeEst: {}", adjustedCurrentLayerSizeEst);
- } else {
- parentLayerSizeEst = cubeStatsReader.estimateLayerSize(level - 1);
- currentLayerSizeEst = cubeStatsReader.estimateLayerSize(level);
- adjustedCurrentLayerSizeEst = totalMapInputMB / parentLayerSizeEst * currentLayerSizeEst;
- logger.info("totalMapInputMB: {}, parentLayerSizeEst: {}, currentLayerSizeEst: {}, adjustedCurrentLayerSizeEst: {}", totalMapInputMB, parentLayerSizeEst, currentLayerSizeEst, adjustedCurrentLayerSizeEst);
- }
-
- // number of reduce tasks
- int numReduceTasks = (int) Math.round(adjustedCurrentLayerSizeEst / perReduceInputMB * reduceCountRatio);
-
- // adjust reducer number for cube which has DISTINCT_COUNT measures for better performance
- if (cubeDesc.hasMemoryHungryMeasures()) {
- numReduceTasks = numReduceTasks * 4;
- }
-
- // at least 1 reducer by default
- numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks);
- // no more than 500 reducer by default
- numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
-
- job.setNumReduceTasks(numReduceTasks);
-
- logger.info("Setting " + Reducer.Context.NUM_REDUCES + "=" + numReduceTasks);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/020cf5f0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
index 012e19f..d9ff616 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
@@ -82,7 +82,7 @@ public class MergeCuboidJob extends CuboidJob {
// TODO actually only dictionaries from merging segments are needed
attachCubeMetadataWithDict(cube, job.getConfiguration());
- LayerReduerNumSizing.setReduceTaskNum(job, cube.getSegmentById(segmentID), getTotalMapInputMB(), -1);
+ LayerReducerNumSizing.setReduceTaskNum(job, cube.getSegmentById(segmentID), getTotalMapInputMB(), -1);
this.deletePath(job.getConfiguration(), output);
[40/42] kylin git commit: KYLIN-2331 by layer spark cubing
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
new file mode 100644
index 0000000..07b636b
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import com.google.common.collect.Sets;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ */
+public class BaseCuboidBuilder implements java.io.Serializable {
+
+ protected static final Logger logger = LoggerFactory.getLogger(BaseCuboidBuilder.class);
+ public static final String HIVE_NULL = "\\N";
+ protected String cubeName;
+ protected Cuboid baseCuboid;
+ protected CubeDesc cubeDesc;
+ protected CubeSegment cubeSegment;
+ protected Set<String> nullStrs;
+ protected CubeJoinedFlatTableEnrich intermediateTableDesc;
+ protected MeasureIngester<?>[] aggrIngesters;
+ protected Map<TblColRef, Dictionary<String>> dictionaryMap;
+ protected AbstractRowKeyEncoder rowKeyEncoder;
+ protected BufferedMeasureCodec measureCodec;
+
+ protected KylinConfig kylinConfig;
+
+ public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc,
+ AbstractRowKeyEncoder rowKeyEncoder, MeasureIngester<?>[] aggrIngesters, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ this.kylinConfig = kylinConfig;
+ this.cubeDesc = cubeDesc;
+ this.cubeSegment = cubeSegment;
+ this.intermediateTableDesc = intermediateTableDesc;
+ this.rowKeyEncoder = rowKeyEncoder;
+ this.aggrIngesters = aggrIngesters;
+ this.dictionaryMap = dictionaryMap;
+
+ init();
+ measureCodec = new BufferedMeasureCodec(cubeDesc.getMeasures());
+ }
+
+ public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc) {
+ this.kylinConfig = kylinConfig;
+ this.cubeDesc = cubeDesc;
+ this.cubeSegment = cubeSegment;
+ this.intermediateTableDesc = intermediateTableDesc;
+
+ init();
+ rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);
+ measureCodec = new BufferedMeasureCodec(cubeDesc.getMeasures());
+ aggrIngesters = MeasureIngester.create(cubeDesc.getMeasures());
+ dictionaryMap = cubeSegment.buildDictionaryMap();
+
+ }
+
+ private void init() {
+ long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+ baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ initNullBytes();
+ }
+
+ private void initNullBytes() {
+ nullStrs = Sets.newHashSet();
+ nullStrs.add(HIVE_NULL);
+ String[] nullStrings = cubeDesc.getNullStrings();
+ if (nullStrings != null) {
+ for (String s : nullStrings) {
+ nullStrs.add(s);
+ }
+ }
+ }
+
+ protected boolean isNull(String v) {
+ return nullStrs.contains(v);
+ }
+
+ public byte[] buildKey(String[] flatRow) {
+ int[] rowKeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
+ List<TblColRef> columns = baseCuboid.getColumns();
+ String[] colValues = new String[columns.size()];
+ for (int i = 0; i < columns.size(); i++) {
+ colValues[i] = getCell(rowKeyColumnIndexes[i], flatRow);
+ }
+ return rowKeyEncoder.encode(colValues);
+ }
+
+ public ByteBuffer buildValue(String[] flatRow) {
+ return measureCodec.encode(buildValueObjects(flatRow));
+ }
+
+ public Object[] buildValueObjects(String[] flatRow) {
+ Object[] measures = new Object[cubeDesc.getMeasures().size()];
+ for (int i = 0; i < measures.length; i++) {
+ measures[i] = buildValueOf(i, flatRow);
+ }
+
+ return measures;
+ }
+
+ public void resetAggrs() {
+ for (int i = 0; i < cubeDesc.getMeasures().size(); i++) {
+ aggrIngesters[i].reset();
+ }
+ }
+
+ private Object buildValueOf(int idxOfMeasure, String[] flatRow) {
+ MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
+ FunctionDesc function = measure.getFunction();
+ int[] colIdxOnFlatTable = intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];
+
+ int paramCount = function.getParameterCount();
+ String[] inputToMeasure = new String[paramCount];
+
+ // pick up parameter values
+ ParameterDesc param = function.getParameter();
+ int colParamIdx = 0; // index among parameters of column type
+ for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
+ String value;
+ if (function.isCount()) {
+ value = "1";
+ } else if (param.isColumnType()) {
+ value = getCell(colIdxOnFlatTable[colParamIdx++], flatRow);
+ } else {
+ value = param.getValue();
+ }
+ inputToMeasure[i] = value;
+ }
+
+ return aggrIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
+ }
+
+ private String getCell(int i, String[] flatRow) {
+ if (isNull(flatRow[i]))
+ return null;
+ else
+ return flatRow[i];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index ffba181..4011915 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -262,6 +262,11 @@ public class CubeStatsReader {
return ret;
}
+ public List<Long> getCuboidsByLayer(int level) {
+ List<List<Long>> layeredCuboids = cuboidScheduler.getCuboidsByLayer();
+ return layeredCuboids.get(level);
+ }
+
private void printCuboidInfoTreeEntry(Map<Long, Long> cuboidRows, Map<Long, Double> cuboidSizes, PrintWriter out) {
long baseCuboid = Cuboid.getBaseCuboidId(seg.getCubeDesc());
int dimensionCount = Long.bitCount(baseCuboid);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
new file mode 100644
index 0000000..4e98618
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.SplittedBytes;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.common.RowKeySplitter;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.cube.kv.RowKeyEncoder;
+import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+
+/**
+ */
+public class NDCuboidBuilder implements Serializable {
+
+ protected static final Logger logger = LoggerFactory.getLogger(NDCuboidBuilder.class);
+ protected String cubeName;
+ protected String segmentID;
+ protected CubeSegment cubeSegment;
+ private RowKeySplitter rowKeySplitter;
+ private RowKeyEncoderProvider rowKeyEncoderProvider;
+ private byte[] newKeyBodyBuf = new byte[RowConstants.ROWKEY_BUFFER_SIZE];
+ private ByteArray newKeyBuf = ByteArray.allocate(RowConstants.ROWKEY_BUFFER_SIZE);
+
+ public NDCuboidBuilder(CubeSegment cubeSegment) {
+ this.cubeSegment = cubeSegment;
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ this.rowKeyEncoderProvider = new RowKeyEncoderProvider(cubeSegment);
+ }
+
+ public NDCuboidBuilder(CubeSegment cubeSegment, RowKeyEncoderProvider rowKeyEncoderProvider) {
+ this.cubeSegment = cubeSegment;
+ this.rowKeyEncoderProvider = rowKeyEncoderProvider;
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ }
+
+
+ public Pair<Integer, ByteArray> buildKey(Cuboid parentCuboid, Cuboid childCuboid, SplittedBytes[] splitBuffers) {
+ RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(childCuboid);
+
+ int offset = 0;
+
+ // rowkey columns
+ long mask = Long.highestOneBit(parentCuboid.getId());
+ long parentCuboidId = parentCuboid.getId();
+ long childCuboidId = childCuboid.getId();
+ long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(parentCuboid.getId());
+ int index = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId
+ for (int i = 0; i < parentCuboidIdActualLength; i++) {
+ if ((mask & parentCuboidId) > 0) {// if the this bit position equals
+ // 1
+ if ((mask & childCuboidId) > 0) {// if the child cuboid has this
+ // column
+ System.arraycopy(splitBuffers[index].value, 0, newKeyBodyBuf, offset, splitBuffers[index].length);
+ offset += splitBuffers[index].length;
+ }
+ index++;
+ }
+ mask = mask >> 1;
+ }
+
+ int fullKeySize = rowkeyEncoder.getBytesLength();
+ while (newKeyBuf.array().length < fullKeySize) {
+ newKeyBuf.set(new byte[newKeyBuf.length() * 2]);
+ }
+ newKeyBuf.set(0, fullKeySize);
+
+ rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf);
+
+ return new Pair<>(Integer.valueOf(fullKeySize), newKeyBuf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 7b719e0..d08e29a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -18,38 +18,25 @@
package org.apache.kylin.engine.mr.steps;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BaseCuboidBuilder;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
/**
*/
@@ -59,131 +46,37 @@ abstract public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
public static final byte[] ONE = Bytes.toBytes("1");
protected String cubeName;
protected String segmentID;
- protected Cuboid baseCuboid;
protected CubeInstance cube;
protected CubeDesc cubeDesc;
protected CubeSegment cubeSegment;
- protected Set<String> nullStrs;
- protected CubeJoinedFlatTableEnrich intermediateTableDesc;
- protected String intermediateTableRowDelimiter;
- protected byte byteRowDelimiter;
protected int counter;
- protected MeasureIngester<?>[] aggrIngesters;
- protected Map<TblColRef, Dictionary<String>> dictionaryMap;
protected Object[] measures;
- protected AbstractRowKeyEncoder rowKeyEncoder;
- protected BufferedMeasureCodec measureCodec;
private int errorRecordCounter;
protected Text outputKey = new Text();
protected Text outputValue = new Text();
+ private BaseCuboidBuilder baseCuboidBuilder;
+
@Override
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
-
cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
- intermediateTableRowDelimiter = context.getConfiguration().get(BatchConstants.CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER, Character.toString(BatchConstants.INTERMEDIATE_TABLE_ROW_DELIMITER));
- if (Bytes.toBytes(intermediateTableRowDelimiter).length > 1) {
- throw new RuntimeException("Expected delimiter byte length is 1, but got " + Bytes.toBytes(intermediateTableRowDelimiter).length);
- }
-
- byteRowDelimiter = Bytes.toBytes(intermediateTableRowDelimiter)[0];
-
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
-
- cube = CubeManager.getInstance(config).getCube(cubeName);
+ final KylinConfig kylinConfig = AbstractHadoopJob.loadKylinPropsAndMetadata();
+ cube = CubeManager.getInstance(kylinConfig).getCube(cubeName);
cubeDesc = cube.getDescriptor();
cubeSegment = cube.getSegmentById(segmentID);
+ CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
+ baseCuboidBuilder = new BaseCuboidBuilder(kylinConfig, cubeDesc, cubeSegment, intermediateTableDesc);
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
-
- intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
-
- rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);
-
- measureCodec = new BufferedMeasureCodec(cubeDesc.getMeasures());
- measures = new Object[cubeDesc.getMeasures().size()];
-
- aggrIngesters = MeasureIngester.create(cubeDesc.getMeasures());
- dictionaryMap = cubeSegment.buildDictionaryMap();
-
- initNullBytes();
- }
-
- private void initNullBytes() {
- nullStrs = Sets.newHashSet();
- nullStrs.add(HIVE_NULL);
- String[] nullStrings = cubeDesc.getNullStrings();
- if (nullStrings != null) {
- for (String s : nullStrings) {
- nullStrs.add(s);
- }
- }
- }
-
- protected boolean isNull(String v) {
- return nullStrs.contains(v);
- }
-
- protected byte[] buildKey(String[] flatRow) {
- int[] rowKeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
- List<TblColRef> columns = baseCuboid.getColumns();
- String[] colValues = new String[columns.size()];
- for (int i = 0; i < columns.size(); i++) {
- colValues[i] = getCell(rowKeyColumnIndexes[i], flatRow);
- }
- return rowKeyEncoder.encode(colValues);
}
- private ByteBuffer buildValue(String[] flatRow) {
-
- for (int i = 0; i < measures.length; i++) {
- measures[i] = buildValueOf(i, flatRow);
- }
-
- return measureCodec.encode(measures);
- }
-
- private Object buildValueOf(int idxOfMeasure, String[] flatRow) {
- MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
- FunctionDesc function = measure.getFunction();
- int[] colIdxOnFlatTable = intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];
-
- int paramCount = function.getParameterCount();
- String[] inputToMeasure = new String[paramCount];
-
- // pick up parameter values
- ParameterDesc param = function.getParameter();
- int colParamIdx = 0; // index among parameters of column type
- for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
- String value;
- if (function.isCount()) {
- value = "1";
- } else if (param.isColumnType()) {
- value = getCell(colIdxOnFlatTable[colParamIdx++], flatRow);
- } else {
- value = param.getValue();
- }
- inputToMeasure[i] = value;
- }
-
- return aggrIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
- }
-
- private String getCell(int i, String[] flatRow) {
- if (isNull(flatRow[i]))
- return null;
- else
- return flatRow[i];
- }
protected void outputKV(String[] flatRow, Context context) throws IOException, InterruptedException {
- byte[] rowKey = buildKey(flatRow);
+ byte[] rowKey = baseCuboidBuilder.buildKey(flatRow);
outputKey.set(rowKey, 0, rowKey.length);
- ByteBuffer valueBuf = buildValue(flatRow);
+ ByteBuffer valueBuf = baseCuboidBuilder.buildValue(flatRow);
outputValue.set(valueBuf.array(), 0, valueBuf.position());
context.write(outputKey, outputValue);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index 01cdd4a..b924edc 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -18,29 +18,27 @@
package org.apache.kylin.engine.mr.steps;
-import java.io.IOException;
-import java.util.Collection;
-
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.SplittedBytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.common.RowKeySplitter;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.kv.RowKeyEncoder;
-import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.NDCuboidBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.Collection;
+
/**
* @author George Song (ysong1)
*
@@ -59,10 +57,9 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
private int handleCounter;
private int skipCounter;
- private byte[] newKeyBodyBuf = new byte[RowConstants.ROWKEY_BUFFER_SIZE];
- private ByteArray newKeyBuf = ByteArray.allocate(RowConstants.ROWKEY_BUFFER_SIZE);
private RowKeySplitter rowKeySplitter;
- private RowKeyEncoderProvider rowKeyEncoderProvider;
+
+ private NDCuboidBuilder ndCuboidBuilder;
@Override
protected void setup(Context context) throws IOException {
@@ -76,48 +73,13 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
cubeSegment = cube.getSegmentById(segmentID);
cubeDesc = cube.getDescriptor();
-
+ ndCuboidBuilder = new NDCuboidBuilder(cubeSegment);
// initialize CubiodScheduler
cuboidScheduler = new CuboidScheduler(cubeDesc);
-
rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
- rowKeyEncoderProvider = new RowKeyEncoderProvider(cubeSegment);
}
- private int buildKey(Cuboid parentCuboid, Cuboid childCuboid, SplittedBytes[] splitBuffers) {
- RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(childCuboid);
-
- int offset = 0;
-
- // rowkey columns
- long mask = Long.highestOneBit(parentCuboid.getId());
- long parentCuboidId = parentCuboid.getId();
- long childCuboidId = childCuboid.getId();
- long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(parentCuboid.getId());
- int index = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId
- for (int i = 0; i < parentCuboidIdActualLength; i++) {
- if ((mask & parentCuboidId) > 0) {// if the this bit position equals
- // 1
- if ((mask & childCuboidId) > 0) {// if the child cuboid has this
- // column
- System.arraycopy(splitBuffers[index].value, 0, newKeyBodyBuf, offset, splitBuffers[index].length);
- offset += splitBuffers[index].length;
- }
- index++;
- }
- mask = mask >> 1;
- }
- int fullKeySize = rowkeyEncoder.getBytesLength();
- while (newKeyBuf.array().length < fullKeySize) {
- newKeyBuf.set(new byte[newKeyBuf.length() * 2]);
- }
- newKeyBuf.set(0, fullKeySize);
-
- rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf);
-
- return fullKeySize;
- }
@Override
public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException {
@@ -143,8 +105,8 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
for (Long child : myChildren) {
Cuboid childCuboid = Cuboid.findById(cubeDesc, child);
- int fullKeySize = buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
- outputKey.set(newKeyBuf.array(), 0, fullKeySize);
+ Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
+ outputKey.set(result.getSecond().array(), 0, result.getFirst());
context.write(outputKey, value);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
index 97dd750..29ca9b8 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
@@ -41,7 +41,6 @@ import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
@@ -161,7 +160,7 @@ public class CubeReducerTest extends LocalFileMetadataTestCase {
}
private Text newValueText(BufferedMeasureCodec codec, String sum, String min, String max, int count, int item_count) {
- Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongMutable(count), new LongMutable(item_count) };
+ Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new Long(count), new Long(item_count) };
ByteBuffer buf = codec.encode(values);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index 504a966..a931fac 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -47,6 +47,11 @@
<artifactId>kylin-core-job</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-engine-mr</artifactId>
+ </dependency>
+
<!-- Spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
@@ -115,6 +120,10 @@
<artifactId>maven-model</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine2.java
new file mode 100644
index 0000000..a7a4151
--- /dev/null
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine2.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.spark;
+
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.MRBatchCubingEngine2;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+
+/**
+ */
+public class SparkBatchCubingEngine2 extends MRBatchCubingEngine2 {
+ @Override
+ public DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) {
+ return new SparkBatchCubingJobBuilder2(newSegment, submitter).build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
new file mode 100644
index 0000000..9532d31
--- /dev/null
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.spark;
+
+import org.apache.hadoop.util.ClassUtil;
+import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.EngineFactory;
+import org.apache.kylin.engine.mr.BatchCubingJobBuilder2;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
+
+ private static final Logger logger = LoggerFactory.getLogger(SparkBatchCubingJobBuilder2.class);
+
+ public SparkBatchCubingJobBuilder2(CubeSegment newSegment, String submitter) {
+ super(newSegment, submitter);
+ }
+
+ protected void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
+
+ }
+
+ @Override
+ protected AbstractExecutable createInMemCubingStep(String jobId, String cuboidRootPath) {
+ IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(seg);
+ final SparkExecutable sparkExecutable = new SparkExecutable();
+ sparkExecutable.setClassName(SparkCubingByLayer.class.getName());
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_CUBE_NAME.getOpt(), seg.getRealization().getName());
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_SEGMENT_ID.getOpt(), seg.getUuid());
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_PATH.getOpt(), flatTableDesc.getTableName());
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_CONF_PATH.getOpt(), "/Users/shishaofeng/workspace/kylin-15/examples/test_case_data/sandbox/"); //FIXME
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_OUTPUT_PATH.getOpt(), cuboidRootPath);
+
+ StringBuilder jars = new StringBuilder();
+
+ StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration")); // htrace-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration"));
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.client.HConnection")); // hbase-client.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.HBaseConfiguration")); // hbase-common.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.util.ByteStringer")); // hbase-protocol.jar
+
+ StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars());
+ sparkExecutable.setJars(jars.toString());
+ // sparkExecutable.setJars("/Users/shishaofeng/.m2/repository/org/cloudera/htrace/htrace-core/2.01/htrace-core-2.01.jar,/Users/shishaofeng/.m2/repository/org/apache/hbase/hbase-protocol/0.98.8-hadoop2/hbase-protocol-0.98.8-hadoop2.jar,/Users/shishaofeng/.m2/repository/org/apache/hbase/hbase-common/0.98.8-hadoop2/hbase-common-0.98.8-hadoop2.jar,/Users/shishaofeng/.m2//repository/org/apache/hbase/hbase-client/0.98.8-hadoop2/hbase-client-0.98.8-hadoop2.jar");
+
+ sparkExecutable.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE + " with Spark");
+ return sparkExecutable;
+
+ }
+
+ private String findJar(String className) {
+ try {
+ return ClassUtil.findContainingJar(Class.forName(className));
+ } catch (ClassNotFoundException e) {
+ logger.error("failed to locate jar for class " + className, e);
+ }
+
+ return "";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 6e894dd..f06c338 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -84,6 +84,7 @@ import org.apache.kylin.engine.spark.util.IteratorUtils;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.MeasureAggregators;
import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -153,6 +154,20 @@ public class SparkCubing extends AbstractApplication {
return options;
}
+ public static KylinConfig loadKylinPropsAndMetadata(String folder) throws IOException {
+ File metaDir = new File(folder);
+ if (!metaDir.getAbsolutePath().equals(System.getProperty(KylinConfig.KYLIN_CONF))) {
+ System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath());
+ logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ System.out.println("setting metadataUrl to " + metaDir.getAbsolutePath());
+ kylinConfig.setMetadataUrl(metaDir.getAbsolutePath());
+ return kylinConfig;
+ } else {
+ return KylinConfig.getInstanceFromEnv();
+ }
+ }
+
private void setupClasspath(JavaSparkContext sc, String confPath) throws Exception {
ClassUtil.addClasspath(confPath);
final File[] files = new File(confPath).listFiles(new FileFilter() {
@@ -462,7 +477,7 @@ public class SparkCubing extends AbstractApplication {
}).saveAsNewAPIHadoopFile(hFileLocation, ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat.class, conf);
}
- private static void prepare() throws Exception {
+ public static void prepare() throws Exception {
final File file = new File(SparkFiles.get("kylin.properties"));
final String confPath = file.getParentFile().getAbsolutePath();
System.out.println("conf directory:" + confPath);
@@ -526,12 +541,18 @@ public class SparkCubing extends AbstractApplication {
}
}
- private Collection<String> getKyroClasses() {
+ public static Collection<String> getKyroClasses() {
Set<Class> kyroClasses = Sets.newHashSet();
kyroClasses.addAll(new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.dimension").getSubTypesOf(Serializable.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.cube").getSubTypesOf(Serializable.class));
kyroClasses.addAll(new Reflections("org.apache.kylin.cube.model").getSubTypesOf(Object.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.metadata").getSubTypesOf(Object.class));
kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.model").getSubTypesOf(Object.class));
kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.measure").getSubTypesOf(Object.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(org.apache.kylin.common.util.BytesSerializer.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class));
+
kyroClasses.add(HashMap.class);
kyroClasses.add(org.apache.spark.sql.Row[].class);
kyroClasses.add(org.apache.spark.sql.Row.class);
@@ -541,11 +562,15 @@ public class SparkCubing extends AbstractApplication {
kyroClasses.add(org.apache.spark.sql.types.StructField.class);
kyroClasses.add(org.apache.spark.sql.types.DateType$.class);
kyroClasses.add(org.apache.spark.sql.types.Metadata.class);
- kyroClasses.add(Object[].class);
kyroClasses.add(org.apache.spark.sql.types.StringType$.class);
kyroClasses.add(Hashing.murmur3_128().getClass());
- kyroClasses.add(org.apache.spark.sql.columnar.CachedBatch.class);
+ kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class);
+ kyroClasses.add(Object[].class);
+ kyroClasses.add(int[].class);
+ kyroClasses.add(byte[].class);
kyroClasses.add(byte[][].class);
+ kyroClasses.add(String[].class);
+ kyroClasses.add(String[][].class);
kyroClasses.add(org.apache.spark.sql.types.Decimal.class);
kyroClasses.add(scala.math.BigDecimal.class);
kyroClasses.add(java.math.BigDecimal.class);
@@ -553,6 +578,61 @@ public class SparkCubing extends AbstractApplication {
kyroClasses.add(java.math.RoundingMode.class);
kyroClasses.add(java.util.ArrayList.class);
kyroClasses.add(java.util.LinkedList.class);
+ kyroClasses.add(java.util.HashSet.class);
+ kyroClasses.add(java.util.LinkedHashSet.class);
+ kyroClasses.add(java.util.LinkedHashMap.class);
+ kyroClasses.add(java.util.TreeMap.class);
+ kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class);
+
+ kyroClasses.add(java.util.HashMap.class);
+ kyroClasses.add(java.util.Properties.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class);
+ kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc[].class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TblColRef[].class);
+ kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class);
+ kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc[].class);
+ kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc[].class);
+ kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class);
+ kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc[].class);
+ kyroClasses.add(org.apache.kylin.common.util.Array.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.Segments.class);
+ kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class);
+ kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class);
+ kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class);
+ kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class);
+ kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class);
+ kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.DateDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.Counter.class);
+
+ try {
+ kyroClasses.add(Class.forName("com.google.common.collect.EmptyImmutableList"));
+ } catch (ClassNotFoundException e) {
+ logger.error("failed to load class", e);
+ }
ArrayList<String> result = Lists.newArrayList();
for (Class kyroClass : kyroClasses) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
new file mode 100644
index 0000000..53c1f96
--- /dev/null
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.engine.spark;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.common.RowKeySplitter;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
+import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.engine.EngineFactory;
+import org.apache.kylin.engine.mr.BatchCubingJobBuilder2;
+import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.engine.mr.common.BaseCuboidBuilder;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.CubeStatsReader;
+import org.apache.kylin.engine.mr.common.NDCuboidBuilder;
+import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.measure.MeasureAggregators;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkFiles;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.function.Function2;
+import org.apache.spark.api.java.function.PairFlatMapFunction;
+import org.apache.spark.api.java.function.PairFunction;
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.hive.HiveContext;
+import org.apache.spark.storage.StorageLevel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Tuple2;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.kylin.engine.spark.SparkCubing.getKyroClasses;
+
+/**
+ */
+public class SparkCubingByLayer extends AbstractApplication implements Serializable {
+
+ protected static final Logger logger = LoggerFactory.getLogger(SparkCubingByLayer.class);
+
+ public static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Hive Intermediate Table").create("hiveTable");
+ public static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube Name").create(BatchConstants.ARG_CUBE_NAME);
+ public static final Option OPTION_SEGMENT_ID = OptionBuilder.withArgName("segment").hasArg().isRequired(true).withDescription("Cube Segment Id").create("segmentId");
+ public static final Option OPTION_CONF_PATH = OptionBuilder.withArgName("confPath").hasArg().isRequired(true).withDescription("Configuration Path").create("confPath");
+ public static final Option OPTION_OUTPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_OUTPUT).hasArg().isRequired(true).withDescription("Cube output path").create(BatchConstants.ARG_OUTPUT);
+
+ private Options options;
+
+ public SparkCubingByLayer() {
+ options = new Options();
+ options.addOption(OPTION_INPUT_PATH);
+ options.addOption(OPTION_CUBE_NAME);
+ options.addOption(OPTION_SEGMENT_ID);
+ options.addOption(OPTION_CONF_PATH);
+ options.addOption(OPTION_OUTPUT_PATH);
+ }
+
+ @Override
+ protected Options getOptions() {
+ return options;
+ }
+
+ private void setupClasspath(JavaSparkContext sc, String confPath) throws Exception {
+ ClassUtil.addClasspath(confPath);
+ final File[] files = new File(confPath).listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ if (pathname.getAbsolutePath().endsWith(".xml")) {
+ return true;
+ }
+ if (pathname.getAbsolutePath().endsWith(".properties")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ for (File file : files) {
+ sc.addFile(file.getAbsolutePath());
+ }
+ }
+
+ private static final void prepare() {
+ final File file = new File(SparkFiles.get("kylin.properties"));
+ final String confPath = file.getParentFile().getAbsolutePath();
+ logger.info("conf directory:" + confPath);
+ System.setProperty(KylinConfig.KYLIN_CONF, confPath);
+ ClassUtil.addClasspath(confPath);
+ }
+
+ @Override
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+ final String hiveTable = optionsHelper.getOptionValue(OPTION_INPUT_PATH);
+ final String cubeName = optionsHelper.getOptionValue(OPTION_CUBE_NAME);
+ final String segmentId = optionsHelper.getOptionValue(OPTION_SEGMENT_ID);
+ final String confPath = optionsHelper.getOptionValue(OPTION_CONF_PATH);
+ final String outputPath = optionsHelper.getOptionValue(OPTION_OUTPUT_PATH);
+
+ SparkConf conf = new SparkConf().setAppName("Cubing Application");
+ //serialization conf
+ conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ conf.set("spark.kryo.registrationRequired", "true");
+ final Iterable<String> allClasses = Iterables.filter(Iterables.concat(Lists.newArrayList(conf.get("spark.kryo.classesToRegister", "").split(",")), getKyroClasses()), new Predicate<String>() {
+ @Override
+ public boolean apply(@Nullable String input) {
+ return input != null && input.trim().length() > 0;
+ }
+ });
+ conf.set("spark.kryo.classesToRegister", StringUtils.join(allClasses, ","));
+
+ JavaSparkContext sc = new JavaSparkContext(conf);
+ setupClasspath(sc, confPath);
+ HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
+
+ HiveContext sqlContext = new HiveContext(sc.sc());
+ final DataFrame intermediateTable = sqlContext.table(hiveTable);
+
+ System.setProperty(KylinConfig.KYLIN_CONF, confPath);
+ final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
+ final CubeInstance cubeInstance = CubeManager.getInstance(envConfig).getCube(cubeName);
+ final CubeDesc cubeDesc = cubeInstance.getDescriptor();
+ final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
+ final CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
+
+ final KylinConfig kylinConfig = cubeDesc.getConfig();
+ final Broadcast<CubeDesc> vCubeDesc = sc.broadcast(cubeDesc);
+ final Broadcast<CubeSegment> vCubeSegment = sc.broadcast(cubeSegment);
+ final NDCuboidBuilder ndCuboidBuilder = new NDCuboidBuilder(vCubeSegment.getValue(), new RowKeyEncoderProvider(vCubeSegment.getValue()));
+
+ final Broadcast<CuboidScheduler> vCuboidScheduler = sc.broadcast(new CuboidScheduler(vCubeDesc.getValue()));
+
+ final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+ final Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ final int measureNum = cubeDesc.getMeasures().size();
+ final BaseCuboidBuilder baseCuboidBuilder = new BaseCuboidBuilder(kylinConfig, vCubeDesc.getValue(), vCubeSegment.getValue(), intermediateTableDesc, AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid), MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
+
+ int countMeasureIndex = 0;
+ for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
+ if (measureDesc.getFunction().isCount() == true) {
+ break;
+ } else {
+ countMeasureIndex++;
+ }
+ }
+ final CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, kylinConfig);
+ boolean[] needAggr = new boolean[cubeDesc.getMeasures().size()];
+ boolean allNormalMeasure = true;
+ for (int i = 0; i < cubeDesc.getMeasures().size(); i++) {
+ needAggr[i] = !cubeDesc.getMeasures().get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid();
+ allNormalMeasure = allNormalMeasure && needAggr[i];
+ }
+ logger.info("All measure are normal (agg on all cuboids) ? : " + allNormalMeasure);
+
+ StorageLevel storageLevel = StorageLevel.MEMORY_AND_DISK_SER();
+
+ // encode with dimension encoding, transform to <ByteArray, Object[]> RDD
+ final JavaPairRDD<ByteArray, Object[]> encodedBaseRDD = intermediateTable.javaRDD().mapToPair(new PairFunction<Row, ByteArray, Object[]>() {
+ transient boolean initialized = false;
+
+ @Override
+ public Tuple2<ByteArray, Object[]> call(Row row) throws Exception {
+ if (initialized == false) {
+ prepare();
+ initialized = true;
+ }
+
+ String[] rowArray = rowToArray(row);
+ baseCuboidBuilder.resetAggrs();
+ byte[] rowKey = baseCuboidBuilder.buildKey(rowArray);
+ Object[] result = baseCuboidBuilder.buildValueObjects(rowArray);
+ return new Tuple2<>(new ByteArray(rowKey), result);
+ }
+
+ private String[] rowToArray(Row row) {
+ String[] result = new String[row.size()];
+ for (int i = 0; i < row.size(); i++) {
+ final Object o = row.get(i);
+ if (o != null) {
+ result[i] = o.toString();
+ } else {
+ result[i] = null;
+ }
+ }
+ return result;
+ }
+
+ });
+
+ logger.info("encodedBaseRDD partition number: " + encodedBaseRDD.getNumPartitions());
+ Long totalCount = 0L;
+ if (kylinConfig.isSparkSanityCheckEnabled()) {
+ totalCount = encodedBaseRDD.count();
+ logger.info("encodedBaseRDD row count: " + encodedBaseRDD.count());
+ }
+
+ final MeasureAggregators measureAggregators = new MeasureAggregators(cubeDesc.getMeasures());
+ final BaseCuboidReducerFunction2 baseCuboidReducerFunction = new BaseCuboidReducerFunction2(measureNum, vCubeDesc.getValue(), measureAggregators);
+ BaseCuboidReducerFunction2 reducerFunction2 = baseCuboidReducerFunction;
+ if (allNormalMeasure == false) {
+ reducerFunction2 = new CuboidReducerFunction2(measureNum, vCubeDesc.getValue(), measureAggregators, needAggr);
+ }
+
+ final int totalLevels = cubeDesc.getBuildLevel();
+ JavaPairRDD<ByteArray, Object[]>[] allRDDs = new JavaPairRDD[totalLevels];
+ int level = 0;
+ int partition = estimateRDDPartitionNum(level, cubeStatsReader, kylinConfig);
+
+ // aggregate to calculate base cuboid
+ allRDDs[0] = encodedBaseRDD.reduceByKey(baseCuboidReducerFunction, partition).persist(storageLevel);
+ saveToHDFS(allRDDs[0], vCubeDesc.getValue(), outputPath, 0, sc.hadoopConfiguration());
+
+ // aggregate to ND cuboids
+ PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> flatMapFunction = new CuboidFlatMap(vCubeSegment.getValue(), vCubeDesc.getValue(), vCuboidScheduler.getValue(), ndCuboidBuilder);
+
+ for (level = 1; level < totalLevels; level++) {
+ partition = estimateRDDPartitionNum(level, cubeStatsReader, kylinConfig);
+ logger.info("Level " + level + " partition number: " + partition);
+ allRDDs[level] = allRDDs[level - 1].flatMapToPair(flatMapFunction).reduceByKey(reducerFunction2, partition).persist(storageLevel);
+ if (kylinConfig.isSparkSanityCheckEnabled() == true) {
+ sanityCheck(allRDDs[level], totalCount, level, cubeStatsReader, countMeasureIndex);
+ }
+ saveToHDFS(allRDDs[level], vCubeDesc.getValue(), outputPath, level, sc.hadoopConfiguration());
+ allRDDs[level - 1].unpersist();
+ }
+ allRDDs[totalLevels - 1].unpersist();
+ logger.info("Finished on calculating all level cuboids.");
+ }
+
+ private static int estimateRDDPartitionNum(int level, CubeStatsReader statsReader, KylinConfig kylinConfig) {
+ double baseCuboidSize = statsReader.estimateLayerSize(level);
+ float rddCut = kylinConfig.getSparkRDDPartitionCutMB();
+ int partition = (int) (baseCuboidSize / rddCut);
+ partition = Math.max(kylinConfig.getSparkMinPartition(), partition);
+ partition = Math.min(kylinConfig.getSparkMaxPartition(), partition);
+ logger.debug("Estimated level " + level + " partition number: " + partition);
+ return partition;
+ }
+
+ private static void saveToHDFS(final JavaPairRDD<ByteArray, Object[]> rdd, final CubeDesc cubeDesc, final String hdfsBaseLocation, int level, Configuration conf) {
+ final String cuboidOutputPath = BatchCubingJobBuilder2.getCuboidOutputPathsByLevel(hdfsBaseLocation, level);
+ rdd.mapToPair(new PairFunction<Tuple2<ByteArray, Object[]>, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text>() {
+ BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());
+ @Override
+ public Tuple2<org.apache.hadoop.io.Text, org.apache.hadoop.io.Text> call(Tuple2<ByteArray, Object[]> tuple2) throws Exception {
+ ByteBuffer valueBuf = codec.encode(tuple2._2());
+ byte[] encodedBytes = new byte[valueBuf.position()];
+ System.arraycopy(valueBuf.array(), 0, encodedBytes, 0, valueBuf.position());
+ return new Tuple2<>(new org.apache.hadoop.io.Text(tuple2._1().array()), new org.apache.hadoop.io.Text(encodedBytes));
+ }
+ }).saveAsNewAPIHadoopFile(cuboidOutputPath, org.apache.hadoop.io.Text.class, org.apache.hadoop.io.Text.class, SequenceFileOutputFormat.class, conf);
+ logger.info("Persisting RDD for level " + level + " into " + cuboidOutputPath);
+ }
+
+ class BaseCuboidReducerFunction2 implements Function2<Object[], Object[], Object[]> {
+ CubeDesc cubeDesc;
+ int measureNum;
+ MeasureAggregators aggregators;
+
+ BaseCuboidReducerFunction2(int measureNum, CubeDesc cubeDesc, MeasureAggregators aggregators) {
+ this.cubeDesc = cubeDesc;
+ this.measureNum = measureNum;
+ this.aggregators = aggregators;
+ }
+
+ @Override
+ public Object[] call(Object[] input1, Object[] input2) throws Exception {
+ Object[] result = new Object[measureNum];
+ aggregators.aggregate(input1, input2, result);
+ return result;
+ }
+ }
+
+ class CuboidReducerFunction2 extends BaseCuboidReducerFunction2 {
+ boolean[] needAggr;
+
+ CuboidReducerFunction2(int measureNum, CubeDesc cubeDesc, MeasureAggregators aggregators, boolean[] needAggr) {
+ super(measureNum, cubeDesc, aggregators);
+ this.needAggr = needAggr;
+ }
+
+ @Override
+ public Object[] call(Object[] input1, Object[] input2) throws Exception {
+ Object[] result = new Object[measureNum];
+ aggregators.aggregate(input1, input2, result, needAggr);
+ return result;
+ }
+ }
+
+ private static final java.lang.Iterable<Tuple2<ByteArray, Object[]>> EMTPY_ITERATOR = Lists.newArrayListWithCapacity(0);
+
+ class CuboidFlatMap implements PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> {
+
+ CubeSegment cubeSegment;
+ CubeDesc cubeDesc;
+ CuboidScheduler cuboidScheduler;
+ NDCuboidBuilder ndCuboidBuilder;
+ RowKeySplitter rowKeySplitter;
+ transient boolean initialized = false;
+
+ CuboidFlatMap(CubeSegment cubeSegment, CubeDesc cubeDesc, CuboidScheduler cuboidScheduler, NDCuboidBuilder ndCuboidBuilder) {
+ this.cubeSegment = cubeSegment;
+ this.cubeDesc = cubeDesc;
+ this.cuboidScheduler = cuboidScheduler;
+ this.ndCuboidBuilder = ndCuboidBuilder;
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ }
+
+ @Override
+ public Iterable<Tuple2<ByteArray, Object[]>> call(Tuple2<ByteArray, Object[]> tuple2) throws Exception {
+ if (initialized == false) {
+ prepare();
+ initialized = true;
+ }
+
+ byte[] key = tuple2._1().array();
+ long cuboidId = rowKeySplitter.split(key);
+ Cuboid parentCuboid = Cuboid.findById(cubeDesc, cuboidId);
+
+ Collection<Long> myChildren = cuboidScheduler.getSpanningCuboid(cuboidId);
+
+ // if still empty or null
+ if (myChildren == null || myChildren.size() == 0) {
+ return EMTPY_ITERATOR;
+ }
+
+ List<Tuple2<ByteArray, Object[]>> tuples = Lists.newArrayListWithCapacity(myChildren.size());
+ for (Long child : myChildren) {
+ Cuboid childCuboid = Cuboid.findById(cubeDesc, child);
+ Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
+
+ byte[] newKey = new byte[result.getFirst()];
+ System.arraycopy(result.getSecond().array(), 0, newKey, 0, result.getFirst());
+
+ tuples.add(new Tuple2<>(new ByteArray(newKey), tuple2._2()));
+ }
+
+ return tuples;
+ }
+ }
+
+ //sanity check
+
+ private void sanityCheck(JavaPairRDD<ByteArray, Object[]> rdd, Long totalCount, int thisLevel, CubeStatsReader cubeStatsReader, final int countMeasureIndex) {
+ int thisCuboidNum = cubeStatsReader.getCuboidsByLayer(thisLevel).size();
+ Long count2 = getRDDCountSum(rdd, countMeasureIndex);
+ if (count2 != totalCount * thisCuboidNum) {
+ throw new IllegalStateException(String.format("Sanity check failed, level %s, total count(*) is %s; cuboid number %s", thisLevel, count2, thisCuboidNum));
+ } else {
+ logger.info("sanity check success for level " + thisLevel + ", count(*) is " + (count2 / thisCuboidNum));
+ }
+ }
+
+ private Long getRDDCountSum(JavaPairRDD<ByteArray, Object[]> rdd, final int countMeasureIndex) {
+ final ByteArray ONE = new ByteArray();
+ Long count = rdd.mapValues(new Function<Object[], Long>() {
+ @Override
+ public Long call(Object[] objects) throws Exception {
+ return (Long) objects[countMeasureIndex]; // assume the first measure is COUNT(*)
+ }
+ }).reduce(new Function2<Tuple2<ByteArray, Long>, Tuple2<ByteArray, Long>, Tuple2<ByteArray, Long>>() {
+ @Override
+ public Tuple2<ByteArray, Long> call(Tuple2<ByteArray, Long> longTuple2, Tuple2<ByteArray, Long> longTuple22) throws Exception {
+ return new Tuple2<>(ONE, longTuple2._2() + longTuple22._2());
+ }
+ })._2();
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 7c88372..644f73f 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.Logger;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -38,11 +39,16 @@ public class SparkExecutable extends AbstractExecutable {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(SparkExecutable.class);
private static final String CLASS_NAME = "className";
+ private static final String JARS = "jars";
public void setClassName(String className) {
this.setParam(CLASS_NAME, className);
}
+ public void setJars(String jars) {
+ this.setParam(JARS, jars);
+ }
+
private String formatArgs() {
StringBuilder stringBuilder = new StringBuilder();
for (Map.Entry<String, String> entry : getParams().entrySet()) {
@@ -50,6 +56,9 @@ public class SparkExecutable extends AbstractExecutable {
tmp.append("-").append(entry.getKey()).append(" ").append(entry.getValue()).append(" ");
if (entry.getKey().equals(CLASS_NAME)) {
stringBuilder.insert(0, tmp);
+ } else if (entry.getKey().equals(JARS)) {
+ // JARS is for spark-submit, not for app
+ continue;
} else {
stringBuilder.append(tmp);
}
@@ -65,12 +74,22 @@ public class SparkExecutable extends AbstractExecutable {
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final KylinConfig config = context.getConfig();
Preconditions.checkNotNull(config.getSparkHome());
- Preconditions.checkNotNull(config.getSparkMaster());
+ Preconditions.checkNotNull(config.getKylinJobJarPath());
+ String sparkConf = config.getSparkConfFile();
+ String jars = this.getParam(JARS);
+
+ String jobJar = config.getKylinJobJarPath();
+
+ if (StringUtils.isEmpty(jars)) {
+ jars = jobJar;
+ }
+
try {
- String cmd = String.format("%s/bin/spark-submit --class \"org.apache.kylin.common.util.SparkEntry\" --master %s %s %s", config.getSparkHome(), config.getSparkMaster(), config.getKylinSparkJobJarPath(), formatArgs());
+ String cmd = String.format("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class \"org.apache.kylin.common.util.SparkEntry\" --properties-file %s --jars %s %s %s", config.getSparkHadoopConfDir(), config.getSparkHome(), sparkConf, jars, jobJar, formatArgs());
logger.info("cmd:" + cmd);
final StringBuilder output = new StringBuilder();
- config.getCliCommandExecutor().execute(cmd, new Logger() {
+ CliCommandExecutor exec = new CliCommandExecutor();
+ exec.execute(cmd, new Logger() {
@Override
public void log(String message) {
output.append(message);
@@ -84,4 +103,5 @@ public class SparkExecutable extends AbstractExecutable {
return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
}
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
index 4f407ff..793cd87 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
@@ -38,7 +38,7 @@ import scala.Tuple2;
*/
public final class DefaultTupleConverter implements TupleConverter {
- private final static ThreadLocal<ByteBuffer> valueBuf = new ThreadLocal<>();
+ private final static transient ThreadLocal<ByteBuffer> valueBuf = new ThreadLocal<>();
private final CubeSegment segment;
private final int measureCount;
private final Map<TblColRef, Integer> columnLengthMap;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/examples/test_case_data/sandbox/kylin-spark-conf.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin-spark-conf.properties b/examples/test_case_data/sandbox/kylin-spark-conf.properties
new file mode 100644
index 0000000..b4a0c97
--- /dev/null
+++ b/examples/test_case_data/sandbox/kylin-spark-conf.properties
@@ -0,0 +1,28 @@
+spark.yarn.submit.file.replication=1
+spark.yarn.executor.memoryOverhead=200
+spark.yarn.driver.memoryOverhead=384
+#spark.master=local[4]
+#spark.submit.deployMode=client
+spark.master=yarn
+spark.submit.deployMode=cluster
+spark.eventLog.enabled=true
+spark.yarn.scheduler.heartbeat.interval-ms=5000
+spark.yarn.preserve.staging.files=true
+spark.yarn.queue=default
+spark.yarn.containerLauncherMaxThreads=25
+spark.yarn.max.executor.failures=3
+spark.eventLog.dir=hdfs\:///spark-history
+spark.history.kerberos.enabled=true
+spark.history.provider=org.apache.spark.deploy.history.FsHistoryProvider
+spark.history.ui.port=18080
+spark.history.fs.logDirectory=hdfs\:///spark-history
+spark.executor.memory=1G
+spark.storage.memoryFraction=0.3
+spark.executor.cores=1
+spark.executor.instances=1
+spark.history.kerberos.keytab=none
+spark.history.kerberos.principal=none
+spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+spark.driver.extraJavaOptions=-Dhdp.version=current
+spark.yarn.am.extraJavaOptions=-Dhdp.version=current
+spark.executor.extraJavaOptions=-Dhdp.version=current
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 1724619..a011911 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -79,6 +79,8 @@ kylin.job.remote-cli-working-dir=/tmp/kylin
# Max count of concurrent jobs running
kylin.job.max-concurrent-jobs=10
+kylin.source.hive.redistribute-flat-table=false
+
# Time interval to check hadoop job status
kylin.engine.mr.yarn-check-interval-seconds=10
@@ -154,3 +156,11 @@ kylin.server.query-metrics-percentiles-intervals=60, 360, 3600
# Env DEV|QA|PROD
kylin.env=DEV
+kylin.source.hive.keep-flat-table=true
+
+### Spark as Engine ###
+#kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+kylin.engine.spark.env.hadoop-conf-dir=../examples/test_case_data/sandbox
+kylin.engine.spark.spark-home=/usr/local/spark
+kylin.engine.spark.properties-file=../examples/test_case_data/sandbox/kylin-spark-conf.properties
+kylin.engine.spark.sanity-check-enabled=false
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0b323cf..0c9a8cb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -84,7 +84,7 @@
<commons-math3.version>3.6.1</commons-math3.version>
<!-- Spark -->
- <spark.version>1.3.0</spark.version>
+ <spark.version>1.6.3</spark.version>
<!-- Utility -->
<log4j.version>1.2.17</log4j.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 978f477..f5b9f7e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -327,10 +327,10 @@ public class CubeController extends BasicController {
throw new InternalErrorException("Cannot find cube '" + cubeName + "'");
}
- if (cube.getSegments() != null && cube.getBuildingSegments().size() > 0) {
- int num = cube.getBuildingSegments().size();
- throw new InternalErrorException("Cannot purge cube '" + cubeName + "' as there is " + num + " building " + (num > 1 ? "segment(s)." : "segment. Discard the related job first."));
- }
+// if (cube.getSegments() != null && cube.getBuildingSegments().size() > 0) {
+// int num = cube.getBuildingSegments().size();
+// throw new InternalErrorException("Cannot purge cube '" + cubeName + "' as there is " + num + " building " + (num > 1 ? "segment(s)." : "segment. Discard the related job first."));
+// }
return cubeService.purgeCube(cube);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 981d64c..1d6bdc9 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -249,6 +249,13 @@
<scope>provided</scope>
</dependency>
+ <!-- Spark dependency -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-testing-util</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index fc2b982..763b537 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.Sets;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -56,8 +57,6 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Sets;
-
public class HiveMRInput implements IMRInput {
public static String getTableNameForHCat(TableDesc table) {
@@ -74,7 +73,7 @@ public class HiveMRInput implements IMRInput {
public IMRTableInputFormat getTableInputFormat(TableDesc table) {
return new HiveTableInputFormat(getTableNameForHCat(table));
}
-
+
@Override
public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg) {
return new IMRBatchMergeInputSide() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
index f0ad6e0..9b1a00d 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
@@ -31,7 +31,6 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
@@ -60,8 +59,8 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase {
BigDecimal sum = new BigDecimal("333.1234567");
BigDecimal min = new BigDecimal("333.1111111");
BigDecimal max = new BigDecimal("333.1999999");
- LongMutable count = new LongMutable(2);
- LongMutable item_count = new LongMutable(100);
+ Long count = new Long(2);
+ Long item_count = new Long(100);
ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });
buf.flip();
@@ -91,7 +90,7 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase {
BigDecimal min = new BigDecimal("333.1111111");
BigDecimal max = new BigDecimal("333.1999999");
LongWritable count = new LongWritable(2);
- LongMutable item_count = new LongMutable(100);
+ Long item_count = new Long(100);
codec.encode(new Object[] { sum, min, max, count, item_count });
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 213f05c..9f192dc 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -40,6 +40,10 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-engine-spark</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
<artifactId>kylin-source-kafka</artifactId>
</dependency>
<dependency>
[16/42] kylin git commit: KYLIN-2327 enable check-style for test code
Posted by sh...@apache.org.
KYLIN-2327 enable check-style for test code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3e0edc84
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3e0edc84
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3e0edc84
Branch: refs/heads/sparkcubing-rebase
Commit: 3e0edc842ec4f6f7d9ef04dfa26afce101d2fd6e
Parents: 1563eb9
Author: Billy Liu <bi...@apache.org>
Authored: Wed Dec 28 01:16:35 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Dec 28 01:16:58 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigTest.java | 6 +-
.../org/apache/kylin/common/util/BasicTest.java | 2 +-
.../apache/kylin/common/util/JacksonBean.java | 7 +-
.../apache/kylin/common/util/JacksonTest.java | 5 +-
.../org/apache/kylin/cube/CubeDescTest.java | 6 +-
.../org/apache/kylin/cube/CubeManagerTest.java | 16 +-
.../inmemcubing/ConcurrentDiskStoreTest.java | 27 +-
.../cube/inmemcubing/MemDiskStoreTest.java | 27 +-
.../inmemcubing/MemoryBudgetControllerTest.java | 27 +-
.../validation/rule/DictionaryRuleTest.java | 1 -
.../gridtable/AggregationCacheMemSizeTest.java | 42 +-
.../kylin/gridtable/SimpleGridTableTest.java | 27 +-
.../gridtable/SimpleInvertedIndexTest.java | 27 +-
.../metadata/measure/MeasureCodecTest.java | 6 +-
.../kylin/dict/AppendTrieDictionaryTest.java | 3 +-
.../apache/kylin/dict/CachedTreeMapTest.java | 21 +-
.../kylin/dict/DictionaryManagerTest.java | 3 +-
.../kylin/dict/TrieDictionaryForestTest.java | 2 +-
.../kylin/dict/lookup/LookupTableTest.java | 1 -
.../impl/threadpool/DefaultSchedulerTest.java | 32 +-
.../measure/AggregatorMemEstimateTest.java | 17 +-
.../kylin/measure/hllc/HLLCounterOldTest.java | 3 +-
.../kylin/measure/hllc/HLLCounterTest.java | 13 +-
.../measure/topn/TopNCounterBasicTest.java | 16 +-
.../topn/TopNCounterCombinationTest.java | 16 +-
.../measure/topn/TopNCounterSerializerTest.java | 2 -
.../kylin/measure/topn/TopNCounterTest.java | 16 +-
.../kylin/metadata/MetadataManagerTest.java | 4 +-
.../DefaultPartitionConditionBuilderTest.java | 24 +-
.../apache/kylin/storage/StorageMockUtils.java | 33 +-
.../storage/gtrecord/DictGridTableTest.java | 2 +-
.../kylin/engine/mr/steps/CubeSamplingTest.java | 2 +-
.../kylin/engine/mr/steps/MockupMapContext.java | 522 ++++++++++---------
.../mr/steps/NumberDictionaryForestTest.java | 18 +
.../mr/steps/SelfDefineSortableKeyTest.java | 18 +
.../apache/kylin/jdbc/SQLResonseStubTest.java | 6 +-
.../inmemcubing/ITDoggedCubeBuilderTest.java | 30 +-
.../inmemcubing/ITInMemCubeBuilderTest.java | 30 +-
pom.xml | 4 +-
.../apache/kylin/query/udf/VersionUDFTest.java | 18 +
.../PasswordPlaceHolderConfigurerTest.java | 1 -
.../rest/controller/CubeControllerTest.java | 1 -
.../config/KafkaConsumerPropertiesTest.java | 18 +
.../hbase/common/HiveMiniClusterTest.java | 131 -----
.../storage/hbase/steps/MockupMapContext.java | 522 ++++++++++---------
45 files changed, 872 insertions(+), 883 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index 209986f..a426fc6 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -18,7 +18,11 @@
package org.apache.kylin.common;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 9105245..bbaab09 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -82,7 +82,7 @@ public class BasicTest {
@Test
public void testxx() throws InterruptedException {
- System.out.println( 0x8fL);
+ System.out.println(0x8fL);
byte[] space = new byte[100];
ByteBuffer buffer = ByteBuffer.wrap(space, 10, 20);
buffer.put((byte) 1);
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
index ffff6d1..214b860 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -18,15 +18,14 @@
package org.apache.kylin.common.util;
+import java.util.HashMap;
+import java.util.Map;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class JacksonBean {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
index 0d7097a..286cdd3 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -18,11 +18,10 @@
package org.apache.kylin.common.util;
-import org.junit.Test;
-
import java.io.IOException;
import java.util.HashMap;
-import java.util.Map;
+
+import org.junit.Test;
public class JacksonTest {
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 39b6433..0a7adf1 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -18,6 +18,10 @@
package org.apache.kylin.cube;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
import java.io.File;
import java.util.Arrays;
import java.util.Collection;
@@ -42,8 +46,6 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
-import static org.junit.Assert.*;
-
/**
* @author yangli9
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index 33dac96..3cae37d 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -136,13 +136,13 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
assertEquals(0, cube.getSegments().size());
Map m1 = Maps.newHashMap();
- m1.put(1, 1000l);
+ m1.put(1, 1000L);
Map m2 = Maps.newHashMap();
- m2.put(1, 2000l);
+ m2.put(1, 2000L);
Map m3 = Maps.newHashMap();
- m3.put(1, 3000l);
+ m3.put(1, 3000L);
Map m4 = Maps.newHashMap();
- m4.put(1, 4000l);
+ m4.put(1, 4000L);
// append first
CubeSegment seg1 = mgr.appendSegment(cube, 0, 0, 0, 1000, null, m1);
@@ -193,13 +193,13 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
// no segment at first
assertEquals(0, cube.getSegments().size());
Map m1 = Maps.newHashMap();
- m1.put(1, 1000l);
+ m1.put(1, 1000L);
Map m2 = Maps.newHashMap();
- m2.put(1, 2000l);
+ m2.put(1, 2000L);
Map m3 = Maps.newHashMap();
- m3.put(1, 3000l);
+ m3.put(1, 3000L);
Map m4 = Maps.newHashMap();
- m4.put(1, 4000l);
+ m4.put(1, 4000L);
// append first
CubeSegment seg1 = mgr.appendSegment(cube, 0, 0, 0, 1000, null, m1);
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStoreTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStoreTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStoreTest.java
index 6355f4a..0de1406 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStoreTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStoreTest.java
@@ -1,19 +1,20 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.kylin.cube.inmemcubing;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemDiskStoreTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemDiskStoreTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemDiskStoreTest.java
index 06ade1c..a64fc21 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemDiskStoreTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemDiskStoreTest.java
@@ -1,19 +1,20 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.kylin.cube.inmemcubing;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemoryBudgetControllerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemoryBudgetControllerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemoryBudgetControllerTest.java
index eb40eaf..24e31e9 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemoryBudgetControllerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemoryBudgetControllerTest.java
@@ -1,19 +1,20 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.kylin.cube.inmemcubing;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index c272bb7..452768f 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -22,7 +22,6 @@ import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_D
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_EMPTY;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_SET;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_TRANSITIVE_REUSE;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 8ffe055..3807b70 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -1,23 +1,32 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.kylin.gridtable;
-import com.google.common.base.Stopwatch;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
@@ -32,8 +41,7 @@ import org.apache.kylin.metadata.datatype.LongMutable;
import org.github.jamm.MemoryMeter;
import org.junit.Test;
-import java.math.BigDecimal;
-import java.util.*;
+import com.google.common.base.Stopwatch;
public class AggregationCacheMemSizeTest {
private static final MemoryMeter meter = new MemoryMeter();
@@ -134,6 +142,8 @@ public class AggregationCacheMemSizeTest {
case WITH_HIGH_CARD_BITMAP:
aggregators.add(createBitmapAggr(false));
break;
+ default:
+ break;
}
return aggregators.toArray(new MeasureAggregator[aggregators.size()]);
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
index fd571d0..f315b1e 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
@@ -1,19 +1,20 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.kylin.gridtable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
index 1026414..2cf58ac 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
@@ -1,19 +1,20 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.kylin.gridtable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 0f3f3a9..fc20418 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -51,7 +51,7 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase {
@Test
public void basicTest() {
- MeasureDesc descs[] = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") };
+ MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") };
BufferedMeasureCodec codec = new BufferedMeasureCodec(descs);
DoubleMutable d = new DoubleMutable(1.0);
@@ -64,13 +64,13 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase {
bitmap.add(123);
bitmap.add(45678);
bitmap.add(Integer.MAX_VALUE - 10);
- Object values[] = new Object[] { d, l, b, hllc, bitmap };
+ Object[] values = new Object[] { d, l, b, hllc, bitmap };
ByteBuffer buf = codec.encode(values);
buf.flip();
System.out.println("size: " + buf.limit());
- Object copy[] = new Object[values.length];
+ Object[] copy = new Object[values.length];
codec.decode(buf, copy);
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index a5ed6f8..f84f08f 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -145,8 +145,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
while ((word = reader.readLine()) != null) {
word = word.trim();
if (!word.isEmpty())
- ;
- b.addValue(word);
+ b.addValue(word);
}
} finally {
reader.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
index 381e0b1..64eb13c 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
@@ -17,18 +17,29 @@
*/
package org.apache.kylin.dict;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Test;
-import java.io.*;
-
-import static org.junit.Assert.*;
-
/**
* Created by sunyerui on 16/7/12.
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
index e6e0374..4820318 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
@@ -18,7 +18,8 @@
package org.apache.kylin.dict;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index 238bb31..ee092c9 100755
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -288,7 +288,7 @@ public class TrieDictionaryForestTest {
TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter());
TrieDictionaryForest<String> dict = b.build();
try{
- int id = dict.getIdFromValue("123",0);
+ int id = dict.getIdFromValue("123", 0);
fail("id should not exist");
}catch (IllegalArgumentException e){
//right
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
index 25d6ae2..5a5c988 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
@@ -25,7 +25,6 @@ import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.dict.TrieDictionaryForest;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
index df521f9..bcd6a59 100644
--- a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -1,22 +1,20 @@
/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- * /
- */
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.kylin.job.impl.threadpool;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
index 0f22610..40e5b29 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -18,10 +18,20 @@
package org.apache.kylin.measure;
-import com.google.common.collect.Lists;
+import java.math.BigDecimal;
+import java.util.List;
+
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.measure.basic.*;
+import org.apache.kylin.measure.basic.BigDecimalMaxAggregator;
+import org.apache.kylin.measure.basic.BigDecimalMinAggregator;
+import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
+import org.apache.kylin.measure.basic.DoubleMaxAggregator;
+import org.apache.kylin.measure.basic.DoubleMinAggregator;
+import org.apache.kylin.measure.basic.DoubleSumAggregator;
+import org.apache.kylin.measure.basic.LongMaxAggregator;
+import org.apache.kylin.measure.basic.LongMinAggregator;
+import org.apache.kylin.measure.basic.LongSumAggregator;
import org.apache.kylin.measure.bitmap.BitmapAggregator;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
@@ -35,8 +45,7 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.math.BigDecimal;
-import java.util.List;
+import com.google.common.collect.Lists;
public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
private static final MemoryMeter meter = new MemoryMeter();
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
index 427d9a7..2f833bf 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
@@ -28,7 +28,6 @@ import java.util.Random;
import java.util.Set;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.measure.hllc.HLLCounterOld;
import org.junit.Assert;
import org.junit.Test;
@@ -227,7 +226,7 @@ public class HLLCounterOldTest {
int N = 3; // reduce N HLLC into one
int M = 1000; // for M times, use 100000 for real perf test
- HLLCounterOld samples[] = new HLLCounterOld[N];
+ HLLCounterOld[] samples = new HLLCounterOld[N];
for (int i = 0; i < N; i++) {
samples[i] = newHLLC();
for (String str : generateTestData(10000))
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
index 4a95bd4..92f2aab 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
@@ -17,12 +17,8 @@
*/
package org.apache.kylin.measure.hllc;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.measure.hllc.HLLCounterOld;
-import org.apache.kylin.measure.hllc.HLLCounter;
-import org.apache.kylin.measure.hllc.RegisterType;
-import org.junit.Assert;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -30,8 +26,9 @@ import java.util.HashSet;
import java.util.Random;
import java.util.Set;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import org.apache.kylin.common.util.Bytes;
+import org.junit.Assert;
+import org.junit.Test;
/**
* Created by xiefan on 16-12-12.
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterBasicTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterBasicTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterBasicTest.java
index 162ef01..9d034fe 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterBasicTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterBasicTest.java
@@ -1,18 +1,20 @@
/*
- * Copyright (C) 2011 Clearspring Technologies, Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- */
+*/
package org.apache.kylin.measure.topn;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterCombinationTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterCombinationTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterCombinationTest.java
index 282f648..7cde4cc 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterCombinationTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterCombinationTest.java
@@ -1,18 +1,20 @@
/*
- * Copyright (C) 2011 Clearspring Technologies, Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- */
+*/
package org.apache.kylin.measure.topn;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterSerializerTest.java
index 2daf3b4..6bc02cd 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterSerializerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterSerializerTest.java
@@ -23,8 +23,6 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.measure.topn.TopNCounter;
-import org.apache.kylin.measure.topn.TopNCounterSerializer;
import org.apache.kylin.metadata.datatype.DataType;
import org.junit.AfterClass;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterTest.java
index aa8b2fe..3eba78b 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterTest.java
@@ -1,18 +1,20 @@
/*
- * Copyright (C) 2011 Clearspring Technologies, Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- */
+*/
package org.apache.kylin.measure.topn;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index b372be2..a861ee1 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -19,7 +19,9 @@
package org.apache.kylin.metadata;
import static org.apache.kylin.metadata.MetadataManager.getInstance;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
index 4e56a71..1b610ca 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
@@ -1,19 +1,21 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- */
+*/
+
package org.apache.kylin.metadata.model;
import org.apache.kylin.common.util.DateFormat;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
index 0706713..8099527 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
@@ -1,23 +1,20 @@
/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- * /
- */
-
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.kylin.storage;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
index bac9c10..dde0481 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -561,7 +561,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
static GTInfo newInfo() {
Builder builder = GTInfo.builder();
builder.setCodeSystem(newDictCodeSystem());
- builder.setColumns( //
+ builder.setColumns(//
DataType.getType("timestamp"), //
DataType.getType("integer"), //
DataType.getType("varchar(10)"), //
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
index beec00f..0c8c13b 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
@@ -51,7 +51,7 @@ public class CubeSamplingTest {
@Before
public void setup() {
- baseCuboidId = (1l << ROW_LENGTH) - 1;
+ baseCuboidId = (1L << ROW_LENGTH) - 1;
List<Long> allCuboids = Lists.newArrayList();
List<Integer[]> allCuboidsBitSetList = Lists.newArrayList();
for (long i = 1; i < baseCuboidId; i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..dfbba14 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -47,267 +47,275 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
*
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext {
+public class MockupMapContext implements MapContext{
+
+ private Configuration hconf;
+
+ private Object[] outKV;
public static Context create(final Configuration hconf, String metadataUrl, String cubeName, final Object[] outKV) {
hconf.set(BatchConstants.CFG_CUBE_NAME, cubeName);
- return new WrappedMapper().getMapContext(new MapContext() {
-
- @Override
- public boolean nextKeyValue() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public Object getCurrentKey() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public Object getCurrentValue() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public void write(Object key, Object value) throws IOException, InterruptedException {
- System.out.println("Write -- k:" + key + ", v:" + value);
- if (outKV != null) {
- outKV[0] = key;
- outKV[1] = value;
- }
- }
-
- @Override
- public OutputCommitter getOutputCommitter() {
- throw new NotImplementedException();
- }
-
- @Override
- public TaskAttemptID getTaskAttemptID() {
- throw new NotImplementedException();
- }
-
- @Override
- public void setStatus(String msg) {
- throw new NotImplementedException();
- }
-
- @Override
- public String getStatus() {
- throw new NotImplementedException();
- }
-
- @Override
- public float getProgress() {
- throw new NotImplementedException();
- }
-
- @Override
- public Counter getCounter(Enum<?> counterName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Counter getCounter(String groupName, String counterName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Configuration getConfiguration() {
- return hconf;
- }
-
- @Override
- public Credentials getCredentials() {
- throw new NotImplementedException();
- }
-
- @Override
- public JobID getJobID() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getNumReduceTasks() {
- throw new NotImplementedException();
- }
-
- @Override
- public Path getWorkingDirectory() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getOutputKeyClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getOutputValueClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getMapOutputKeyClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getMapOutputValueClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getJobName() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Partitioner<?, ?>> getPartitionerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getSortComparator() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getJar() {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getGroupingComparator() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getJobSetupCleanupNeeded() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getTaskCleanupNeeded() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getProfileEnabled() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getProfileParams() {
- throw new NotImplementedException();
- }
-
- @Override
- public IntegerRanges getProfileTaskRange(boolean isMap) {
- throw new NotImplementedException();
- }
-
- @Override
- public String getUser() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getSymlink() {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getArchiveClassPaths() {
- throw new NotImplementedException();
- }
-
- @Override
- public URI[] getCacheArchives() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public URI[] getCacheFiles() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getLocalCacheArchives() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getLocalCacheFiles() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getFileClassPaths() {
- throw new NotImplementedException();
- }
-
- @Override
- public String[] getArchiveTimestamps() {
- throw new NotImplementedException();
- }
-
- @Override
- public String[] getFileTimestamps() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getMaxMapAttempts() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getMaxReduceAttempts() {
- throw new NotImplementedException();
- }
-
- @Override
- public void progress() {
- throw new NotImplementedException();
- }
-
- @Override
- public InputSplit getInputSplit() {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getCombinerKeyGroupingComparator() {
- throw new NotImplementedException();
- }
- });
+ return new WrappedMapper().getMapContext(new MockupMapContext(hconf, outKV));
+ }
+
+ public MockupMapContext(Configuration hconf, Object[] outKV){
+ this.hconf = hconf;
+ this.outKV = outKV;
+ }
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Object getCurrentKey() throws IOException, InterruptedException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Object getCurrentValue() throws IOException, InterruptedException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void write(Object key, Object value) throws IOException, InterruptedException {
+ System.out.println("Write -- k:" + key + ", v:" + value);
+ if (outKV != null) {
+ outKV[0] = key;
+ outKV[1] = value;
+ }
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public TaskAttemptID getTaskAttemptID() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getStatus() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void setStatus(String msg) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public float getProgress() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Counter getCounter(Enum<?> counterName) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Counter getCounter(String groupName, String counterName) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return hconf;
+ }
+
+ @Override
+ public Credentials getCredentials() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public JobID getJobID() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int getNumReduceTasks() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path getWorkingDirectory() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getOutputKeyClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getOutputValueClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getMapOutputKeyClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getMapOutputValueClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getJobName() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Partitioner<?, ?>> getPartitionerClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public RawComparator<?> getSortComparator() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getJar() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public RawComparator<?> getGroupingComparator() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getJobSetupCleanupNeeded() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getTaskCleanupNeeded() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getProfileEnabled() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getProfileParams() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public IntegerRanges getProfileTaskRange(boolean isMap) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getUser() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getSymlink() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getArchiveClassPaths() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public URI[] getCacheArchives() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public URI[] getCacheFiles() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getLocalCacheArchives() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getLocalCacheFiles() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getFileClassPaths() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String[] getArchiveTimestamps() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String[] getFileTimestamps() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int getMaxMapAttempts() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int getMaxReduceAttempts() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void progress() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public InputSplit getInputSplit() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public RawComparator<?> getCombinerKeyGroupingComparator() {
+ throw new NotImplementedException();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
index bb10b4a..f6664bf 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.engine.mr.steps;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
index d71e366..df6287f 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.engine.mr.steps;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/jdbc/src/test/java/org/apache/kylin/jdbc/SQLResonseStubTest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/SQLResonseStubTest.java b/jdbc/src/test/java/org/apache/kylin/jdbc/SQLResonseStubTest.java
index 9e4b3f6..e090af6 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/SQLResonseStubTest.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/SQLResonseStubTest.java
@@ -36,7 +36,8 @@ public class SQLResonseStubTest {
@Test
public void testReadValuePartRecognizedField() throws IOException {
- final String payload = "{ \"columnMetas\":[ { \"isNullable\":1, \"displaySize\":0, \"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":0, \"scale\":0, \"columnType\":91, \"columnTypeName\":\"DATE\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false }, { \"isNullable\":1, \"displaySize\":10, \"label\":\"LEAF_CATEG_ID\", \"name\":\"LEAF_CATEG_ID\", \"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":10, \"scale\":0, \"columnType\":4, \"columnTypeName\":\"INTEGER\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false } ], \"results\":[ [ \"2013-08-07\", \"32996\", \"15\", \"15\", \"Auction\", \"10000000\", \"49.048952730908745\", \"49.048952730908745\", \"49.048952730908745\", \"1\"
], [ \"2013-08-07\", \"43398\", \"0\", \"14\", \"ABIN\", \"10000633\", \"85.78317064220418\", \"85.78317064220418\", \"85.78317064220418\", \"1\" ] ], \"cube\":\"test_kylin_cube_with_slr_desc\", \"affectedRowCount\":0, \"isException\":false, \"exceptionMessage\":null, \"duration\":3451, \"partial\":false }";
+ final String payload = "{ \"columnMetas\":[ { \"isNullable\":1, \"displaySize\":0, \"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":0, \"scale\":0, \"columnType\":91, \"columnTypeName\":\"DATE\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false }," + "{ \"isNullable\":1, \"displaySize\":10, \"label\":\"LEAF_CATEG_ID\", \"name\":\"LEAF_CATEG_ID\", "
+ + "\"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":10, \"scale\":0, \"columnType\":4, \"columnTypeName\":\"INTEGER\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false } ], \"results\":[ [ \"2013-08-07\", \"32996\", \"15\", \"15\", \"Auction\", \"10000000\", \"49.048952730908745\", \"49.048952730908745\", \"49.048952730908745\", \"1\" ], [ \"2013-08-07\", \"43398\", \"0\", \"14\", \"ABIN\", \"10000633\", \"85.78317064220418\", \"85.78317064220418\", \"85.78317064220418\", \"1\" ] ], \"cube\":\"test_kylin_cube_with_slr_desc\", \"affectedRowCount\":0, \"isException\":false, \"exceptionMessage\":null, \"duration\":3451, \"partial\":false }";
final SQLResponseStub stub = new ObjectMapper().readValue(payload, SQLResponseStub.class);
assertEquals("test_kylin_cube_with_slr_desc", stub.getCube());
assertEquals(3451, stub.getDuration());
@@ -49,7 +50,8 @@ public class SQLResonseStubTest {
@Test
public void testReadValueWithUnrecognizedField() throws IOException {
- final String payload = "{ \"columnMetas\":[ { \"Unrecognized\":0, \"isNullable\":1, \"displaySize\":0, \"label\":\"CAL_DT\", \"name\":\"CAL_DT\", \"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":0, \"scale\":0, \"columnType\":91, \"columnTypeName\":\"DATE\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false }, { \"isNullable\":1, \"displaySize\":10, \"label\":\"LEAF_CATEG_ID\", \"name\":\"LEAF_CATEG_ID\", \"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":10, \"scale\":0, \"columnType\":4, \"columnTypeName\":\"INTEGER\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false } ], \"results\":[ [ \"2013-08-07\", \"32996\", \"15\", \"15\", \"Auction\", \"10000000\", \"49.0489527309
08745\", \"49.048952730908745\", \"49.048952730908745\", \"1\" ], [ \"2013-08-07\", \"43398\", \"0\", \"14\", \"ABIN\", \"10000633\", \"85.78317064220418\", \"85.78317064220418\", \"85.78317064220418\", \"1\" ] ], \"cube\":\"test_kylin_cube_with_slr_desc\", \"affectedRowCount\":0, \"isException\":false, \"exceptionMessage\":null, \"duration\":3451, \"partial\":false, \"hitCache\":false }";
+ final String payload = "{ \"columnMetas\":[ { \"Unrecognized\":0, \"isNullable\":1, \"displaySize\":0, " + "\"label\":\"CAL_DT\", \"name\":\"CAL_DT\", \"schemaName\":null, \"catelogName\":null, " + "\"tableName\":null, \"precision\":0, \"scale\":0, \"columnType\":91, \"columnTypeName\":\"DATE\", " + "\"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false },"
+ + " { \"isNullable\":1, \"displaySize\":10, \"label\":\"LEAF_CATEG_ID\", \"name\":\"LEAF_CATEG_ID\", \"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":10, \"scale\":0, \"columnType\":4, \"columnTypeName\":\"INTEGER\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false } ], \"results\":[ [ \"2013-08-07\", \"32996\", \"15\", \"15\", \"Auction\", \"10000000\", \"49.048952730908745\", \"49.048952730908745\", \"49.048952730908745\", \"1\" ], [ \"2013-08-07\", \"43398\", \"0\", \"14\", \"ABIN\", \"10000633\", \"85.78317064220418\", \"85.78317064220418\", \"85.78317064220418\", \"1\" ] ], \"cube\":\"test_kylin_cube_with_slr_desc\", \"affectedRowCount\":0, \"isException\":false, \"exceptionMessage\":null, \"duration\":3451, \"partial\":false, \"hitCache\":false }";
final SQLResponseStub stub = new ObjectMapper().readValue(payload, SQLResponseStub.class);
assertEquals("test_kylin_cube_with_slr_desc", stub.getCube());
assertEquals(3451, stub.getDuration());
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
index 1db7a24..6f29812 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
@@ -1,21 +1,19 @@
/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- * /
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.kylin.cube.inmemcubing;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
index f9b6ef2..35a4f05 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
@@ -1,21 +1,19 @@
/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- * /
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.kylin.cube.inmemcubing;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2ff4c54..48c075f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -942,12 +942,12 @@
<version>2.17</version>
<executions>
<execution>
- <id>validate</id>
+ <id>check-style</id>
<phase>validate</phase>
<configuration>
<configLocation>dev-support/checkstyle.xml</configLocation>
<suppressionsLocation>dev-support/checkstyle-suppressions.xml</suppressionsLocation>
- <encoding>UTF-8</encoding>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
<consoleOutput>true</consoleOutput>
<failsOnError>true</failsOnError>
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/query/src/test/java/org/apache/kylin/query/udf/VersionUDFTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/udf/VersionUDFTest.java b/query/src/test/java/org/apache/kylin/query/udf/VersionUDFTest.java
index 98cced4..7f106ec 100644
--- a/query/src/test/java/org/apache/kylin/query/udf/VersionUDFTest.java
+++ b/query/src/test/java/org/apache/kylin/query/udf/VersionUDFTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.query.udf;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java b/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
index 8f53084..ac42ad7 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
@@ -20,7 +20,6 @@ package org.apache.kylin.rest.security;
import org.junit.Assert;
import org.junit.Test;
-import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
public class PasswordPlaceHolderConfigurerTest {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index d192a7d..5c010ef 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.service.CubeService;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/source-kafka/src/test/java/org/apache/kylin/source/kafka/config/KafkaConsumerPropertiesTest.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/test/java/org/apache/kylin/source/kafka/config/KafkaConsumerPropertiesTest.java b/source-kafka/src/test/java/org/apache/kylin/source/kafka/config/KafkaConsumerPropertiesTest.java
index 1d7863b..208fdb6 100644
--- a/source-kafka/src/test/java/org/apache/kylin/source/kafka/config/KafkaConsumerPropertiesTest.java
+++ b/source-kafka/src/test/java/org/apache/kylin/source/kafka/config/KafkaConsumerPropertiesTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.source.kafka.config;
import static org.junit.Assert.assertEquals;
[35/42] kylin git commit: KYLIN-2323 Refactor table load/unload
Posted by sh...@apache.org.
KYLIN-2323 Refactor table load/unload
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2e2a81c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2e2a81c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2e2a81c
Branch: refs/heads/sparkcubing-rebase
Commit: e2e2a81c1f1d7e3f9af5c81bb4f1ad14d2d1b859
Parents: 6ea03b8
Author: Billy Liu <bi...@apache.org>
Authored: Thu Dec 29 18:48:38 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Thu Dec 29 18:48:38 2016 +0800
----------------------------------------------------------------------
.../hive/ITHiveSourceTableLoaderTest.java | 2 +-
.../rest/controller/StreamingController.java | 22 +-
.../kylin/rest/controller/TableController.java | 268 ++++------------
.../apache/kylin/rest/service/CubeService.java | 96 ------
.../apache/kylin/rest/service/TableService.java | 318 +++++++++++++++++++
.../source/hive/HiveSourceTableLoader.java | 2 +-
webapp/app/js/controllers/sourceMeta.js | 16 +-
webapp/app/js/services/tables.js | 1 -
8 files changed, 391 insertions(+), 334 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e2a81c/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
index c4f0777..7aff3ba 100644
--- a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
@@ -45,7 +45,7 @@ public class ITHiveSourceTableLoaderTest extends HBaseMetadataTestCase {
public void test() throws IOException {
KylinConfig config = getTestConfig();
String[] toLoad = new String[] { "DEFAULT.TEST_KYLIN_FACT", "EDW.TEST_CAL_DT" };
- Set<String> loaded = HiveSourceTableLoader.reloadHiveTables(toLoad, config);
+ Set<String> loaded = HiveSourceTableLoader.loadHiveTables(toLoad, config);
assertTrue(loaded.size() == toLoad.length);
for (String str : toLoad)
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e2a81c/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index e04ebc8..0ced9ad 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -23,10 +23,8 @@ import java.util.List;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -34,9 +32,9 @@ import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.request.StreamingRequest;
-import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.KafkaConfigService;
import org.apache.kylin.rest.service.StreamingService;
+import org.apache.kylin.rest.service.TableService;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,7 +67,7 @@ public class StreamingController extends BasicController {
@Autowired
private KafkaConfigService kafkaConfigService;
@Autowired
- private CubeService cubeMgmtService;
+ private TableService tableService;
@RequestMapping(value = "/getConfig", method = { RequestMethod.GET })
@ResponseBody
@@ -113,10 +111,7 @@ public class StreamingController extends BasicController {
boolean saveStreamingSuccess = false, saveKafkaSuccess = false;
try {
- tableDesc.setUuid(UUID.randomUUID().toString());
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- metaMgr.saveSourceTable(tableDesc);
- cubeMgmtService.syncTableToProject(new String[] { tableDesc.getIdentity() }, project);
+ tableService.addStreamingTable(tableDesc, project);
} catch (IOException e) {
throw new BadRequestException("Failed to add streaming table.");
}
@@ -289,15 +284,4 @@ public class StreamingController extends BasicController {
request.setMessage(message);
}
- public void setStreamingService(StreamingService streamingService) {
- this.streamingService = streamingService;
- }
-
- public void setKafkaConfigService(KafkaConfigService kafkaConfigService) {
- this.kafkaConfigService = kafkaConfigService;
- }
-
- public void setCubeService(CubeService cubeService) {
- this.cubeMgmtService = cubeService;
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e2a81c/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 74d1b28..eed5413 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -19,37 +19,18 @@
package org.apache.kylin.rest.controller;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
-import java.util.Iterator;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.UUID;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TableExtDesc;
-import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.request.CardinalityRequest;
import org.apache.kylin.rest.request.HiveTableRequest;
-import org.apache.kylin.rest.request.StreamingRequest;
-import org.apache.kylin.rest.response.TableDescResponse;
-import org.apache.kylin.rest.service.CubeService;
-import org.apache.kylin.rest.service.KafkaConfigService;
-import org.apache.kylin.rest.service.ModelService;
-import org.apache.kylin.rest.service.ProjectService;
-import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClientFactory;
-import org.apache.kylin.source.hive.IHiveClient;
-import org.apache.kylin.source.kafka.config.KafkaConfig;
+import org.apache.kylin.rest.service.TableService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -70,44 +51,27 @@ import com.google.common.collect.Sets;
@Controller
@RequestMapping(value = "/tables")
public class TableController extends BasicController {
+
private static final Logger logger = LoggerFactory.getLogger(TableController.class);
@Autowired
- private CubeService cubeMgmtService;
- @Autowired
- private ProjectService projectService;
- @Autowired
- private StreamingService streamingService;
- @Autowired
- private KafkaConfigService kafkaConfigService;
- @Autowired
- private ModelService modelService;
+ private TableService tableService;
/**
- * Get available table list of the input database
+ * Get available table list of the project
*
* @return Table metadata array
* @throws IOException
*/
@RequestMapping(value = "", method = { RequestMethod.GET })
@ResponseBody
- public List<TableDesc> getHiveTables(@RequestParam(value = "ext", required = false) boolean withExt, @RequestParam(value = "project", required = true) String project) throws IOException {
- long start = System.currentTimeMillis();
- List<TableDesc> tables = null;
+ public List<TableDesc> getTableDesc(@RequestParam(value = "ext", required = false) boolean withExt, @RequestParam(value = "project", required = true) String project) throws IOException {
try {
- tables = cubeMgmtService.getProjectManager().listDefinedTables(project);
- } catch (Exception e) {
- logger.error("Failed to deal with the request.", e);
+ return tableService.getTableDescByProject(project, withExt);
+ } catch (IOException e) {
+ logger.error("Failed to get Hive Tables", e);
throw new InternalErrorException(e.getLocalizedMessage());
}
-
- if (withExt) {
- tables = cloneTableDesc(tables);
- }
- long end = System.currentTimeMillis();
- logger.info("Return all table metadata in " + (end - start) + " seconds");
-
- return tables;
}
/**
@@ -118,29 +82,39 @@ public class TableController extends BasicController {
*/
@RequestMapping(value = "/{tableName:.+}", method = { RequestMethod.GET })
@ResponseBody
- public TableDesc getHiveTable(@PathVariable String tableName) {
- return cubeMgmtService.getMetadataManager().getTableDesc(tableName);
- }
-
- @RequestMapping(value = "/reload", method = { RequestMethod.PUT })
- @ResponseBody
- public String reloadSourceTable() {
- cubeMgmtService.getMetadataManager().reload();
- return "ok";
+ public TableDesc getTableDesc(@PathVariable String tableName) {
+ TableDesc table = tableService.getTableDescByName(tableName, false);
+ if (table == null)
+ throw new NotFoundException("Could not find Hive table: " + tableName);
+ return table;
}
@RequestMapping(value = "/{tables}/{project}", method = { RequestMethod.POST })
@ResponseBody
- public Map<String, String[]> loadHiveTable(@PathVariable String tables, @PathVariable String project, @RequestBody HiveTableRequest request) throws IOException {
+ public Map<String, String[]> loadHiveTables(@PathVariable String tables, @PathVariable String project, @RequestBody HiveTableRequest request) throws IOException {
String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
- String[] loaded = cubeMgmtService.reloadHiveTable(tables);
- if (request.isCalculate()) {
- cubeMgmtService.calculateCardinalityIfNotPresent(loaded, submitter);
- }
- cubeMgmtService.syncTableToProject(loaded, project);
Map<String, String[]> result = new HashMap<String, String[]>();
- result.put("result.loaded", loaded);
- result.put("result.unloaded", new String[] {});
+ String[] tableNames = tables.split(",");
+ try {
+ String[] loaded = tableService.loadHiveTablesToProject(tableNames, project);
+ result.put("result.loaded", loaded);
+ Set<String> allTables = new HashSet<String>();
+ for (String tableName : tableNames) {
+ allTables.add(tableService.normalizeHiveTableName(tableName));
+ }
+ for (String loadedTableName : loaded) {
+ allTables.remove(loadedTableName);
+ }
+ String[] unloaded = new String[allTables.size()];
+ allTables.toArray(unloaded);
+ result.put("result.unloaded", unloaded);
+ if (request.isCalculate()) {
+ tableService.calculateCardinalityIfNotPresent(loaded, submitter);
+ }
+ } catch (Exception e) {
+ logger.error("Failed to load Hive Table", e);
+ throw new InternalErrorException(e.getLocalizedMessage());
+ }
return result;
}
@@ -150,12 +124,17 @@ public class TableController extends BasicController {
Set<String> unLoadSuccess = Sets.newHashSet();
Set<String> unLoadFail = Sets.newHashSet();
Map<String, String[]> result = new HashMap<String, String[]>();
- for (String tableName : tables.split(",")) {
- if (unLoadHiveTable(tableName, project)) {
- unLoadSuccess.add(tableName);
- } else {
- unLoadFail.add(tableName);
+ try {
+ for (String tableName : tables.split(",")) {
+ if (tableService.unLoadHiveTable(tableName, project)) {
+ unLoadSuccess.add(tableName);
+ } else {
+ unLoadFail.add(tableName);
+ }
}
+ } catch (Exception e) {
+ logger.error("Failed to unload Hive Table", e);
+ throw new InternalErrorException(e.getLocalizedMessage());
}
result.put("result.unload.success", (String[]) unLoadSuccess.toArray(new String[unLoadSuccess.size()]));
result.put("result.unload.fail", (String[]) unLoadFail.toArray(new String[unLoadFail.size()]));
@@ -163,77 +142,6 @@ public class TableController extends BasicController {
}
/**
- * table may referenced by several projects, and kylin only keep one copy of meta for each table,
- * that's why we have two if statement here.
- * @param tableName
- * @param project
- * @return
- */
- private boolean unLoadHiveTable(String tableName, String project) {
- boolean rtn = false;
- int tableType = 0;
-
- //remove streaming info
- String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
- tableName = dbTableName[0] + "." + dbTableName[1];
- TableDesc desc = cubeMgmtService.getMetadataManager().getTableDesc(tableName);
- if (desc == null)
- return false;
- tableType = desc.getSourceType();
-
- try {
- if (!modelService.isTableInModel(tableName, project)) {
- cubeMgmtService.removeTableFromProject(tableName, project);
- rtn = true;
- } else {
- List<String> models = modelService.getModelsUsingTable(tableName, project);
- throw new InternalErrorException("Table is already in use by models " + models);
- }
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
- }
- if (!projectService.isTableInAnyProject(tableName) && !modelService.isTableInAnyModel(tableName)) {
- try {
- cubeMgmtService.unLoadHiveTable(tableName);
- rtn = true;
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
- rtn = false;
- }
- }
-
- if (tableType == 1 && !projectService.isTableInAnyProject(tableName) && !modelService.isTableInAnyModel(tableName)) {
- StreamingConfig config = null;
- KafkaConfig kafkaConfig = null;
- try {
- config = streamingService.getStreamingManager().getStreamingConfig(tableName);
- kafkaConfig = kafkaConfigService.getKafkaConfig(tableName);
- streamingService.dropStreamingConfig(config);
- kafkaConfigService.dropKafkaConfig(kafkaConfig);
- rtn = true;
- } catch (Exception e) {
- rtn = false;
- logger.error(e.getLocalizedMessage(), e);
- }
- }
- return rtn;
- }
-
- @RequestMapping(value = "/addStreamingSrc", method = { RequestMethod.POST })
- @ResponseBody
- public Map<String, String> addStreamingTable(@RequestBody StreamingRequest request) throws IOException {
- Map<String, String> result = new HashMap<String, String>();
- String project = request.getProject();
- TableDesc desc = JsonUtil.readValue(request.getTableData(), TableDesc.class);
- desc.setUuid(UUID.randomUUID().toString());
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- metaMgr.saveSourceTable(desc);
- cubeMgmtService.syncTableToProject(new String[] { desc.getName() }, project);
- result.put("success", "true");
- return result;
- }
-
- /**
* Regenerate table cardinality
*
* @return Table metadata array
@@ -244,57 +152,15 @@ public class TableController extends BasicController {
public CardinalityRequest generateCardinality(@PathVariable String tableNames, @RequestBody CardinalityRequest request) throws IOException {
String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
String[] tables = tableNames.split(",");
- for (String table : tables) {
- cubeMgmtService.calculateCardinality(table.trim().toUpperCase(), submitter);
- }
- return request;
- }
-
- /**
- * @param tables
- * @return
- */
- private List<TableDesc> cloneTableDesc(List<TableDesc> tables) throws IOException {
- if (null == tables) {
- return Collections.emptyList();
- }
-
- List<TableDesc> descs = new ArrayList<TableDesc>();
- Iterator<TableDesc> it = tables.iterator();
- while (it.hasNext()) {
- TableDesc table = it.next();
- TableExtDesc tableExtDesc = cubeMgmtService.getMetadataManager().getTableExt(table.getIdentity());
-
- // Clone TableDesc
- TableDescResponse rtableDesc = new TableDescResponse(table);
- Map<String, Long> cardinality = new HashMap<String, Long>();
- Map<String, String> dataSourceProp = new HashMap<>();
- String scard = tableExtDesc.getCardinality();
- if (!StringUtils.isEmpty(scard)) {
- String[] cards = StringUtils.split(scard, ",");
- ColumnDesc[] cdescs = rtableDesc.getColumns();
- for (int i = 0; i < cdescs.length; i++) {
- ColumnDesc columnDesc = cdescs[i];
- if (cards.length > i) {
- cardinality.put(columnDesc.getName(), Long.parseLong(cards[i]));
- } else {
- logger.error("The result cardinality is not identical with hive table metadata, cardinaly : " + scard + " column array length: " + cdescs.length);
- break;
- }
- }
- rtableDesc.setCardinality(cardinality);
+ try {
+ for (String table : tables) {
+ tableService.calculateCardinality(table.trim().toUpperCase(), submitter);
}
- dataSourceProp.putAll(tableExtDesc.getDataSourceProp());
- dataSourceProp.put("location", tableExtDesc.getStorageLocation());
- dataSourceProp.put("owner", tableExtDesc.getOwner());
- dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime());
- dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn());
- dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize());
- rtableDesc.setDescExd(dataSourceProp);
- descs.add(rtableDesc);
+ } catch (IOException e) {
+ logger.error("Failed to calculate cardinality", e);
+ throw new InternalErrorException(e.getLocalizedMessage());
}
-
- return descs;
+ return request;
}
/**
@@ -305,17 +171,12 @@ public class TableController extends BasicController {
*/
@RequestMapping(value = "/hive", method = { RequestMethod.GET })
@ResponseBody
- private static List<String> showHiveDatabases() throws IOException {
- IHiveClient hiveClient = HiveClientFactory.getHiveClient();
- List<String> results = null;
-
+ private List<String> showHiveDatabases() throws IOException {
try {
- results = hiveClient.getHiveDbNames();
+ return tableService.getHiveDbNames();
} catch (Exception e) {
- e.printStackTrace();
- throw new IOException(e);
+ throw new InternalErrorException(e.getLocalizedMessage());
}
- return results;
}
/**
@@ -326,21 +187,12 @@ public class TableController extends BasicController {
*/
@RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
@ResponseBody
- private static List<String> showHiveTables(@PathVariable String database) throws IOException {
- IHiveClient hiveClient = HiveClientFactory.getHiveClient();
- List<String> results = null;
-
+ private List<String> showHiveTables(@PathVariable String database) throws IOException {
try {
- results = hiveClient.getHiveTableNames(database);
+ return tableService.getHiveTableNames(database);
} catch (Exception e) {
- e.printStackTrace();
- throw new IOException(e);
+ throw new InternalErrorException(e.getLocalizedMessage());
}
- return results;
- }
-
- public void setCubeService(CubeService cubeService) {
- this.cubeMgmtService = cubeService;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e2a81c/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 85c9284..23aa5a4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -25,7 +25,6 @@ import java.util.Date;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.WeakHashMap;
import org.apache.commons.io.IOUtils;
@@ -41,17 +40,10 @@ import org.apache.kylin.cube.cuboid.CuboidCLI;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
-import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
@@ -63,9 +55,6 @@ import org.apache.kylin.rest.request.MetricsRequest;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.security.AclPermission;
-import org.apache.kylin.source.hive.HiveSourceTableLoader;
-import org.apache.kylin.source.hive.cardinality.HiveColumnCardinalityJob;
-import org.apache.kylin.source.hive.cardinality.HiveColumnCardinalityUpdateJob;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.slf4j.Logger;
@@ -438,53 +427,6 @@ public class CubeService extends BasicService {
return hr;
}
- /**
- * Generate cardinality for table This will trigger a hadoop job
- * The result will be merged into table exd info
- *
- * @param tableName
- */
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
- public void calculateCardinality(String tableName, String submitter) throws IOException {
- String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
- tableName = dbTableName[0] + "." + dbTableName[1];
- TableDesc table = getMetadataManager().getTableDesc(tableName);
- final TableExtDesc tableExt = getMetadataManager().getTableExt(tableName);
- if (table == null) {
- IllegalArgumentException e = new IllegalArgumentException("Cannot find table descirptor " + tableName);
- logger.error("Cannot find table descirptor " + tableName, e);
- throw e;
- }
-
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- //make sure the job could be scheduled when the DistributedScheduler is enable.
- job.setParam("segmentId", tableName);
- job.setName("Hive Column Cardinality calculation for table '" + tableName + "'");
- job.setSubmitter(submitter);
-
- String outPath = getConfig().getHdfsWorkingDirectory() + "cardinality/" + job.getId() + "/" + tableName;
- String param = "-table " + tableName + " -output " + outPath;
-
- MapReduceExecutable step1 = new MapReduceExecutable();
-
- step1.setMapReduceJobClass(HiveColumnCardinalityJob.class);
- step1.setMapReduceParams(param);
- step1.setParam("segmentId", tableName);
-
- job.addTask(step1);
-
- HadoopShellExecutable step2 = new HadoopShellExecutable();
-
- step2.setJobClass(HiveColumnCardinalityUpdateJob.class);
- step2.setJobParams(param);
- step2.setParam("segmentId", tableName);
- job.addTask(step2);
- tableExt.setJodID(job.getId());
- getMetadataManager().saveTableExt(tableExt);
-
- getExecutableManager().addJob(job);
- }
-
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
public void updateCubeNotifyList(CubeInstance cube, List<String> notifyList) throws IOException {
CubeDesc desc = cube.getDescriptor();
@@ -546,44 +488,6 @@ public class CubeService extends BasicService {
CubeManager.getInstance(getConfig()).updateCube(update);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
- public String[] reloadHiveTable(String tables) throws IOException {
- Set<String> loaded = HiveSourceTableLoader.reloadHiveTables(tables.split(","), getConfig());
- return (String[]) loaded.toArray(new String[loaded.size()]);
- }
-
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
- public void unLoadHiveTable(String tableName) throws IOException {
- String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
- tableName = dbTableName[0] + "." + dbTableName[1];
- HiveSourceTableLoader.unLoadHiveTable(tableName.toUpperCase());
- }
-
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
- public void syncTableToProject(String[] tables, String project) throws IOException {
- getProjectManager().addTableDescToProject(tables, project);
- }
-
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
- public void removeTableFromProject(String tableName, String projectName) throws IOException {
- String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
- tableName = dbTableName[0] + "." + dbTableName[1];
- getProjectManager().removeTableDescFromProject(tableName, projectName);
- }
-
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
- public void calculateCardinalityIfNotPresent(String[] tables, String submitter) throws IOException {
- MetadataManager metaMgr = getMetadataManager();
- ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig());
- for (String table : tables) {
- TableExtDesc tableExtDesc = metaMgr.getTableExt(table);
- String jobID = tableExtDesc.getJodID();
- if (null == jobID || ExecutableState.RUNNING != exeMgt.getOutput(jobID).getState()) {
- calculateCardinality(table, submitter);
- }
- }
- }
-
public void updateOnNewSegmentReady(String cubeName) {
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
String serverMode = kylinConfig.getServerMode();
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e2a81c/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
new file mode 100644
index 0000000..461800e
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.service;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
+import org.apache.kylin.engine.mr.common.MapReduceExecutable;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.job.execution.ExecutableManager;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableExtDesc;
+import org.apache.kylin.metadata.streaming.StreamingConfig;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.response.TableDescResponse;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.HiveSourceTableLoader;
+import org.apache.kylin.source.hive.IHiveClient;
+import org.apache.kylin.source.hive.cardinality.HiveColumnCardinalityJob;
+import org.apache.kylin.source.hive.cardinality.HiveColumnCardinalityUpdateJob;
+import org.apache.kylin.source.kafka.config.KafkaConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Component;
+
+@Component("tableService")
+public class TableService extends BasicService {
+
+ private static final Logger logger = LoggerFactory.getLogger(TableService.class);
+
+ @Autowired
+ private ModelService modelService;
+
+ @Autowired
+ private ProjectService projectService;
+
+ @Autowired
+ private StreamingService streamingService;
+
+ @Autowired
+ private KafkaConfigService kafkaConfigService;
+
+ public List<TableDesc> getTableDescByProject(String project, boolean withExt) throws IOException {
+ List<TableDesc> tables = getProjectManager().listDefinedTables(project);
+ if (null == tables) {
+ return Collections.emptyList();
+ }
+ if (withExt) {
+ tables = cloneTableDesc(tables);
+ }
+ return tables;
+ }
+
+ public TableDesc getTableDescByName(String tableName, boolean withExt) {
+ TableDesc table = getMetadataManager().getTableDesc(tableName);
+ if(withExt){
+ table = cloneTableDesc(table);
+ }
+ return table;
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
+ public String[] loadHiveTablesToProject(String[] tables, String project) throws IOException {
+ Set<String> loaded = HiveSourceTableLoader.loadHiveTables(tables, getConfig());
+ String[] result = (String[]) loaded.toArray(new String[loaded.size()]);
+ syncTableToProject(result, project);
+ return result;
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+ private void unLoadHiveTable(String tableName) throws IOException {
+ tableName = normalizeHiveTableName(tableName);
+ HiveSourceTableLoader.unLoadHiveTable(tableName.toUpperCase());
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+ private void syncTableToProject(String[] tables, String project) throws IOException {
+ getProjectManager().addTableDescToProject(tables, project);
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+ private void removeTableFromProject(String tableName, String projectName) throws IOException {
+ tableName = normalizeHiveTableName(tableName);
+ getProjectManager().removeTableDescFromProject(tableName, projectName);
+ }
+
+ /**
+ * table may referenced by several projects, and kylin only keep one copy of meta for each table,
+ * that's why we have two if statement here.
+ * @param tableName
+ * @param project
+ * @return
+ */
+ public boolean unLoadHiveTable(String tableName, String project) {
+ boolean rtn = false;
+ int tableType = 0;
+
+ //remove streaming info
+ tableName = normalizeHiveTableName(tableName);
+ TableDesc desc = getMetadataManager().getTableDesc(tableName);
+ if (desc == null)
+ return false;
+ tableType = desc.getSourceType();
+
+ try {
+ if (!modelService.isTableInModel(tableName, project)) {
+ removeTableFromProject(tableName, project);
+ rtn = true;
+ } else {
+ List<String> models = modelService.getModelsUsingTable(tableName, project);
+ throw new InternalErrorException("Table is already in use by models " + models);
+ }
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
+ if (!projectService.isTableInAnyProject(tableName) && !modelService.isTableInAnyModel(tableName)) {
+ try {
+ unLoadHiveTable(tableName);
+ rtn = true;
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ rtn = false;
+ }
+ }
+
+ if (tableType == 1 && !projectService.isTableInAnyProject(tableName) && !modelService.isTableInAnyModel(tableName)) {
+ StreamingConfig config = null;
+ KafkaConfig kafkaConfig = null;
+ try {
+ config = streamingService.getStreamingManager().getStreamingConfig(tableName);
+ kafkaConfig = kafkaConfigService.getKafkaConfig(tableName);
+ streamingService.dropStreamingConfig(config);
+ kafkaConfigService.dropKafkaConfig(kafkaConfig);
+ rtn = true;
+ } catch (Exception e) {
+ rtn = false;
+ logger.error(e.getLocalizedMessage(), e);
+ }
+ }
+ return rtn;
+ }
+
+ /**
+ *
+ * @param desc
+ * @param project
+ * @throws IOException
+ */
+ public void addStreamingTable(TableDesc desc, String project) throws IOException {
+ desc.setUuid(UUID.randomUUID().toString());
+ getMetadataManager().saveSourceTable(desc);
+ syncTableToProject(new String[] { desc.getIdentity() }, project);
+ }
+
+ /**
+ *
+ * @return
+ * @throws Exception
+ */
+ public List<String> getHiveDbNames() throws Exception {
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
+ List<String> results = hiveClient.getHiveDbNames();
+ return results;
+ }
+
+ /**
+ *
+ * @param database
+ * @return
+ * @throws Exception
+ */
+ public List<String> getHiveTableNames(String database) throws Exception {
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
+ List<String> results = hiveClient.getHiveTableNames(database);
+ return results;
+ }
+
+ private TableDescResponse cloneTableDesc(TableDesc table) {
+ TableExtDesc tableExtDesc = getMetadataManager().getTableExt(table.getIdentity());
+
+ // Clone TableDesc
+ TableDescResponse rtableDesc = new TableDescResponse(table);
+ Map<String, Long> cardinality = new HashMap<String, Long>();
+ Map<String, String> dataSourceProp = new HashMap<>();
+ String scard = tableExtDesc.getCardinality();
+ if (!StringUtils.isEmpty(scard)) {
+ String[] cards = StringUtils.split(scard, ",");
+ ColumnDesc[] cdescs = rtableDesc.getColumns();
+ for (int i = 0; i < cdescs.length; i++) {
+ ColumnDesc columnDesc = cdescs[i];
+ if (cards.length > i) {
+ cardinality.put(columnDesc.getName(), Long.parseLong(cards[i]));
+ } else {
+ logger.error("The result cardinality is not identical with hive table metadata, cardinality : " + scard + " column array length: " + cdescs.length);
+ break;
+ }
+ }
+ rtableDesc.setCardinality(cardinality);
+ }
+ dataSourceProp.putAll(tableExtDesc.getDataSourceProp());
+ dataSourceProp.put("location", tableExtDesc.getStorageLocation());
+ dataSourceProp.put("owner", tableExtDesc.getOwner());
+ dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime());
+ dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn());
+ dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize());
+ rtableDesc.setDescExd(dataSourceProp);
+ return rtableDesc;
+ }
+
+
+ private List<TableDesc> cloneTableDesc(List<TableDesc> tables) throws IOException {
+ List<TableDesc> descs = new ArrayList<TableDesc>();
+ Iterator<TableDesc> it = tables.iterator();
+ while (it.hasNext()) {
+ TableDesc table = it.next();
+ TableDescResponse rtableDesc = cloneTableDesc(table);
+ descs.add(rtableDesc);
+ }
+
+ return descs;
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
+ public void calculateCardinalityIfNotPresent(String[] tables, String submitter) throws IOException {
+ MetadataManager metaMgr = getMetadataManager();
+ ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig());
+ for (String table : tables) {
+ TableExtDesc tableExtDesc = metaMgr.getTableExt(table);
+ String jobID = tableExtDesc.getJodID();
+ if (null == jobID || ExecutableState.RUNNING != exeMgt.getOutput(jobID).getState()) {
+ calculateCardinality(table, submitter);
+ }
+ }
+ }
+
+ /**
+ * Generate cardinality for table This will trigger a hadoop job
+ * The result will be merged into table exd info
+ *
+ * @param tableName
+ */
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
+ public void calculateCardinality(String tableName, String submitter) throws IOException {
+ tableName = normalizeHiveTableName(tableName);
+ TableDesc table = getMetadataManager().getTableDesc(tableName);
+ final TableExtDesc tableExt = getMetadataManager().getTableExt(tableName);
+ if (table == null) {
+ IllegalArgumentException e = new IllegalArgumentException("Cannot find table descirptor " + tableName);
+ logger.error("Cannot find table descirptor " + tableName, e);
+ throw e;
+ }
+
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ //make sure the job could be scheduled when the DistributedScheduler is enable.
+ job.setParam("segmentId", tableName);
+ job.setName("Hive Column Cardinality calculation for table '" + tableName + "'");
+ job.setSubmitter(submitter);
+
+ String outPath = getConfig().getHdfsWorkingDirectory() + "cardinality/" + job.getId() + "/" + tableName;
+ String param = "-table " + tableName + " -output " + outPath;
+
+ MapReduceExecutable step1 = new MapReduceExecutable();
+
+ step1.setMapReduceJobClass(HiveColumnCardinalityJob.class);
+ step1.setMapReduceParams(param);
+ step1.setParam("segmentId", tableName);
+
+ job.addTask(step1);
+
+ HadoopShellExecutable step2 = new HadoopShellExecutable();
+
+ step2.setJobClass(HiveColumnCardinalityUpdateJob.class);
+ step2.setJobParams(param);
+ step2.setParam("segmentId", tableName);
+ job.addTask(step2);
+ tableExt.setJodID(job.getId());
+ getMetadataManager().saveTableExt(tableExt);
+
+ getExecutableManager().addJob(job);
+ }
+
+ public String normalizeHiveTableName(String tableName){
+ String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
+ return dbTableName[0] + "." + dbTableName[1];
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e2a81c/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 77e1084..b56009a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -51,7 +51,7 @@ public class HiveSourceTableLoader {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(HiveSourceTableLoader.class);
- public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException {
+ public static Set<String> loadHiveTables(String[] hiveTables, KylinConfig config) throws IOException {
SetMultimap<String, String> db2tables = LinkedHashMultimap.create();
for (String fullTableName : hiveTables) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e2a81c/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
index bbb9915..a53a35f 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -330,7 +330,7 @@ KylinApp
}
if ($scope.tableNames.trim() === "") {
- SweetAlert.swal('', 'Please input table(s) you want to synchronize.', 'info');
+ SweetAlert.swal('', 'Please input table(s) you want to load.', 'info');
return;
}
@@ -352,13 +352,13 @@ KylinApp
})
if (result['result.unloaded'].length != 0 && result['result.loaded'].length == 0) {
- SweetAlert.swal('Failed!', 'Failed to synchronize following table(s): ' + unloadedTableInfo, 'error');
+ SweetAlert.swal('Failed!', 'Failed to load following table(s): ' + unloadedTableInfo, 'error');
}
if (result['result.loaded'].length != 0 && result['result.unloaded'].length == 0) {
- SweetAlert.swal('Success!', 'The following table(s) have been successfully synchronized: ' + loadTableInfo, 'success');
+ SweetAlert.swal('Success!', 'The following table(s) have been successfully loaded: ' + loadTableInfo, 'success');
}
if (result['result.loaded'].length != 0 && result['result.unloaded'].length != 0) {
- SweetAlert.swal('Partial loaded!', 'The following table(s) have been successfully synchronized: ' + loadTableInfo + "\n\n Failed to synchronize following table(s):" + unloadedTableInfo, 'warning');
+ SweetAlert.swal('Partial loaded!', 'The following table(s) have been successfully loaded: ' + loadTableInfo + "\n\n Failed to load following table(s):" + unloadedTableInfo, 'warning');
}
loadingRequest.hide();
scope.aceSrcTbLoaded(true);
@@ -378,7 +378,7 @@ KylinApp
$scope.remove = function () {
if ($scope.tableNames.trim() === "") {
- SweetAlert.swal('', 'Please input table(s) you want to synchronize.', 'info');
+ SweetAlert.swal('', 'Please input table(s) you want to unload.', 'info');
return;
}
@@ -400,13 +400,13 @@ KylinApp
})
if (result['result.unload.fail'].length != 0 && result['result.unload.success'].length == 0) {
- SweetAlert.swal('Failed!', 'Failed to synchronize following table(s): ' + unRemovedTableInfo, 'error');
+ SweetAlert.swal('Failed!', 'Failed to unload following table(s): ' + unRemovedTableInfo, 'error');
}
if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length == 0) {
- SweetAlert.swal('Success!', 'The following table(s) have been successfully synchronized: ' + removedTableInfo, 'success');
+ SweetAlert.swal('Success!', 'The following table(s) have been successfully unloaded: ' + removedTableInfo, 'success');
}
if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length != 0) {
- SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully synchronized: ' + removedTableInfo + "\n\n Failed to synchronize following table(s):" + unRemovedTableInfo, 'warning');
+ SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully unloaded: ' + removedTableInfo + "\n\n Failed to unload following table(s):" + unRemovedTableInfo, 'warning');
}
loadingRequest.hide();
scope.aceSrcTbLoaded(true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e2a81c/webapp/app/js/services/tables.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/tables.js b/webapp/app/js/services/tables.js
index 4199d6c..4e7a7c4 100755
--- a/webapp/app/js/services/tables.js
+++ b/webapp/app/js/services/tables.js
@@ -24,7 +24,6 @@ KylinApp.factory('TableService', ['$resource', function ($resource, config) {
reload: {method: 'PUT', params: {action: 'reload'}, isArray: false},
loadHiveTable: {method: 'POST', params: {}, isArray: false},
unLoadHiveTable: {method: 'DELETE', params: {}, isArray: false},
- addStreamingSrc: {method: 'POST', params: {action:'addStreamingSrc'}, isArray: false},
genCardinality: {method: 'PUT', params: {action: 'cardinality'}, isArray: false},
showHiveDatabases: {method: 'GET', params: {action:'hive'}, cache: true, isArray: true},
showHiveTables: {method: 'GET', params: {action:'hive'}, cache: true, isArray: true}
[08/42] kylin git commit: introduce CheckUtil to make multiple
condition checks clearer in log
Posted by sh...@apache.org.
introduce CheckUtil to make multiple condition checks clearer in log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2436140e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2436140e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2436140e
Branch: refs/heads/sparkcubing-rebase
Commit: 2436140ef093c8d5b061124a5dbc2f5a1c3e9f3f
Parents: 182c565
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:15:44 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/CheckUtil.java | 33 ++++++++++++++++++++
.../apache/kylin/rest/service/QueryService.java | 17 ++++++----
2 files changed, 44 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2436140e/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 0000000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+ public static final org.slf4j.Logger logger = LoggerFactory.getLogger(CheckUtil.class);
+
+ public static boolean checkCondition(boolean condition, String message, Object... args) {
+ if (condition) {
+ return true;
+ } else {
+ logger.debug(message, args);
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2436140e/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.service;
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
long startTime = System.currentTimeMillis();
SQLResponse sqlResponse = null;
- boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() && !BackdoorToggles.getDisableCache();
+ boolean queryCacheEnabled = checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in KylinConfig") && //
+ checkCondition(!BackdoorToggles.getDisableCache(), "query cache disabled in BackdoorToggles");
+
if (queryCacheEnabled) {
sqlResponse = searchQueryInCache(sqlRequest);
}
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
sqlResponse.setDuration(System.currentTimeMillis() - startTime);
logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", //
String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()));
- if (queryCacheEnabled && //
- !sqlResponse.getIsException() && //
- (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold) && //
- (sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold())) { //don't cache too large response
+ if (checkCondition(queryCacheEnabled, "query cache is disabled") && //
+ checkCondition(!sqlResponse.getIsException(), "query has exception") && //
+ checkCondition(sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold, "query is too lightweight with duration: {} ({}), scan count: {} ({})", sqlResponse.getDuration(), durationThreshold, sqlResponse.getTotalScanCount(), scancountThreshold) && //
+ checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(), "query response is too large: {} ({})", sqlResponse.getResults().size(), kylinConfig.getLargeQueryThreshold())) {
cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse));
}
+
} else {
sqlResponse.setDuration(System.currentTimeMillis() - startTime);
}
@@ -437,7 +442,7 @@ public class QueryService extends BasicService {
String correctedSql = QueryUtil.massageSql(sqlRequest);
if (!correctedSql.equals(sqlRequest.getSql())) {
logger.info("The corrected query: " + correctedSql);
-
+
//CAUTION: should not change sqlRequest content!
//sqlRequest.setSql(correctedSql);
}
[05/42] kylin git commit: minor, fix UT
Posted by sh...@apache.org.
minor, fix UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f9a8402d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f9a8402d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f9a8402d
Branch: refs/heads/sparkcubing-rebase
Commit: f9a8402d57755a9c3c3132ff676cf3c7492775c9
Parents: a78b650
Author: Billy Liu <bi...@apache.org>
Authored: Tue Dec 27 00:02:35 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Tue Dec 27 00:02:35 2016 +0800
----------------------------------------------------------------------
.../kylin/dict/DictionaryProviderTest.java | 36 +++++++++++++++++---
1 file changed, 32 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f9a8402d/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
index a4aee76..4b386a7 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.dict;
import static org.junit.Assert.assertTrue;
@@ -13,13 +31,23 @@ import java.util.Iterator;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
-/**
- * Created by xiefan on 16-11-23.
- */
-public class DictionaryProviderTest {
+public class DictionaryProviderTest extends LocalFileMetadataTestCase{
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
@Test
public void testReadWrite() throws Exception {
[33/42] kylin git commit: enhance LookupTable
Posted by sh...@apache.org.
enhance LookupTable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64c3c618
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64c3c618
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64c3c618
Branch: refs/heads/sparkcubing-rebase
Commit: 64c3c61898d10f05c4350609517b2096c4171167
Parents: 8cd416a
Author: Li Yang <li...@apache.org>
Authored: Thu Dec 29 13:44:40 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 29 18:26:59 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/dict/lookup/LookupTable.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/64c3c618/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index cd700e9..21221f4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -46,7 +47,7 @@ abstract public class LookupTable<T> {
protected TableDesc tableDesc;
protected String[] keyColumns;
protected ReadableTable table;
- protected ConcurrentHashMap<Array<T>, T[]> data;
+ protected Map<Array<T>, T[]> data;
public LookupTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table) throws IOException {
this.tableDesc = tableDesc;
[19/42] kylin git commit: KYLIN-2318 bug fix again...
Posted by sh...@apache.org.
KYLIN-2318 bug fix again...
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9898a07b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9898a07b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9898a07b
Branch: refs/heads/sparkcubing-rebase
Commit: 9898a07b1451c29b25fb295b22ed62015073df93
Parents: cccf3a9
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Dec 28 10:20:31 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Dec 28 10:21:28 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/service/QueryService.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9898a07b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index de28b17..bc644cc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,7 +327,8 @@ public class QueryService extends BasicService {
}
final String queryId = UUID.randomUUID().toString();
- BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
+ if (sqlRequest.getBackdoorToggles() != null)
+ BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
QueryContext.setQueryId(queryId);
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
[11/42] kylin git commit: minor code refactors
Posted by sh...@apache.org.
minor code refactors
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa4a5ee2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa4a5ee2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa4a5ee2
Branch: refs/heads/sparkcubing-rebase
Commit: fa4a5ee269eea66eee5b8ab8278305dfe0144c92
Parents: 00de441
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Dec 22 09:59:43 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:15:44 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 6 ++-
.../org/apache/kylin/common/util/BasicTest.java | 1 +
.../gtrecord/DummyPartitionStreamer.java | 40 ++++++++++++++++++++
.../storage/gtrecord/IPartitionStreamer.java | 26 +++++++++++++
.../gtrecord/StorageResponseGTScatter.java | 9 +++--
.../apache/kylin/rest/service/QueryService.java | 6 ++-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 9 +++--
7 files changed, 87 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8080577..d73b694 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -750,7 +750,7 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true"));
}
- @Deprecated//Limit is good even it's large. This config is meaning less since we already have scan threshold
+ @Deprecated //Limit is good even it's large. This config is meaning less since we already have scan threshold
public int getStoragePushDownLimitMax() {
return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000"));
}
@@ -759,6 +759,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000"));
}
+ public int getLargeQueryThreshold() {
+ return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf((int) (getScanThreshold() * 0.1))));
+ }
+
public int getDerivedInThreshold() {
return Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold", "20"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5eaa011..9105245 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import com.google.common.collect.Iterators;
import org.apache.commons.lang3.time.FastDateFormat;
import org.junit.Ignore;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
new file mode 100644
index 0000000..4caaed0
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+public class DummyPartitionStreamer implements IPartitionStreamer {
+ private Iterator<byte[]> iterator;
+
+ public DummyPartitionStreamer(Iterator<byte[]> iterator) {
+ this.iterator = iterator;
+ }
+
+ @Override
+ public void close() throws IOException {
+ //do nothing
+ }
+
+ @Override
+ public Iterator<byte[]> asByteArrayIterator() {
+ return this.iterator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java
new file mode 100644
index 0000000..42f1151
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import java.io.Closeable;
+import java.util.Iterator;
+
+public interface IPartitionStreamer extends Closeable {
+ public Iterator<byte[]> asByteArrayIterator();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index fe1afd3..6283340 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -42,14 +42,16 @@ public class StorageResponseGTScatter implements IGTScanner {
private static final Logger logger = LoggerFactory.getLogger(StorageResponseGTScatter.class);
private GTInfo info;
+ private IPartitionStreamer partitionStreamer;
private Iterator<byte[]> blocks;
private ImmutableBitSet columns;
private long totalScannedCount;
private int storagePushDownLimit = -1;
- public StorageResponseGTScatter(GTInfo info, Iterator<byte[]> blocks, ImmutableBitSet columns, long totalScannedCount, int storagePushDownLimit) {
+ public StorageResponseGTScatter(GTInfo info, IPartitionStreamer partitionStreamer, ImmutableBitSet columns, long totalScannedCount, int storagePushDownLimit) {
this.info = info;
- this.blocks = blocks;
+ this.partitionStreamer = partitionStreamer;
+ this.blocks = partitionStreamer.asByteArrayIterator();
this.columns = columns;
this.totalScannedCount = totalScannedCount;
this.storagePushDownLimit = storagePushDownLimit;
@@ -67,7 +69,8 @@ public class StorageResponseGTScatter implements IGTScanner {
@Override
public void close() throws IOException {
- //do nothing
+ //If upper consumer failed while consuming the GTRecords, the consumer should call IGTScanner's close method to ensure releasing resource
+ partitionStreamer.close();
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 8810c85..82e4a87 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -360,8 +360,10 @@ public class QueryService extends BasicService {
sqlResponse.setDuration(System.currentTimeMillis() - startTime);
logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", //
String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()));
- if (queryCacheEnabled && !sqlResponse.getIsException() //
- && (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold)) {
+ if (queryCacheEnabled && //
+ !sqlResponse.getIsException() && //
+ (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold) && //
+ (sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold())) { //don't cache too large response
cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse));
}
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index d99f80e..ebacb26 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -40,12 +40,13 @@ import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LoggableCachedThreadPool;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.metadata.model.ISegment;
+import org.apache.kylin.storage.gtrecord.DummyPartitionStreamer;
import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
@@ -127,14 +128,14 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
//TODO: raw scan can be constructed at region side to reduce traffic
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
rawScanByteString = serializeRawScans(rawScans);
-
+
int coprocessorTimeout = getCoprocessorTimeoutMillis();
scanRequest.setTimeout(coprocessorTimeout);
scanRequest.clearScanRanges();//since raw scans are sent to coprocessor, we don't need to duplicate sending it
scanRequestByteString = serializeGTScanReq(scanRequest);
final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(shardNum, coprocessorTimeout);
-
+
logger.info("Serialized scanRequestBytes {} bytes, rawScanBytesString {} bytes", scanRequestByteString.size(), rawScanByteString.size());
logger.info("The scan {} for segment {} is as below with {} separate raw scans, shard part of start/end key is set to 0", Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg, rawScans.size());
@@ -230,7 +231,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
});
}
- return new StorageResponseGTScatter(fullGTInfo, epResultItr, scanRequest.getColumns(), totalScannedCount.get(), scanRequest.getStoragePushDownLimit());
+ return new StorageResponseGTScatter(fullGTInfo, new DummyPartitionStreamer(epResultItr), scanRequest.getColumns(), totalScannedCount.get(), scanRequest.getStoragePushDownLimit());
}
private ByteString serializeGTScanReq(GTScanRequest scanRequest) {
[21/42] kylin git commit: minor set session expire time to 8 hours by
deault
Posted by sh...@apache.org.
minor set session expire time to 8 hours by deault
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8747e24d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8747e24d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8747e24d
Branch: refs/heads/sparkcubing-rebase
Commit: 8747e24da5053cc8bda8ec27b48c01bd97b7c63b
Parents: 7aabe4c
Author: zhongjian <ji...@163.com>
Authored: Wed Dec 28 14:55:50 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Wed Dec 28 14:55:50 2016 +0800
----------------------------------------------------------------------
server/src/main/webapp/WEB-INF/web.xml | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8747e24d/server/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml
index 8a43b33..b9cf620 100644
--- a/server/src/main/webapp/WEB-INF/web.xml
+++ b/server/src/main/webapp/WEB-INF/web.xml
@@ -30,6 +30,10 @@
<location>/index.html</location>
</error-page>
+ <session-config>
+ <session-timeout>480</session-timeout>
+ </session-config>
+
<context-param>
<param-name>log4jConfigLocation</param-name>
<param-value>classpath:kylin-server-log4j.properties</param-value>
[18/42] kylin git commit: fix NPE in
OLAPEnumerator.setConnectionProperties()
Posted by sh...@apache.org.
fix NPE in OLAPEnumerator.setConnectionProperties()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cccf3a9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cccf3a9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cccf3a9c
Branch: refs/heads/sparkcubing-rebase
Commit: cccf3a9c1a8dd0c7ac326c958aafab050a3886cb
Parents: 84ec908
Author: Li Yang <li...@apache.org>
Authored: Wed Dec 28 09:45:18 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Dec 28 09:45:18 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cccf3a9c/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index e4fc937..f012638 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -25,6 +25,7 @@ import java.util.Properties;
import org.apache.calcite.DataContext;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.linq4j.Enumerator;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -161,8 +162,10 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
Properties connProps = conn.getProperties();
String propThreshold = connProps.getProperty(OLAPQuery.PROP_SCAN_THRESHOLD);
- int threshold = Integer.valueOf(propThreshold);
- olapContext.storageContext.setThreshold(threshold);
+ if (!StringUtils.isBlank(propThreshold)) {
+ int threshold = Integer.valueOf(propThreshold);
+ olapContext.storageContext.setThreshold(threshold);
+ }
}
}
[42/42] kylin git commit: KYLIN-2331 by layer spark cubing
Posted by sh...@apache.org.
KYLIN-2331 by layer spark cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cf6bba8b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cf6bba8b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cf6bba8b
Branch: refs/heads/sparkcubing-rebase
Commit: cf6bba8b039a27eb24edcab8689f61b9a09d3a7f
Parents: a9ee6d3
Author: shaofengshi <sh...@apache.org>
Authored: Sun Dec 25 15:59:16 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Dec 30 10:30:20 2016 +0800
----------------------------------------------------------------------
assembly/pom.xml | 5 +-
.../main/config/assemblies/source-assembly.xml | 3 -
.../java/org/apache/kylin/job/DeployUtil.java | 1 -
build/conf/kylin-spark-conf.properties | 26 ++
build/conf/kylin.properties | 20 +
.../apache/kylin/common/KylinConfigBase.java | 64 ++-
.../kylin/common/persistence/ResourceStore.java | 35 ++
.../org/apache/kylin/common/util/Array.java | 2 +-
.../apache/kylin/common/util/SplittedBytes.java | 2 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 2 +-
.../kylin/cube/common/RowKeySplitter.java | 15 +-
.../org/apache/kylin/cube/cuboid/Cuboid.java | 27 +-
.../kylin/cube/cuboid/CuboidScheduler.java | 15 +-
.../cube/inmemcubing/InMemCubeBuilder.java | 2 +
.../kylin/cube/kv/AbstractRowKeyEncoder.java | 6 +-
.../org/apache/kylin/cube/kv/CubeDimEncMap.java | 7 +-
.../apache/kylin/cube/kv/RowKeyColumnIO.java | 2 +-
.../org/apache/kylin/cube/kv/RowKeyEncoder.java | 19 +-
.../kylin/cube/kv/RowKeyEncoderProvider.java | 2 +-
.../kylin/cube/model/AggregationGroup.java | 4 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
.../cube/model/CubeJoinedFlatTableDesc.java | 2 +-
.../cube/model/CubeJoinedFlatTableEnrich.java | 6 +-
.../apache/kylin/cube/model/DictionaryDesc.java | 2 +-
.../apache/kylin/cube/model/DimensionDesc.java | 13 +-
.../kylin/cube/model/HBaseColumnDesc.java | 11 +-
.../kylin/cube/model/HBaseColumnFamilyDesc.java | 9 +-
.../kylin/cube/model/HBaseMappingDesc.java | 15 +-
.../apache/kylin/cube/model/HierarchyDesc.java | 2 +-
.../apache/kylin/cube/model/RowKeyColDesc.java | 2 +-
.../org/apache/kylin/cube/model/RowKeyDesc.java | 17 +-
.../org/apache/kylin/cube/model/SelectRule.java | 2 +-
.../apache/kylin/gridtable/UnitTestSupport.java | 41 +-
.../gridtable/AggregationCacheMemSizeTest.java | 6 +-
.../gridtable/AggregationCacheSpillTest.java | 5 +-
.../kylin/gridtable/SimpleGridTableTest.java | 11 +-
.../gridtable/SimpleInvertedIndexTest.java | 3 +-
.../metadata/measure/MeasureCodecTest.java | 6 +-
.../apache/kylin/dict/AppendTrieDictionary.java | 6 +-
.../org/apache/kylin/dict/NumberDictionary.java | 2 +-
.../apache/kylin/dict/NumberDictionary2.java | 2 +-
.../dict/NumberDictionaryForestBuilder.java | 4 +-
.../apache/kylin/dict/StringBytesConverter.java | 2 +-
.../kylin/job/execution/ExecutableManager.java | 21 +-
.../kylin/dimension/AbstractDateDimEnc.java | 12 +-
.../apache/kylin/dimension/BooleanDimEnc.java | 4 +-
.../kylin/dimension/DictionaryDimEnc.java | 21 +-
.../apache/kylin/dimension/FixedLenDimEnc.java | 4 +-
.../kylin/dimension/FixedLenHexDimEnc.java | 4 +-
.../org/apache/kylin/dimension/IntDimEnc.java | 4 +-
.../apache/kylin/dimension/IntegerDimEnc.java | 4 +-
.../kylin/dimension/OneMoreByteVLongDimEnc.java | 14 +-
.../kylin/measure/BufferedMeasureCodec.java | 10 +-
.../apache/kylin/measure/MeasureAggregator.java | 2 +
.../kylin/measure/MeasureAggregators.java | 18 +
.../org/apache/kylin/measure/MeasureCodec.java | 8 +-
.../apache/kylin/measure/MeasureIngester.java | 12 +-
.../org/apache/kylin/measure/MeasureType.java | 12 +-
.../measure/basic/BigDecimalMaxAggregator.java | 14 +
.../measure/basic/BigDecimalMinAggregator.java | 28 +-
.../measure/basic/BigDecimalSumAggregator.java | 10 +
.../kylin/measure/basic/DoubleIngester.java | 19 +-
.../measure/basic/DoubleMaxAggregator.java | 20 +-
.../measure/basic/DoubleMinAggregator.java | 20 +-
.../measure/basic/DoubleSumAggregator.java | 18 +-
.../kylin/measure/basic/LongIngester.java | 19 +-
.../kylin/measure/basic/LongMaxAggregator.java | 20 +-
.../kylin/measure/basic/LongMinAggregator.java | 20 +-
.../kylin/measure/basic/LongSumAggregator.java | 18 +-
.../kylin/measure/bitmap/BitmapAggregator.java | 13 +
.../kylin/measure/bitmap/BitmapCounter.java | 8 +-
.../kylin/measure/bitmap/BitmapMeasureType.java | 5 +
.../kylin/measure/bitmap/BitmapSerializer.java | 2 +-
.../ExtendedColumnMeasureType.java | 15 +
.../kylin/measure/hllc/DenseRegister.java | 2 +-
.../kylin/measure/hllc/HLLCAggregator.java | 7 +
.../kylin/measure/hllc/HLLCMeasureType.java | 5 +
.../kylin/measure/hllc/HLLCSerializer.java | 11 +-
.../measure/hllc/HyperLogLogPlusTable.java | 2 +-
.../kylin/measure/hllc/SingleValueRegister.java | 2 +-
.../kylin/measure/hllc/SparseRegister.java | 2 +-
.../apache/kylin/measure/raw/RawAggregator.java | 14 +
.../apache/kylin/measure/raw/RawSerializer.java | 4 +-
.../measure/topn/DoubleDeltaSerializer.java | 7 +-
.../kylin/measure/topn/TopNAggregator.java | 9 +
.../apache/kylin/measure/topn/TopNCounter.java | 2 +-
.../kylin/measure/topn/TopNMeasureType.java | 4 +-
.../metadata/datatype/BigDecimalSerializer.java | 2 +-
.../metadata/datatype/BooleanSerializer.java | 36 +-
.../metadata/datatype/DataTypeSerializer.java | 17 +-
.../metadata/datatype/DateTimeSerializer.java | 30 +-
.../metadata/datatype/DoubleSerializer.java | 28 +-
.../kylin/metadata/datatype/Int4Serializer.java | 32 +-
.../metadata/datatype/Long8Serializer.java | 31 +-
.../kylin/metadata/datatype/LongSerializer.java | 32 +-
.../kylin/metadata/model/FunctionDesc.java | 22 +-
.../apache/kylin/metadata/model/JoinDesc.java | 7 +-
.../kylin/metadata/model/JoinTableDesc.java | 4 +-
.../apache/kylin/metadata/model/JoinsTree.java | 9 +-
.../kylin/metadata/model/MeasureDesc.java | 9 +-
.../metadata/model/ModelDimensionDesc.java | 10 +-
.../kylin/metadata/model/ParameterDesc.java | 13 +-
.../kylin/metadata/model/PartitionDesc.java | 6 +-
.../apache/kylin/metadata/model/TableRef.java | 17 +-
.../measure/AggregatorMemEstimateTest.java | 6 +-
.../storage/gtrecord/DictGridTableTest.java | 41 +-
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 9 +-
.../org/apache/kylin/engine/mr/HadoopUtil.java | 2 +-
.../kylin/engine/mr/JobBuilderSupport.java | 12 +-
.../engine/mr/common/BaseCuboidBuilder.java | 173 ++++++++
.../kylin/engine/mr/common/CubeStatsReader.java | 5 +
.../kylin/engine/mr/common/NDCuboidBuilder.java | 96 +++++
.../engine/mr/steps/BaseCuboidMapperBase.java | 131 +-----
.../kylin/engine/mr/steps/NDCuboidMapper.java | 58 +--
.../kylin/engine/mr/steps/CubeReducerTest.java | 3 +-
engine-spark/pom.xml | 9 +
.../engine/spark/SparkBatchCubingEngine2.java | 33 ++
.../spark/SparkBatchCubingJobBuilder2.java | 85 ++++
.../apache/kylin/engine/spark/SparkCubing.java | 88 +++-
.../kylin/engine/spark/SparkCubingByLayer.java | 416 +++++++++++++++++++
.../kylin/engine/spark/SparkExecutable.java | 26 +-
.../spark/cube/DefaultTupleConverter.java | 2 +-
.../sandbox/kylin-spark-conf.properties | 28 ++
.../test_case_data/sandbox/kylin.properties | 10 +
pom.xml | 2 +-
.../kylin/rest/controller/CubeController.java | 8 +-
server/pom.xml | 7 +
.../apache/kylin/source/hive/HiveMRInput.java | 5 +-
.../hbase/steps/RowValueDecoderTest.java | 7 +-
tool/pom.xml | 4 +
130 files changed, 1738 insertions(+), 733 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 65e3a8c..873abff 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -47,6 +47,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-engine-mr</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-engine-spark</artifactId>
+ </dependency>
<!-- Env & Test -->
<dependency>
@@ -173,7 +177,6 @@
<shadedClassifierName>job</shadedClassifierName>
<artifactSet>
<excludes>
- <exclude>io.netty:*</exclude>
<exclude>org.apache.zookeeper:*</exclude>
<exclude>net.sf.ehcache:*</exclude>
<exclude>org.apache.httpcomponents:*</exclude>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/assembly/src/main/config/assemblies/source-assembly.xml
----------------------------------------------------------------------
diff --git a/assembly/src/main/config/assemblies/source-assembly.xml b/assembly/src/main/config/assemblies/source-assembly.xml
index fad45aa..92584b5 100644
--- a/assembly/src/main/config/assemblies/source-assembly.xml
+++ b/assembly/src/main/config/assemblies/source-assembly.xml
@@ -97,9 +97,6 @@ limitations under the License.
<exclude>%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?docs(/.*)?]
</exclude>
- <!-- exclude unmaintained -->
- <exclude>%regex[(?!((?!${project.build.directory}/)))?engine-spark(/.*)?]
- </exclude>
</excludes>
</fileSet>
<!-- LICENSE, NOTICE, DEPENDENCIES, git.properties, etc. calculated at build time -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index a42b125..e8c7fae 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -82,7 +82,6 @@ public class DeployUtil {
config().overrideMRJobJarPath(jobJar.getAbsolutePath());
config().overrideCoprocessorLocalJar(coprocessorJar.getAbsolutePath());
- config().overrideSparkJobJarPath(getSparkJobJarFile().getAbsolutePath());
}
private static String getPomVersion() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/build/conf/kylin-spark-conf.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin-spark-conf.properties b/build/conf/kylin-spark-conf.properties
new file mode 100644
index 0000000..81567bb
--- /dev/null
+++ b/build/conf/kylin-spark-conf.properties
@@ -0,0 +1,26 @@
+spark.yarn.submit.file.replication=1
+spark.yarn.executor.memoryOverhead=200
+spark.yarn.driver.memoryOverhead=384
+spark.master=yarn
+spark.submit.deployMode=cluster
+spark.eventLog.enabled=true
+spark.yarn.scheduler.heartbeat.interval-ms=5000
+spark.yarn.preserve.staging.files=true
+spark.yarn.queue=default
+spark.yarn.containerLauncherMaxThreads=25
+spark.yarn.max.executor.failures=3
+spark.eventLog.dir=hdfs\:///kylin/spark-history
+spark.history.kerberos.enabled=true
+spark.history.provider=org.apache.spark.deploy.history.FsHistoryProvider
+spark.history.ui.port=18080
+spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+spark.executor.memory=4G
+spark.storage.memoryFraction=0.3
+spark.executor.cores=4
+spark.executor.instances=8
+spark.history.kerberos.keytab=none
+spark.history.kerberos.principal=none
+#spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+spark.driver.extraJavaOptions=-Dhdp.version=current
+spark.yarn.am.extraJavaOptions=-Dhdp.version=current
+spark.executor.extraJavaOptions=-Dhdp.version=current
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 5e7d235..98b66cb 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -128,6 +128,26 @@ kylin.engine.mr.max-reducer-number=500
kylin.engine.mr.mapper-input-rows=1000000
+### Spark Engine ###
+
+# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit
+kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+
+# Spark install home, default be $KYLIN_HOME/spark/
+#kylin.engine.spark.spark-home=
+
+# Spark job submission properties file, default be $KYLIN_HOME/conf/kylin-spark-conf.properties
+#kylin.engine.spark.properties-file=
+
+# Estimate the RDD partition numbers
+kylin.engine.spark.rdd-partition-cut-mb=200
+
+# Minimal partition numbers of rdd
+kylin.engine.spark.min-partition=1
+
+# Max partition numbers of rdd
+kylin.engine.spark.max-partition=500
+
### CUBE | DICTIONARY ###
# 'auto', 'inmem', 'layer' or 'random' for testing
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d73b694..b0b18ce 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -640,6 +640,7 @@ abstract public class KylinConfigBase implements Serializable {
// ref constants in IEngineAware
r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine");
r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2");
+ r.put(4, "org.apache.kylin.engine.spark.SparkBatchCubingEngine2");
return r;
}
@@ -716,29 +717,62 @@ abstract public class KylinConfigBase implements Serializable {
// ENGINE.SPARK
// ============================================================================
- public String getKylinSparkJobJarPath() {
- final String jobJar = getOptional("kylin.engine.spark.job-jar");
- if (StringUtils.isNotEmpty(jobJar)) {
- return jobJar;
+ public String getSparkHome() {
+ String sparkHome = getOptional("kylin.engine.spark.spark-home", "spark");
+ File f = new File(sparkHome);
+ if (f.exists()) {
+ return f.getAbsolutePath();
+ } else {
+ String home = getKylinHome();
+ f = new File(home, sparkHome);
+ if (f.exists()) {
+ return f.getAbsolutePath();
+ }
}
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome)) {
- return "";
+
+ throw new IllegalArgumentException("Spark home '" + sparkHome + "' does not exist, check 'kylin.engine.spark.spark-home' in kylin.properties");
+
+ }
+
+ public String getSparkHadoopConfDir() {
+ return getRequired("kylin.engine.spark.env.hadoop-conf-dir");
+ }
+
+ public String getSparkConfFile() {
+ String conf = getOptional("kylin.engine.spark.properties-file", "conf/kylin-spark-conf.properties");
+ File f = new File(conf);
+ if (f.exists()) {
+ return f.getAbsolutePath();
+ } else {
+ String home = getKylinHome();
+ f = new File(home, conf);
+ if (f.exists()) {
+ return f.getAbsolutePath();
+ }
}
- return getFileName(kylinHome + File.separator + "lib", SPARK_JOB_JAR_NAME_PATTERN);
+
+ throw new IllegalArgumentException("Spark conf properties file '" + conf + "' does not exist.");
}
- public void overrideSparkJobJarPath(String path) {
- logger.info("override " + "kylin.engine.spark.job-jar" + " to " + path);
- System.setProperty("kylin.engine.spark.job-jar", path);
+ public String getSparkAdditionalJars() {
+ return getOptional("kylin.engine.spark.additional-jars", "");
}
- public String getSparkHome() {
- return getRequired("kylin.engine.spark.spark-home");
+ public float getSparkRDDPartitionCutMB() {
+ return Float.valueOf(getOptional("kylin.engine.spark.rdd-partition-cut-mb", "200.0"));
+ }
+
+
+ public int getSparkMinPartition() {
+ return Integer.valueOf(getOptional("kylin.engine.spark.min-partition", "1"));
+ }
+
+ public int getSparkMaxPartition() {
+ return Integer.valueOf(getOptional("kylin.engine.spark.max-partition", "500"));
}
- public String getSparkMaster() {
- return getRequired("kylin.engine.spark.spark-master");
+ public boolean isSparkSanityCheckEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.engine.spark.sanity-check-enabled", "false"));
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 0580576..9549569 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -21,17 +21,21 @@ package org.apache.kylin.common.persistence;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.OptionsHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -311,4 +315,35 @@ abstract public class ResourceStore {
return collector;
}
+ public static String dumpResources(KylinConfig kylinConfig, Collection<String> dumpList) throws IOException {
+ File tmp = File.createTempFile("kylin_job_meta", "");
+ FileUtils.forceDelete(tmp); // we need a directory, so delete the file first
+
+ File metaDir = new File(tmp, "meta");
+ metaDir.mkdirs();
+
+ // write kylin.properties
+ File kylinPropsFile = new File(metaDir, "kylin.properties");
+ kylinConfig.writeProperties(kylinPropsFile);
+
+ ResourceStore from = ResourceStore.getStore(kylinConfig);
+ KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath());
+ ResourceStore to = ResourceStore.getStore(localConfig);
+ for (String path : dumpList) {
+ RawResource res = from.getResource(path);
+ if (res == null)
+ throw new IllegalStateException("No resource found at -- " + path);
+ to.putResource(path, res.inputStream, res.timestamp);
+ res.inputStream.close();
+ }
+
+ String metaDirURI = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+ if (metaDirURI.startsWith("/")) // note Path on windows is like "d:/../..."
+ metaDirURI = "file://" + metaDirURI;
+ else
+ metaDirURI = "file:///" + metaDirURI;
+ logger.info("meta dir is: " + metaDirURI);
+
+ return metaDirURI;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-common/src/main/java/org/apache/kylin/common/util/Array.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Array.java b/core-common/src/main/java/org/apache/kylin/common/util/Array.java
index 7447b46..b25b764 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Array.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Array.java
@@ -23,7 +23,7 @@ import java.util.Arrays;
/*
* An array with correct equals(), hashCode(), compareTo() and toString()
*/
-public class Array<T> implements Comparable<Array<T>> {
+public class Array<T> implements Comparable<Array<T>>, java.io.Serializable {
public T[] data;
public Array(T[] data) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java b/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
index 8751b78..ae380cb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
@@ -22,7 +22,7 @@ package org.apache.kylin.common.util;
* @author George Song (ysong1)
*
*/
-public class SplittedBytes {
+public class SplittedBytes implements java.io.Serializable {
public SplittedBytes(int length) {
this.value = new byte[length];
this.length = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 36a6044..45310f0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -51,7 +51,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment {
+public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment, java.io.Serializable {
@JsonBackReference
private CubeInstance cubeInstance;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
index 67f1751..acebce4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
@@ -26,9 +26,11 @@ import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.RowKeyColDesc;
+import org.apache.kylin.dimension.IDimensionEncodingMap;
import org.apache.kylin.metadata.model.TblColRef;
-public class RowKeySplitter {
+public class RowKeySplitter implements java.io.Serializable {
private CubeDesc cubeDesc;
private RowKeyColumnIO colIO;
@@ -37,7 +39,6 @@ public class RowKeySplitter {
private int[] splitOffsets;
private int bufferSize;
- private long lastSplittedCuboidId;
private boolean enableSharding;
private short shardId;
@@ -64,7 +65,13 @@ public class RowKeySplitter {
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
this.enableSharding = cubeSeg.isEnableSharding();
this.cubeDesc = cubeSeg.getCubeDesc();
- this.colIO = new RowKeyColumnIO(new CubeDimEncMap(cubeSeg));
+ IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg);
+
+ for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
+ dimEncoding.get(rowKeyColDesc.getColRef());
+ }
+
+ this.colIO = new RowKeyColumnIO(dimEncoding);
this.splitBuffers = new SplittedBytes[splitLen];
this.splitOffsets = new int[splitLen];
@@ -105,7 +112,7 @@ public class RowKeySplitter {
System.arraycopy(bytes, offset, cuboidIdSplit.value, 0, RowConstants.ROWKEY_CUBOIDID_LEN);
offset += RowConstants.ROWKEY_CUBOIDID_LEN;
- lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length);
+ long lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length);
Cuboid cuboid = Cuboid.findById(cubeDesc, lastSplittedCuboidId);
// rowkey columns
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 7503fbf..dd22d6a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -18,15 +18,10 @@
package org.apache.kylin.cube.cuboid;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
@@ -37,12 +32,16 @@ import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.ComparisonChain;
-import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
-public class Cuboid implements Comparable<Cuboid> {
+public class Cuboid implements Comparable<Cuboid>, java.io.Serializable {
private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index 733aded..ffb0a5e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -21,6 +21,12 @@ package org.apache.kylin.cube.cuboid;
/**
*/
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.CubeDesc;
+
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -28,14 +34,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.kylin.cube.model.AggregationGroup;
-import org.apache.kylin.cube.model.CubeDesc;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-public class CuboidScheduler {
+public class CuboidScheduler implements java.io.Serializable {
private final CubeDesc cubeDesc;
private final long max;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 66a444e..b4cba39 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -462,6 +462,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
for (int i = 0; i < totalSum.length; i++) {
if (totalSum[i] instanceof DoubleMutable) {
totalSum[i] = Math.round(((DoubleMutable) totalSum[i]).get());
+ } else if (totalSum[i] instanceof Double) {
+ totalSum[i] = Math.round(((Double) totalSum[i]).doubleValue());
} else if (totalSum[i] instanceof TopNCounter) {
TopNCounter counter = (TopNCounter) totalSum[i];
Iterator<Counter> iterator = counter.iterator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
index bfe6eb4..2becde4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
@@ -18,8 +18,6 @@
package org.apache.kylin.cube.kv;
-import java.util.Map;
-
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
@@ -30,12 +28,14 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Map;
+
/**
*
* @author xjiang
*
*/
-public abstract class AbstractRowKeyEncoder {
+public abstract class AbstractRowKeyEncoder implements java.io.Serializable {
protected static final Logger logger = LoggerFactory.getLogger(AbstractRowKeyEncoder.class);
public static final byte DEFAULT_BLANK_BYTE = DimensionEncoding.NULL;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
index a4d2d6f..bd9554a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
@@ -18,8 +18,7 @@
package org.apache.kylin.cube.kv;
-import java.util.Map;
-
+import com.google.common.collect.Maps;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
@@ -33,9 +32,9 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Maps;
+import java.util.Map;
-public class CubeDimEncMap implements IDimensionEncodingMap {
+public class CubeDimEncMap implements IDimensionEncodingMap, java.io.Serializable {
private static final Logger logger = LoggerFactory.getLogger(CubeDimEncMap.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
index fbb93db..65911a0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
@@ -29,7 +29,7 @@ import org.apache.kylin.metadata.model.TblColRef;
*
* @author yangli9
*/
-public class RowKeyColumnIO {
+public class RowKeyColumnIO implements java.io.Serializable {
//private static final Logger logger = LoggerFactory.getLogger(RowKeyColumnIO.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index bf20de1..a669fb1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -18,11 +18,7 @@
package org.apache.kylin.cube.kv;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.base.Preconditions;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -32,9 +28,12 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.base.Preconditions;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
-public class RowKeyEncoder extends AbstractRowKeyEncoder {
+public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Serializable {
private int bodyLength = 0;
private RowKeyColumnIO colIO;
@@ -42,11 +41,13 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
protected boolean enableSharding;
private int uhcOffset = -1;//it's a offset to the beginning of body
private int uhcLength = -1;
+ private int headerLength;
public RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) {
super(cubeSeg, cuboid);
enableSharding = cubeSeg.isEnableSharding();
- Set<TblColRef> shardByColumns = cubeSeg.getShardByColumns();
+ headerLength = cubeSeg.getRowKeyPreambleSize();
+ Set<TblColRef> shardByColumns = cubeSeg.getCubeDesc().getShardByColumns();
if (shardByColumns.size() > 1) {
throw new IllegalStateException("Does not support multiple UHC now");
}
@@ -61,7 +62,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
}
public int getHeaderLength() {
- return cubeSeg.getRowKeyPreambleSize();
+ return headerLength;
}
public int getBytesLength() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java
index 2b1dea7..791b564 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java
@@ -25,7 +25,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import com.google.common.collect.Maps;
-public class RowKeyEncoderProvider {
+public class RowKeyEncoderProvider implements java.io.Serializable {
private CubeSegment cubeSegment;
private HashMap<Long, RowKeyEncoder> rowKeyEncoders;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 9bd082f..5f3c92c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -35,8 +35,8 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class AggregationGroup {
- public static class HierarchyMask {
+public class AggregationGroup implements java.io.Serializable {
+ public static class HierarchyMask implements java.io.Serializable {
public long fullMask; // 00000111
public long[] allMasks; // 00000100,00000110,00000111
public long[] dims; // 00000100,00000010,00000001
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4686a15..0e54b98 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -100,7 +100,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
LOOKUP, PK_FK, EXTENDED_COLUMN
}
- public static class DeriveInfo {
+ public static class DeriveInfo implements java.io.Serializable {
public DeriveType type;
public JoinDesc join;
public TblColRef[] columns;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index 94e1a7c..2a68116 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -36,7 +36,7 @@ import com.google.common.collect.Maps;
/**
*/
-public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
+public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, java.io.Serializable {
protected final String tableName;
protected final CubeDesc cubeDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
index a1312b5..e829aeb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
@@ -18,8 +18,6 @@
package org.apache.kylin.cube.model;
-import java.util.List;
-
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -28,10 +26,12 @@ import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import java.util.List;
+
/**
* An enrich of IJoinedFlatTableDesc for cubes
*/
-public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc {
+public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc, java.io.Serializable {
private CubeDesc cubeDesc;
private IJoinedFlatTableDesc flatDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
index f471f9f..ca2183a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
@@ -27,7 +27,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class DictionaryDesc {
+public class DictionaryDesc implements java.io.Serializable {
@JsonProperty("column")
private String column;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
index cd75228..3488c36 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
@@ -20,7 +20,6 @@ package org.apache.kylin.cube.model;
import java.util.Arrays;
-import org.apache.commons.lang.NotImplementedException;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
@@ -35,7 +34,7 @@ import com.google.common.base.Objects;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class DimensionDesc {
+public class DimensionDesc implements java.io.Serializable {
@JsonProperty("name")
private String name;
@@ -139,16 +138,6 @@ public class DimensionDesc {
}
@Override
- public boolean equals(Object o) {
- throw new NotImplementedException();
- }
-
- @Override
- public int hashCode() {
- throw new NotImplementedException();
- }
-
- @Override
public String toString() {
return Objects.toStringHelper(this).add("name", name).add("table", table).add("column", column).add("derived", Arrays.toString(derived)).add("join", join).toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
index fb491f8..7007342 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
@@ -18,19 +18,18 @@
package org.apache.kylin.cube.model;
-import java.util.Arrays;
-
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+import java.util.Arrays;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseColumnDesc {
+public class HBaseColumnDesc implements java.io.Serializable {
@JsonProperty("qualifier")
private String qualifier;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
index c5b2e19..85c2c17 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
@@ -18,18 +18,17 @@
package org.apache.kylin.cube.model;
-import java.util.Arrays;
-
-import org.apache.kylin.metadata.model.MeasureDesc;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+import java.util.Arrays;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseColumnFamilyDesc {
+public class HBaseColumnFamilyDesc implements java.io.Serializable {
@JsonProperty("name")
private String name;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index 2ef1e17..9ad8407 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -18,22 +18,21 @@
package org.apache.kylin.cube.model;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.LinkedList;
-
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedList;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseMappingDesc {
+public class HBaseMappingDesc implements java.io.Serializable {
@JsonProperty("column_family")
private HBaseColumnFamilyDesc[] columnFamily;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
index f88d4d2..b72f220 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
@@ -27,7 +27,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HierarchyDesc {
+public class HierarchyDesc implements java.io.Serializable {
@JsonProperty("level")
private String level;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index ef34a9b..71fd4b9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -40,7 +40,7 @@ import com.google.common.base.Preconditions;
*
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class RowKeyColDesc {
+public class RowKeyColDesc implements java.io.Serializable {
@JsonProperty("column")
private String column;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
index f1a403d..00557c5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
@@ -18,24 +18,23 @@
package org.apache.kylin.cube.model;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.kylin.metadata.model.TblColRef;
+
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class RowKeyDesc {
+public class RowKeyDesc implements java.io.Serializable {
@JsonProperty("rowkey_columns")
private RowKeyColDesc[] rowkeyColumns;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
index 63b0fc4..4a6c510 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
@@ -20,7 +20,7 @@ package org.apache.kylin.cube.model;
import com.fasterxml.jackson.annotation.JsonProperty;
-public class SelectRule {
+public class SelectRule implements java.io.Serializable {
@JsonProperty("hierarchy_dims")
public String[][] hierarchy_dims;
@JsonProperty("mandatory_dims")
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
index b8d116c..6190f29 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
@@ -28,7 +28,6 @@ import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo.Builder;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
public class UnitTestSupport {
@@ -84,16 +83,16 @@ public class UnitTestSupport {
String d_01_15 = datePlus("2015-01-15", i * 4);
String d_01_16 = datePlus("2015-01-16", i * 4);
String d_01_17 = datePlus("2015-01-17", i * 4);
- result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_14, "Yang", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5")));
}
return result;
}
@@ -106,16 +105,16 @@ public class UnitTestSupport {
String d_01_15 = datePlus("2015-01-15", i * 4);
String d_01_16 = datePlus("2015-01-16", i * 4);
String d_01_17 = datePlus("2015-01-17", i * 4);
- result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_14, "Yang", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 3807b70..b164e54 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -36,8 +36,6 @@ import org.apache.kylin.measure.bitmap.BitmapAggregator;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.hllc.HLLCAggregator;
import org.apache.kylin.measure.hllc.HLLCounter;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.github.jamm.MemoryMeter;
import org.junit.Test;
@@ -100,10 +98,10 @@ public class AggregationCacheMemSizeTest {
private MeasureAggregator<?>[] createNoMemHungryAggrs() {
LongSumAggregator longSum = new LongSumAggregator();
- longSum.aggregate(new LongMutable(10));
+ longSum.aggregate(new Long(10));
DoubleSumAggregator doubleSum = new DoubleSumAggregator();
- doubleSum.aggregate(new DoubleMutable(10));
+ doubleSum.aggregate(new Double(10));
BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator();
decimalSum.aggregate(new BigDecimal("12345678901234567890.123456789"));
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
index d7787e0..74c2659 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
@@ -28,7 +28,6 @@ import java.util.List;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -91,7 +90,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
for (GTRecord record : scanner) {
assertNotNull(record);
Object[] returnRecord = record.getValues();
- assertEquals(20, ((LongMutable) returnRecord[3]).get());
+ assertEquals(20, ((Long) returnRecord[3]).longValue());
assertEquals(21, ((BigDecimal) returnRecord[4]).longValue());
count++;
@@ -133,7 +132,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
for (GTRecord record : scanner) {
assertNotNull(record);
Object[] returnRecord = record.getValues();
- assertEquals(80000, ((LongMutable) returnRecord[3]).get());
+ assertEquals(80000, ((Long) returnRecord[3]).longValue());
assertEquals(84000, ((BigDecimal) returnRecord[4]).longValue());
count++;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
index f315b1e..4ac6644 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
@@ -29,7 +29,6 @@ import java.util.List;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -97,7 +96,7 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
Object[] v = r.getValues();
assertTrue(((String) v[0]).startsWith("2015-"));
assertTrue(((String) v[2]).equals("Food"));
- assertTrue(((LongMutable) v[3]).get() == 10);
+ assertTrue(((Long) v[3]).longValue() == 10);
assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
System.out.println(r);
}
@@ -114,19 +113,19 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
Object[] v = r.getValues();
switch (i) {
case 0:
- assertTrue(((LongMutable) v[3]).get() == 20);
+ assertTrue(((Long) v[3]).longValue() == 20);
assertTrue(((BigDecimal) v[4]).doubleValue() == 21.0);
break;
case 1:
- assertTrue(((LongMutable) v[3]).get() == 30);
+ assertTrue(((Long) v[3]).longValue() == 30);
assertTrue(((BigDecimal) v[4]).doubleValue() == 31.5);
break;
case 2:
- assertTrue(((LongMutable) v[3]).get() == 40);
+ assertTrue(((Long) v[3]).longValue() == 40);
assertTrue(((BigDecimal) v[4]).doubleValue() == 42.0);
break;
case 3:
- assertTrue(((LongMutable) v[3]).get() == 10);
+ assertTrue(((Long) v[3]).longValue() == 10);
assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
break;
default:
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
index 2cf58ac..5e3e771 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
@@ -27,7 +27,6 @@ import java.util.ArrayList;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.datatype.StringSerializer;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -72,7 +71,7 @@ public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase {
GTRowBlock.Writer writer = mockBlock.getWriter();
GTRecord record = new GTRecord(info);
for (int i = 0; i < 10; i++) {
- record.setValues(i < 9 ? "" + i : null, "", "", new LongMutable(0), new BigDecimal(0));
+ record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0));
for (int j = 0; j < info.getRowBlockSize(); j++) {
writer.append(record);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index fc20418..ad4d90b 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -27,8 +27,6 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.hllc.HLLCounter;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.AfterClass;
@@ -54,8 +52,8 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase {
MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") };
BufferedMeasureCodec codec = new BufferedMeasureCodec(descs);
- DoubleMutable d = new DoubleMutable(1.0);
- LongMutable l = new LongMutable(2);
+ Double d = new Double(1.0);
+ Long l = new Long(2);
BigDecimal b = new BigDecimal("333.1234");
HLLCounter hllc = new HLLCounter(16);
hllc.add("1234567");
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 32bfde6..a029f9f 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -141,7 +141,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
}
}
- public static class DictSliceKey implements WritableComparable {
+ public static class DictSliceKey implements WritableComparable, java.io.Serializable {
byte[] key;
public static DictSliceKey wrap(byte[] key) {
@@ -182,7 +182,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
}
}
- public static class DictSlice<T> implements Writable {
+ public static class DictSlice<T> implements Writable, java.io.Serializable {
public DictSlice() {
}
@@ -446,7 +446,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
}
}
- public static class DictNode implements Writable {
+ public static class DictNode implements Writable, java.io.Serializable {
public byte[] part;
public int id = -1;
public boolean isEndOfValue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index 12efbd3..5da93ef 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -153,7 +153,7 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
}
}
- static ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>();
+ static transient ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>();
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
index b5ce1ff..80e9940 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
@@ -24,7 +24,7 @@ package org.apache.kylin.dict;
@SuppressWarnings("serial")
public class NumberDictionary2<T> extends NumberDictionary<T> {
- static ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>();
+ static transient ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>();
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
index 5502a74..7c750f4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
@@ -26,10 +26,10 @@ import org.apache.kylin.dict.NumberDictionary.NumberBytesCodec;
*/
public class NumberDictionaryForestBuilder extends TrieDictionaryForestBuilder<String> {
- public static class Number2BytesConverter implements BytesConverter<String> {
+ public static class Number2BytesConverter implements BytesConverter<String>, java.io.Serializable {
static final int MAX_DIGITS_BEFORE_DECIMAL_POINT = NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT;
- static final ThreadLocal<NumberBytesCodec> LOCAL = new ThreadLocal<NumberBytesCodec>();
+ static final transient ThreadLocal<NumberBytesCodec> LOCAL = new ThreadLocal<NumberBytesCodec>();
static NumberBytesCodec getCodec() {
NumberBytesCodec codec = LOCAL.get();
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java b/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
index 0bec6a1..9107a4c 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
@@ -20,7 +20,7 @@ package org.apache.kylin.dict;
import org.apache.kylin.common.util.Bytes;
-public class StringBytesConverter implements BytesConverter<String> {
+public class StringBytesConverter implements BytesConverter<String>, java.io.Serializable {
@Override
public byte[] convertToBytes(String v) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0273fd8..466cdad 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -18,12 +18,9 @@
package org.apache.kylin.job.execution;
-import java.lang.reflect.Constructor;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.job.dao.ExecutableDao;
@@ -34,9 +31,11 @@ import org.apache.kylin.job.exception.PersistentException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.lang.reflect.Constructor;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
/**
*/
@@ -352,6 +351,10 @@ public class ExecutableManager {
}
}
}
+
+ if (job.getStatus() == ExecutableState.SUCCEED) {
+ updateJobOutput(job.getId(), ExecutableState.READY, null, null);
+ }
}
public void pauseJob(String jobId) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
index ec6347f..a54bcda 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
@@ -18,6 +18,10 @@
package org.apache.kylin.dimension;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
@@ -25,10 +29,6 @@ import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-
public class AbstractDateDimEnc extends DimensionEncoding {
private static final long serialVersionUID = 1L;
@@ -81,11 +81,9 @@ public class AbstractDateDimEnc extends DimensionEncoding {
@Override
public DataTypeSerializer<Object> asDataTypeSerializer() {
return new DataTypeSerializer<Object>() {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[fixedLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
index c3f4c11..75e50a1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
@@ -109,11 +109,9 @@ public class BooleanDimEnc extends DimensionEncoding {
}
private class BooleanSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[fixedLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
index 500b410..48238dc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
@@ -39,12 +39,15 @@ public class DictionaryDimEnc extends DimensionEncoding {
// ============================================================================
// could use a lazy loading trick here, to prevent loading all dictionaries of a segment at once
- private final Dictionary<String> dict;
- private final int fixedLen;
+ private Dictionary<String> dict;
+ private int fixedLen;
// used in encode(), when a value does not exist in dictionary
- private final int roundingFlag;
- private final byte defaultByte;
+ private int roundingFlag;
+ private byte defaultByte;
+
+ public DictionaryDimEnc() {
+ }
public DictionaryDimEnc(Dictionary<String> dict) {
this(dict, 0, NULL);
@@ -145,12 +148,18 @@ public class DictionaryDimEnc extends DimensionEncoding {
@Override
public void writeExternal(ObjectOutput out) throws IOException {
- throw new UnsupportedOperationException();
+ out.writeInt(fixedLen);
+ out.writeInt(roundingFlag);
+ out.write(defaultByte);
+ out.writeObject(dict);
}
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- throw new UnsupportedOperationException();
+ this.fixedLen = in.readInt();
+ this.roundingFlag = in.readInt();
+ this.defaultByte = in.readByte();
+ this.dict = (Dictionary<String>) in.readObject();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
index b219766..f7f02a0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
@@ -129,11 +129,9 @@ public class FixedLenDimEnc extends DimensionEncoding {
}
public class FixedLenSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[fixedLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
index 83118fc..f90a40e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
@@ -224,11 +224,9 @@ public class FixedLenHexDimEnc extends DimensionEncoding {
}
public class FixedLenSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[bytelen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
index f25f2a6..aa954da 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
@@ -109,11 +109,9 @@ public class IntDimEnc extends DimensionEncoding {
}
public class IntegerSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[fixedLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 090dc83..0875a7f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -135,11 +135,9 @@ public class IntegerDimEnc extends DimensionEncoding {
}
public class IntegerSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[fixedLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
index 993aac3..c0c52d1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
@@ -18,17 +18,17 @@
package org.apache.kylin.dimension;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* not being used yet, prepared for future
*/
@@ -118,11 +118,9 @@ public class OneMoreByteVLongDimEnc extends DimensionEncoding {
}
public class VLongSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[byteLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java
index 8a5481c..44e5708 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java
@@ -18,26 +18,26 @@
package org.apache.kylin.measure;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.Collection;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
/**
* This class embeds a reusable byte buffer for measure encoding, and is not thread-safe.
* The buffer will grow to accommodate BufferOverflowException until a limit.
* The problem here to solve is some measure type cannot provide accurate DataTypeSerializer.maxLength()
*/
@SuppressWarnings({ "unchecked" })
-public class BufferedMeasureCodec {
+public class BufferedMeasureCodec implements java.io.Serializable {
public static final int DEFAULT_BUFFER_SIZE = 1024 * 1024; // 1 MB
public static final int MAX_BUFFER_SIZE = 1 * 1024 * DEFAULT_BUFFER_SIZE; // 1 GB
final private MeasureCodec codec;
- private ByteBuffer buf;
+ private transient ByteBuffer buf;
final private int[] measureSizes;
public BufferedMeasureCodec(Collection<MeasureDesc> measureDescs) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf6bba8b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
index 7b74225..929eff0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
@@ -71,6 +71,8 @@ abstract public class MeasureAggregator<V> implements Serializable {
abstract public void aggregate(V value);
+ abstract public V aggregate(V value1, V value2);
+
abstract public V getState();
// get an estimate of memory consumption UPPER BOUND
[07/42] kylin git commit: minor, drop scan threshold in KylinTestBase
Posted by sh...@apache.org.
minor, drop scan threshold in KylinTestBase
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/00de4411
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/00de4411
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/00de4411
Branch: refs/heads/sparkcubing-rebase
Commit: 00de44116126bd585f906c25606579a1ce2961f3
Parents: 74c2d6b
Author: Li Yang <li...@apache.org>
Authored: Tue Dec 27 18:38:16 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Dec 27 18:38:16 2016 +0800
----------------------------------------------------------------------
kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/00de4411/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 0a795ef..3ae7521 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -631,7 +631,6 @@ public class KylinTestBase {
//setup cube conn
File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME, config);
Properties props = new Properties();
- props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "20001");
cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath(), props);
//setup h2
[03/42] kylin git commit: minor code reformat
Posted by sh...@apache.org.
minor code reformat
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f466a07
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f466a07
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f466a07
Branch: refs/heads/sparkcubing-rebase
Commit: 4f466a07bfb5438a1c573699f0b01cd206b0423a
Parents: 0453d3d
Author: Li Yang <li...@apache.org>
Authored: Mon Dec 26 15:59:03 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Dec 26 15:59:03 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/provision/ProvisionWithoutBuild.java | 2 +-
.../main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f466a07/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java b/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
index 71bb0ca..039d583 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
@@ -24,6 +24,6 @@ public class ProvisionWithoutBuild extends BuildCubeWithEngine {
public static void main(String[] args) throws Exception {
beforeClass();
DeployUtil.deployMetadata();
- DeployUtil.prepareTestDataForNormalCubes("test_kylin_cube_with_slr_empty");
+ DeployUtil.prepareTestDataForNormalCubes("ci_left_join_model");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f466a07/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
index 93f06dd..132e65a 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
@@ -163,6 +163,6 @@ public class OLAPSchemaFactory implements SchemaFactory {
}
index++;
}
- out.write(" ]\n");
+ out.write(" ]\n");
}
}
[34/42] kylin git commit: KYLIN-2328 Reduce the size of metadata
uploaded to distributed cache
Posted by sh...@apache.org.
KYLIN-2328 Reduce the size of metadata uploaded to distributed cache
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ea03b86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ea03b86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ea03b86
Branch: refs/heads/sparkcubing-rebase
Commit: 6ea03b8638d72853959aba0666aea18f1ba97391
Parents: 64c3c61
Author: gaodayue <ga...@meituan.com>
Authored: Wed Dec 28 15:27:49 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Thu Dec 29 18:39:13 2016 +0800
----------------------------------------------------------------------
.../engine/mr/common/AbstractHadoopJob.java | 43 ++++++++++++++------
.../apache/kylin/engine/mr/steps/CuboidJob.java | 2 +-
.../engine/mr/steps/FactDistinctColumnsJob.java | 7 ++--
.../kylin/engine/mr/steps/InMemCuboidJob.java | 10 ++---
.../kylin/engine/mr/steps/MergeCuboidJob.java | 3 +-
.../cardinality/HiveColumnCardinalityJob.java | 2 +-
.../source/kafka/hadoop/KafkaFlatTableJob.java | 3 --
.../kylin/storage/hbase/steps/CubeHFileJob.java | 11 +++--
8 files changed, 48 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ea03b86/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 80636d3..4693ac3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -29,7 +29,6 @@ import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashSet;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -449,33 +448,49 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
}
- protected void attachKylinPropsAndMetadata(TableDesc table, Configuration conf) throws IOException {
+ protected void attachTableMetadata(TableDesc table, Configuration conf) throws IOException {
Set<String> dumpList = new LinkedHashSet<>();
dumpList.add(table.getResourcePath());
attachKylinPropsAndMetadata(dumpList, KylinConfig.getInstanceFromEnv(), conf);
}
- protected void attachKylinPropsAndMetadata(CubeInstance cube, Configuration conf) throws IOException {
- // write cube / model_desc / cube_desc / dict / table
+ protected void attachCubeMetadata(CubeInstance cube, Configuration conf) throws IOException {
+ attachKylinPropsAndMetadata(collectCubeMetadata(cube), cube.getConfig(), conf);
+ }
+
+ protected void attachCubeMetadataWithDict(CubeInstance cube, Configuration conf) throws IOException {
+ Set<String> dumpList = new LinkedHashSet<>();
+ dumpList.addAll(collectCubeMetadata(cube));
+ for (CubeSegment segment : cube.getSegments()) {
+ dumpList.addAll(segment.getDictionaryPaths());
+ }
+ attachKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
+ }
+
+ protected void attachSegmentMetadataWithDict(CubeSegment segment, Configuration conf) throws IOException {
+ Set<String> dumpList = new LinkedHashSet<>();
+ dumpList.addAll(collectCubeMetadata(segment.getCubeInstance()));
+ dumpList.addAll(segment.getDictionaryPaths());
+ attachKylinPropsAndMetadata(dumpList, segment.getConfig(), conf);
+ }
+
+ private Set<String> collectCubeMetadata(CubeInstance cube) {
+ // cube, model_desc, cube_desc, table
Set<String> dumpList = new LinkedHashSet<>();
dumpList.add(cube.getResourcePath());
dumpList.add(cube.getDescriptor().getModel().getResourcePath());
dumpList.add(cube.getDescriptor().getResourcePath());
- for (TableRef tableRef: cube.getDescriptor().getModel().getAllTables()) {
+ for (TableRef tableRef : cube.getDescriptor().getModel().getAllTables()) {
TableDesc table = tableRef.getTableDesc();
dumpList.add(table.getResourcePath());
- List<String> dependentResources = SourceFactory.getMRDependentResources(table);
- dumpList.addAll(dependentResources);
- }
- for (CubeSegment segment : cube.getSegments()) {
- dumpList.addAll(segment.getDictionaryPaths());
+ dumpList.addAll(SourceFactory.getMRDependentResources(table));
}
- attachKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
+ return dumpList;
}
- protected void attachKylinPropsAndMetadata(Set<String> dumpList, KylinConfig kylinConfig, Configuration conf) throws IOException {
+ private void attachKylinPropsAndMetadata(Set<String> dumpList, KylinConfig kylinConfig, Configuration conf) throws IOException {
File tmp = File.createTempFile("kylin_job_meta", "");
FileUtils.forceDelete(tmp); // we need a directory, so delete the file first
@@ -524,6 +539,8 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
private void dumpResources(KylinConfig kylinConfig, File metaDir, Set<String> dumpList) throws IOException {
+ long startTime = System.currentTimeMillis();
+
ResourceStore from = ResourceStore.getStore(kylinConfig);
KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath());
ResourceStore to = ResourceStore.getStore(localConfig);
@@ -534,6 +551,8 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
to.putResource(path, res.inputStream, res.timestamp);
res.inputStream.close();
}
+
+ logger.debug("Dump resources to {} took {} ms", metaDir, System.currentTimeMillis() - startTime);
}
protected void deletePath(Configuration conf, Path path) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ea03b86/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index d3cb494..bd305c1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -129,7 +129,7 @@ public class CuboidJob extends AbstractHadoopJob {
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
job.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, nCuboidLevel);
// add metadata to distributed cache
- attachKylinPropsAndMetadata(cube, job.getConfiguration());
+ attachSegmentMetadataWithDict(segment, job.getConfiguration());
LayerReduerNumSizing.setReduceTaskNum(job, segment, getTotalMapInputMB(), nCuboidLevel);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ea03b86/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
index 9fc8922..ce01eb6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
@@ -110,13 +110,12 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
logger.error(s.getName() + " with status " + s.getStatus());
}
throw new IllegalStateException();
- } else {
- logger.info("Found segment: " + segment);
}
- setupMapper(cube.getSegmentById(segmentID));
+
+ setupMapper(segment);
setupReducer(output, "true".equalsIgnoreCase(statistics_enabled) ? reducerCount + 2 : reducerCount);
- attachKylinPropsAndMetadata(cube, job.getConfiguration());
+ attachCubeMetadata(cube, job.getConfiguration());
return waitForCompletion(job);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ea03b86/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
index 576ace9..1612866 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
@@ -86,11 +86,11 @@ public class InMemCuboidJob extends AbstractHadoopJob {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cube = cubeMgr.getCube(cubeName);
- CubeSegment cubeSeg = cube.getSegmentById(segmentID);
+ CubeSegment segment = cube.getSegmentById(segmentID);
String cubingJobId = getOptionValue(OPTION_CUBING_JOB_ID);
if (checkSkip(cubingJobId)) {
- logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + cubeSeg);
+ logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + segment);
return 0;
}
@@ -101,14 +101,14 @@ public class InMemCuboidJob extends AbstractHadoopJob {
setJobClasspath(job, cube.getConfig());
// add metadata to distributed cache
- attachKylinPropsAndMetadata(cube, job.getConfiguration());
+ attachSegmentMetadataWithDict(segment, job.getConfiguration());
// set job configuration
job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
// set input
- IMRTableInputFormat flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSeg).getFlatTableInputFormat();
+ IMRTableInputFormat flatTableInputFormat = MRUtil.getBatchCubingInputSide(segment).getFlatTableInputFormat();
flatTableInputFormat.configureJob(job);
// set mapper
@@ -118,7 +118,7 @@ public class InMemCuboidJob extends AbstractHadoopJob {
// set output
job.setReducerClass(InMemCuboidReducer.class);
- job.setNumReduceTasks(calculateReducerNum(cubeSeg));
+ job.setNumReduceTasks(calculateReducerNum(segment));
// the cuboid file and KV class must be compatible with 0.7 version for smooth upgrade
job.setOutputFormatClass(SequenceFileOutputFormat.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ea03b86/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
index e805d25..012e19f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
@@ -79,7 +79,8 @@ public class MergeCuboidJob extends CuboidJob {
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
// add metadata to distributed cache
- attachKylinPropsAndMetadata(cube, job.getConfiguration());
+ // TODO actually only dictionaries from merging segments are needed
+ attachCubeMetadataWithDict(cube, job.getConfiguration());
LayerReduerNumSizing.setReduceTaskNum(job, cube.getSegmentById(segmentID), getTotalMapInputMB(), -1);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ea03b86/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
index 3c88024..ea72b54 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
@@ -103,7 +103,7 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob {
logger.info("Going to submit HiveColumnCardinalityJob for table '" + table + "'");
TableDesc tableDesc = MetadataManager.getInstance(kylinConfig).getTableDesc(table);
- attachKylinPropsAndMetadata(tableDesc, job.getConfiguration());
+ attachTableMetadata(tableDesc, job.getConfiguration());
int result = waitForCompletion(job);
return result;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ea03b86/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
index f0f48c0..11466e5 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
@@ -120,9 +120,6 @@ public class KafkaFlatTableJob extends AbstractHadoopJob {
job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
deletePath(job.getConfiguration(), output);
-
- attachKylinPropsAndMetadata(cube, job.getConfiguration());
-
return waitForCompletion(job);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ea03b86/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 9593372..1a624c4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -88,19 +88,18 @@ public class CubeHFileJob extends AbstractHadoopJob {
// set job configuration
job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- Configuration conf = HBaseConfiguration.create(getConf());
// add metadata to distributed cache
- attachKylinPropsAndMetadata(cube, job.getConfiguration());
+ attachCubeMetadata(cube, job.getConfiguration());
- String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
- HTable htable = new HTable(conf, tableName);
+ Configuration hbaseConf = HBaseConfiguration.create(getConf());
+ HTable htable = new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME).toUpperCase());
// Automatic config !
HFileOutputFormat.configureIncrementalLoad(job, htable);
- reconfigurePartitions(conf, partitionFilePath);
+ reconfigurePartitions(hbaseConf, partitionFilePath);
// set block replication to 3 for hfiles
- conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "3");
+ hbaseConf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "3");
this.deletePath(job.getConfiguration(), output);
[12/42] kylin git commit: close OLAPEnumerator at exception
Posted by sh...@apache.org.
close OLAPEnumerator at exception
bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/596ab51b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/596ab51b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/596ab51b
Branch: refs/heads/sparkcubing-rebase
Commit: 596ab51b4b98ae0923a134ede56c0ceb2f23bb66
Parents: 53b6c8c
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:17:30 2016 +0800
----------------------------------------------------------------------
.../kylin/query/enumerator/OLAPEnumerator.java | 33 +++++++++++++-------
1 file changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/596ab51b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,31 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
@Override
public boolean moveNext() {
- if (cursor == null) {
- cursor = queryStorage();
- }
+ try {
+ if (cursor == null) {
+ cursor = queryStorage();
+ }
- if (!cursor.hasNext()) {
- return false;
- }
+ if (!cursor.hasNext()) {
+ return false;
+ }
- ITuple tuple = cursor.next();
- if (tuple == null) {
- return false;
+ ITuple tuple = cursor.next();
+ if (tuple == null) {
+ return false;
+ }
+ convertCurrentRow(tuple);
+ return true;
+ } catch (Exception e) {
+ try {
+ if (cursor != null) {
+ cursor.close();
+ }
+ } catch (Exception ee) {
+ logger.info("Error when closing cursor, ignore it", ee);
+ }
+ throw e;
}
- convertCurrentRow(tuple);
- return true;
}
private Object[] convertCurrentRow(ITuple tuple) {
[02/42] kylin git commit: KYLIN-2301 various fixes to support NOT IN
subquery
Posted by sh...@apache.org.
KYLIN-2301 various fixes to support NOT IN subquery
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0453d3da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0453d3da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0453d3da
Branch: refs/heads/sparkcubing-rebase
Commit: 0453d3daada2e32f79954dfd12ac9f19062e825c
Parents: ed8b168
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 23 08:17:28 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Dec 26 15:26:25 2016 +0800
----------------------------------------------------------------------
.../kylin/metadata/filter/LogicalTupleFilter.java | 13 -------------
.../org/apache/kylin/metadata/filter/TupleFilter.java | 7 ++++++-
.../org/apache/kylin/metadata/model/TblColRef.java | 7 ++-----
.../java/org/apache/kylin/query/ITKylinQueryTest.java | 2 +-
.../src/test/resources/query/sql_subquery/query13.sql | 7 +++++++
.../apache/kylin/query/relnode/OLAPAggregateRel.java | 11 +++++++++--
.../org/apache/kylin/query/relnode/OLAPJoinRel.java | 8 +++++++-
7 files changed, 32 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 1744309..373acdb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -47,19 +47,6 @@ public class LogicalTupleFilter extends TupleFilter {
return cloneTuple;
}
- // private TupleFilter reverseNestedNots(TupleFilter filter, int depth) {
- // if ((filter instanceof LogicalTupleFilter) && (filter.operator == FilterOperatorEnum.NOT)) {
- // assert (filter.children.size() == 1);
- // return reverseNestedNots(filter.children.get(0), depth + 1);
- // }
- //
- // if (depth % 2 == 1) {
- // return filter;
- // } else {
- // return filter.reverse();
- // }
- // }
-
@Override
public TupleFilter reverse() {
switch (operator) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 285172c..31de50b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -27,6 +27,8 @@ import java.util.Set;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Maps;
@@ -37,6 +39,8 @@ import com.google.common.collect.Maps;
*/
public abstract class TupleFilter {
+ static final Logger logger = LoggerFactory.getLogger(TupleFilter.class);
+
public enum FilterOperatorEnum {
EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), EVAL_FUNC(37), UNSUPPORTED(38);
@@ -112,7 +116,8 @@ public abstract class TupleFilter {
}
public TupleFilter reverse() {
- throw new UnsupportedOperationException();
+ logger.warn("Cannot reverse " + this + ", loosen the filter to true");
+ return ConstantTupleFilter.TRUE;
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index fd0224f..88abb9d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -27,7 +27,7 @@ import org.apache.kylin.metadata.datatype.DataType;
/**
*/
-@SuppressWarnings("serial")
+@SuppressWarnings({ "serial", "deprecation" })
public class TblColRef implements Serializable {
private static final String INNER_TABLE_NAME = "_kylin_table";
@@ -82,6 +82,7 @@ public class TblColRef implements Serializable {
TableRef tableRef = model.findTable(alias);
checkArgument(tableRef.getTableDesc() == col.column.getTable());
col.table = tableRef;
+ col.identity = null;
}
// for test mainly
@@ -160,10 +161,6 @@ public class TblColRef implements Serializable {
return InnerDataTypeEnum.contains(getDatatype());
}
- public boolean isDerivedDataType() {
- return InnerDataTypeEnum.DERIVED.getDataType().equals(getDatatype());
- }
-
public int hashCode() {
// NOTE: tableRef MUST NOT participate in hashCode().
// Because fixUnknownModel() can change tableRef while TblColRef is held as set/map keys.
http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 4329141..840a03f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -149,7 +149,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_limit/query01.sql";
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/temp/sample.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/kylin-it/src/test/resources/query/sql_subquery/query13.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query13.sql b/kylin-it/src/test/resources/query/sql_subquery/query13.sql
new file mode 100644
index 0000000..d7f6109
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query13.sql
@@ -0,0 +1,7 @@
+select
+ f.cal_dt
+from test_kylin_fact f
+where
+ f.cal_dt not in (
+ select cal_dt from EDW.TEST_CAL_DT where week_beg_dt = date'2012-01-01'
+ )
http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index c457545..8d7c597 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -153,7 +153,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
// only translate the innermost aggregation
if (!this.afterAggregate) {
- this.context.groupByColumns.addAll(this.groups);
+ addToContextGroupBy(this.groups);
this.context.aggregations.addAll(this.aggregations);
this.context.afterAggregate = true;
@@ -326,7 +326,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
FunctionDesc aggFunc = this.aggregations.get(i);
if (aggFunc.isDimensionAsMetric()) {
- this.context.groupByColumns.addAll(aggFunc.getParameter().getColRefs());
+ addToContextGroupBy(aggFunc.getParameter().getColRefs());
continue; // skip rewrite, let calcite handle
}
@@ -350,6 +350,13 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
}
+ private void addToContextGroupBy(List<TblColRef> colRefs) {
+ for (TblColRef col : colRefs) {
+ if (col.isInnerColumn() == false)
+ this.context.groupByColumns.add(col);
+ }
+ }
+
private void fillbackOptimizedColumn() {
// some aggcall will be optimized out in sub-query (e.g. tableau generated sql), we need to fill them back
RelDataType inputAggRow = getInput().getRowType();
http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 6869858..54b851f 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -164,7 +164,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
} else if (leftHasSubquery != rightHasSubquery) {
//When join contains subquery, the join-condition fields of fact_table will add into context.
Map<TblColRef, TblColRef> joinCol = new HashMap<TblColRef, TblColRef>();
- translateJoinColumn((RexCall) this.getCondition(), joinCol);
+ translateJoinColumn(this.getCondition(), joinCol);
for (Map.Entry<TblColRef, TblColRef> columnPair : joinCol.entrySet()) {
TblColRef fromCol = (rightHasSubquery ? columnPair.getKey() : columnPair.getValue());
@@ -219,6 +219,12 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
return join;
}
+ private void translateJoinColumn(RexNode condition, Map<TblColRef, TblColRef> joinCol) {
+ if (condition instanceof RexCall) {
+ translateJoinColumn((RexCall) condition, joinCol);
+ }
+ }
+
private void translateJoinColumn(RexCall condition, Map<TblColRef, TblColRef> joinColumns) {
SqlKind kind = condition.getOperator().getKind();
if (kind == SqlKind.AND) {
[31/42] kylin git commit: KYLIN-2332 Refactor TupleConverter a bit
Posted by sh...@apache.org.
KYLIN-2332 Refactor TupleConverter a bit
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/13c34079
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/13c34079
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/13c34079
Branch: refs/heads/sparkcubing-rebase
Commit: 13c34079daa0e102b817d15ccfe2d70b8b364c34
Parents: 687d5fd
Author: Li Yang <li...@apache.org>
Authored: Thu Dec 29 12:18:45 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 29 13:38:19 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/storage/IStorageQuery.java | 3 -
.../apache/kylin/storage/StorageContext.java | 10 ++++
.../storage/gtrecord/CubeSegmentScanner.java | 3 +-
.../storage/gtrecord/CubeTupleConverter.java | 60 +-------------------
.../gtrecord/GTCubeStorageQueryBase.java | 7 ++-
.../kylin/storage/gtrecord/ITupleConverter.java | 30 ++++++++++
.../gtrecord/SegmentCubeTupleIterator.java | 8 +--
.../gtrecord/SequentialCubeTupleIterator.java | 55 +++++++++++++++++-
.../storage/hbase/cube/v2/CubeStorageQuery.java | 1 +
9 files changed, 108 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
index 6b53b5b..5455cc3 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
@@ -23,9 +23,6 @@ import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.TupleInfo;
/**
- *
- * @author xjiang
- *
*/
public interface IStorageQuery {
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index 9ef59fd..ec46f83 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -47,6 +47,7 @@ public class StorageContext {
private boolean limitEnabled = false;
private boolean enableCoprocessor = false;
+ private IStorageQuery storageQuery;
private AtomicLong totalScanCount = new AtomicLong();
private Cuboid cuboid;
private boolean partialResultReturned = false;
@@ -193,4 +194,13 @@ public class StorageContext {
public void setReusedPeriod(Range<Long> reusedPeriod) {
this.reusedPeriod = reusedPeriod;
}
+
+ public IStorageQuery getStorageQuery() {
+ return storageQuery;
+ }
+
+ public void setStorageQuery(IStorageQuery storageQuery) {
+ this.storageQuery = storageQuery;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index c6a6daa..9d6f946 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -52,7 +52,7 @@ public class CubeSegmentScanner implements IGTScanner {
final GTScanRequest scanRequest;
public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, //
- Collection<FunctionDesc> metrics, TupleFilter originalfilter, StorageContext context, String gtStorage) {
+ Collection<FunctionDesc> metrics, TupleFilter originalfilter, StorageContext context) {
logger.info("Init CubeSegmentScanner for segment {}", cubeSeg.getName());
@@ -78,6 +78,7 @@ public class CubeSegmentScanner implements IGTScanner {
throw new RuntimeException(e);
}
scanRequest = scanRangePlanner.planScanRequest();
+ String gtStorage = ((GTCubeStorageQueryBase) context.getStorageQuery()).getGTStorage();
scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest, gtStorage);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 3159318..280718f 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@ -18,11 +18,10 @@
package org.apache.kylin.storage.gtrecord;
-import java.util.Comparator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.Map.Entry;
+import java.util.Set;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.Dictionary;
@@ -37,18 +36,16 @@ import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.metadata.tuple.TupleInfo;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
/**
* convert GTRecord to tuple
*/
-public class CubeTupleConverter {
+public class CubeTupleConverter implements ITupleConverter {
final CubeSegment cubeSeg;
final Cuboid cuboid;
@@ -65,8 +62,6 @@ public class CubeTupleConverter {
private final int nSelectedDims;
- private final int[] dimensionIndexOnTuple;
-
public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
@@ -88,18 +83,6 @@ public class CubeTupleConverter {
advMeasureFillers = Lists.newArrayListWithCapacity(1);
advMeasureIndexInGTValues = Lists.newArrayListWithCapacity(1);
- // dimensionIndexOnTuple is for SQL with limit
- List<Integer> temp = Lists.newArrayList();
- for (TblColRef dim : cuboid.getColumns()) {
- if (tupleInfo.hasColumn(dim)) {
- temp.add(tupleInfo.getColumnIndex(dim));
- }
- }
- dimensionIndexOnTuple = new int[temp.size()];
- for (int i = 0; i < temp.size(); i++) {
- dimensionIndexOnTuple[i] = temp.get(i);
- }
-
////////////
int i = 0;
@@ -155,44 +138,6 @@ public class CubeTupleConverter {
}
}
- public Comparator<ITuple> getTupleDimensionComparator() {
- return new Comparator<ITuple>() {
- @Override
- public int compare(ITuple o1, ITuple o2) {
- Preconditions.checkNotNull(o1);
- Preconditions.checkNotNull(o2);
- for (int i = 0; i < dimensionIndexOnTuple.length; i++) {
- int index = dimensionIndexOnTuple[i];
-
- if (index == -1) {
- //TODO:
- continue;
- }
-
- Comparable a = (Comparable) o1.getAllValues()[index];
- Comparable b = (Comparable) o2.getAllValues()[index];
-
- if (a == null && b == null) {
- continue;
- } else if (a == null) {
- return 1;
- } else if (b == null) {
- return -1;
- } else {
- int temp = a.compareTo(b);
- if (temp != 0) {
- return temp;
- } else {
- continue;
- }
- }
- }
-
- return 0;
- }
- };
- }
-
// load only needed dictionaries
private Map<TblColRef, Dictionary<String>> buildDictionaryMap(List<TblColRef> columnsNeedDictionary) {
Map<TblColRef, Dictionary<String>> result = Maps.newHashMap();
@@ -202,6 +147,7 @@ public class CubeTupleConverter {
return result;
}
+ @Override
public List<IAdvMeasureFiller> translateResult(GTRecord record, Tuple tuple) {
record.getValues(gtColIdx, gtValues);
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 7b0cbb9..4fcfad1 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -72,6 +72,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ context.setStorageQuery(this);
//deal with participant columns in subquery join
sqlDigest.includeSubqueryJoinParticipants();
@@ -131,7 +132,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
continue;
}
- scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context, getGTStorage());
+ scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context);
scanners.add(scanner);
}
@@ -142,6 +143,10 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
protected abstract String getGTStorage();
+
+ protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo tupleInfo) {
+ return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
+ }
private void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
for (FunctionDesc func : sqlDigest.aggregations) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ITupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ITupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ITupleConverter.java
new file mode 100644
index 0000000..9c50d0c
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ITupleConverter.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import java.util.List;
+
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
+import org.apache.kylin.metadata.tuple.Tuple;
+
+public interface ITupleConverter {
+
+ public List<IAdvMeasureFiller> translateResult(GTRecord record, Tuple tuple);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 00ba247..37699a3 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -50,7 +50,7 @@ public class SegmentCubeTupleIterator implements ITupleIterator {
protected final StorageContext context;
protected Iterator<GTRecord> gtItr;
- protected CubeTupleConverter cubeTupleConverter;
+ protected ITupleConverter cubeTupleConverter;
protected Tuple next;
private List<IAdvMeasureFiller> advMeasureFillers;
@@ -67,7 +67,7 @@ public class SegmentCubeTupleIterator implements ITupleIterator {
this.tuple = new Tuple(returnTupleInfo);
this.context = context;
this.gtItr = getGTItr(scanner);
- this.cubeTupleConverter = new CubeTupleConverter(scanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
+ this.cubeTupleConverter = ((GTCubeStorageQueryBase) context.getStorageQuery()).newCubeTupleConverter(scanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
}
private Iterator<GTRecord> getGTItr(CubeSegmentScanner scanner) {
@@ -151,8 +151,4 @@ public class SegmentCubeTupleIterator implements ITupleIterator {
logger.error("Exception when close CubeScanner", e);
}
}
-
- public CubeTupleConverter getCubeTupleConverter() {
- return cubeTupleConverter;
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index 3a64de7..ee868c7 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -19,6 +19,7 @@
package org.apache.kylin.storage.gtrecord;
import java.io.IOException;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -36,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
@@ -73,9 +75,60 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
return input;
}
});
- tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator();
+ tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, returnTupleInfo)).getIterator();
}
}
+
+ public Comparator<ITuple> getTupleDimensionComparator(Cuboid cuboid, TupleInfo returnTupleInfo) {
+ // dimensionIndexOnTuple is for SQL with limit
+ List<Integer> temp = Lists.newArrayList();
+ for (TblColRef dim : cuboid.getColumns()) {
+ if (returnTupleInfo.hasColumn(dim)) {
+ temp.add(returnTupleInfo.getColumnIndex(dim));
+ }
+ }
+
+ final int[] dimensionIndexOnTuple = new int[temp.size()];
+ for (int i = 0; i < temp.size(); i++) {
+ dimensionIndexOnTuple[i] = temp.get(i);
+ }
+
+ return new Comparator<ITuple>() {
+ @Override
+ public int compare(ITuple o1, ITuple o2) {
+ Preconditions.checkNotNull(o1);
+ Preconditions.checkNotNull(o2);
+ for (int i = 0; i < dimensionIndexOnTuple.length; i++) {
+ int index = dimensionIndexOnTuple[i];
+
+ if (index == -1) {
+ //TODO:
+ continue;
+ }
+
+ Comparable a = (Comparable) o1.getAllValues()[index];
+ Comparable b = (Comparable) o2.getAllValues()[index];
+
+ if (a == null && b == null) {
+ continue;
+ } else if (a == null) {
+ return 1;
+ } else if (b == null) {
+ return -1;
+ } else {
+ int temp = a.compareTo(b);
+ if (temp != 0) {
+ return temp;
+ } else {
+ continue;
+ }
+ }
+ }
+
+ return 0;
+ }
+ };
+ }
@Override
public boolean hasNext() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index f9c9a2b..fe483ba 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory;
public class CubeStorageQuery extends GTCubeStorageQueryBase {
+ @SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class);
public CubeStorageQuery(CubeInstance cube) {
[15/42] kylin git commit: KYLIN-2327 enable check-style for test code
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
deleted file mode 100644
index 2edf11e..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
+++ /dev/null
@@ -1,131 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-//*/
-//
-//package org.apache.kylin.storage.minicluster;
-//
-//import java.io.File;
-//import java.io.IOException;
-//import java.sql.Connection;
-//import java.sql.DriverManager;
-//import java.sql.SQLException;
-//
-//import org.apache.commons.io.FileUtils;
-//import org.apache.hadoop.hdfs.MiniDFSCluster;
-//import org.apache.hadoop.hive.conf.HiveConf;
-//import org.apache.hadoop.hive.service.HiveInterface;
-//import org.apache.hadoop.mapred.JobConf;
-//import org.apache.hadoop.mapred.MiniMRCluster;
-//
-///**
-// * This is a test case to verify whether the query can be executed on Hive minicluster;
-// * You need set $HADOOP_HOME environment parameter before run it;
-// * @author shaoshi
-// *
-// */
-//public class HiveMiniClusterTest extends HiveJDBCClientTest {
-// public static final File HIVE_BASE_DIR = new File("target/hive");
-// public static final File HIVE_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/scratchdir");
-// public static final File HIVE_LOCAL_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/localscratchdir");
-// public static final File HIVE_METADB_DIR = new File(HIVE_BASE_DIR + "/metastoredb");
-// public static final File HIVE_LOGS_DIR = new File(HIVE_BASE_DIR + "/logs");
-// public static final File HIVE_TMP_DIR = new File(HIVE_BASE_DIR + "/tmp");
-// public static final File HIVE_WAREHOUSE_DIR = new File(HIVE_BASE_DIR + "/warehouse");
-// public static final File HIVE_TESTDATA_DIR = new File(HIVE_BASE_DIR + "/testdata");
-// public static final File HIVE_HADOOP_TMP_DIR = new File(HIVE_BASE_DIR + "/hadooptmp");
-// protected HiveInterface client;
-//
-// protected MiniDFSCluster miniDFS;
-// protected MiniMRCluster miniMR;
-//
-// //@Before
-// public void setup() {
-// super.setup();
-// startHiveMiniCluster();
-// }
-//
-// protected void startHiveMiniCluster() {
-// //Create and configure location for hive to dump junk in target folder
-// try {
-// FileUtils.forceMkdir(HIVE_BASE_DIR);
-// FileUtils.forceMkdir(HIVE_SCRATCH_DIR);
-// FileUtils.forceMkdir(HIVE_LOCAL_SCRATCH_DIR);
-// FileUtils.forceMkdir(HIVE_LOGS_DIR);
-// FileUtils.forceMkdir(HIVE_TMP_DIR);
-// FileUtils.forceMkdir(HIVE_WAREHOUSE_DIR);
-// FileUtils.forceMkdir(HIVE_HADOOP_TMP_DIR);
-// FileUtils.forceMkdir(HIVE_TESTDATA_DIR);
-// } catch (IOException e1) {
-// e1.printStackTrace();
-// System.exit(1);
-// }
-//
-// System.setProperty("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=" + HIVE_METADB_DIR.getAbsolutePath() + ";create=true");
-// System.setProperty("hive.metastore.warehouse.dir", HIVE_WAREHOUSE_DIR.getAbsolutePath());
-// System.setProperty("hive.exec.scratchdir", HIVE_SCRATCH_DIR.getAbsolutePath());
-// System.setProperty("hive.exec.local.scratchdir", HIVE_LOCAL_SCRATCH_DIR.getAbsolutePath());
-// System.setProperty("hive.metastore.metadb.dir", HIVE_METADB_DIR.getAbsolutePath());
-// System.setProperty("test.log.dir", HIVE_LOGS_DIR.getAbsolutePath());
-// System.setProperty("hive.querylog.location", HIVE_TMP_DIR.getAbsolutePath());
-// System.setProperty("hadoop.tmp.dir", HIVE_HADOOP_TMP_DIR.getAbsolutePath());
-// System.setProperty("derby.stream.error.file", HIVE_BASE_DIR.getAbsolutePath() + "/derby.log");
-//
-// // custom properties
-// System.setProperty("hive.server2.long.polling.timeout", "5000");
-//
-// HiveConf conf = new HiveConf();
-//
-// /* Build MiniDFSCluster */
-// try {
-// miniDFS = new MiniDFSCluster.Builder(conf).build();
-//
-// /* Build MiniMR Cluster */
-// int numTaskTrackers = 1;
-// int numTaskTrackerDirectories = 1;
-// String[] racks = null;
-// String[] hosts = null;
-// miniMR = new MiniMRCluster(numTaskTrackers, miniDFS.getFileSystem().getUri().toString(), numTaskTrackerDirectories, racks, hosts, new JobConf(conf));
-// JobConf jobConf = miniMR.createJobConf(new JobConf(conf));
-// System.out.println("-------" + jobConf.get("fs.defaultFS"));
-// System.out.println("-------" + miniDFS.getFileSystem().getUri().toString());
-// System.setProperty("mapred.job.tracker", jobConf.get("mapred.job.tracker"));
-// } catch (IOException e) {
-// e.printStackTrace();
-// System.exit(1);
-// }
-// }
-//
-// //@After
-// public void tearDown() {
-// if (miniMR != null)
-// miniMR.shutdown();
-//
-// if (miniDFS != null)
-// miniDFS.shutdown();
-//
-// super.tearDown();
-// }
-//
-// protected Connection getHiveConnection() throws SQLException {
-// return DriverManager.getConnection("jdbc:hive2:///", "", "");
-// }
-//
-// public static void main(String[] args) throws SQLException {
-// HiveMiniClusterTest test = new HiveMiniClusterTest();
-// test.runTests();
-// }
-//}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
index d5c3f60..d953d9c 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
@@ -47,267 +47,275 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
*
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext {
+public class MockupMapContext implements MapContext {
+
+ private Configuration hconf;
+
+ private Object[] outKV;
public static Context create(final Configuration hconf, String metadataUrl, String cubeName, final Object[] outKV) {
hconf.set(BatchConstants.CFG_CUBE_NAME, cubeName);
- return new WrappedMapper().getMapContext(new MapContext() {
-
- @Override
- public boolean nextKeyValue() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public Object getCurrentKey() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public Object getCurrentValue() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public void write(Object key, Object value) throws IOException, InterruptedException {
- System.out.println("Write -- k:" + key + ", v:" + value);
- if (outKV != null) {
- outKV[0] = key;
- outKV[1] = value;
- }
- }
-
- @Override
- public OutputCommitter getOutputCommitter() {
- throw new NotImplementedException();
- }
-
- @Override
- public TaskAttemptID getTaskAttemptID() {
- throw new NotImplementedException();
- }
-
- @Override
- public void setStatus(String msg) {
- throw new NotImplementedException();
- }
-
- @Override
- public String getStatus() {
- throw new NotImplementedException();
- }
-
- @Override
- public float getProgress() {
- throw new NotImplementedException();
- }
-
- @Override
- public Counter getCounter(Enum<?> counterName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Counter getCounter(String groupName, String counterName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Configuration getConfiguration() {
- return hconf;
- }
-
- @Override
- public Credentials getCredentials() {
- throw new NotImplementedException();
- }
-
- @Override
- public JobID getJobID() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getNumReduceTasks() {
- throw new NotImplementedException();
- }
-
- @Override
- public Path getWorkingDirectory() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getOutputKeyClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getOutputValueClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getMapOutputKeyClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getMapOutputValueClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getJobName() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Partitioner<?, ?>> getPartitionerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getSortComparator() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getJar() {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getGroupingComparator() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getJobSetupCleanupNeeded() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getTaskCleanupNeeded() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getProfileEnabled() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getProfileParams() {
- throw new NotImplementedException();
- }
-
- @Override
- public IntegerRanges getProfileTaskRange(boolean isMap) {
- throw new NotImplementedException();
- }
-
- @Override
- public String getUser() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getSymlink() {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getArchiveClassPaths() {
- throw new NotImplementedException();
- }
-
- @Override
- public URI[] getCacheArchives() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public URI[] getCacheFiles() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getLocalCacheArchives() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getLocalCacheFiles() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getFileClassPaths() {
- throw new NotImplementedException();
- }
-
- @Override
- public String[] getArchiveTimestamps() {
- throw new NotImplementedException();
- }
-
- @Override
- public String[] getFileTimestamps() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getMaxMapAttempts() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getMaxReduceAttempts() {
- throw new NotImplementedException();
- }
-
- @Override
- public void progress() {
- throw new NotImplementedException();
- }
-
- @Override
- public InputSplit getInputSplit() {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getCombinerKeyGroupingComparator() {
- throw new NotImplementedException();
- }
- });
+ return new WrappedMapper().getMapContext(new MockupMapContext(hconf, outKV));
+ }
+
+ public MockupMapContext(Configuration hconf, Object[] outKV){
+ this.hconf = hconf;
+ this.outKV = outKV;
+ }
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Object getCurrentKey() throws IOException, InterruptedException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Object getCurrentValue() throws IOException, InterruptedException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void write(Object key, Object value) throws IOException, InterruptedException {
+ System.out.println("Write -- k:" + key + ", v:" + value);
+ if (outKV != null) {
+ outKV[0] = key;
+ outKV[1] = value;
+ }
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public TaskAttemptID getTaskAttemptID() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void setStatus(String msg) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getStatus() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public float getProgress() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Counter getCounter(Enum<?> counterName) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Counter getCounter(String groupName, String counterName) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return hconf;
+ }
+
+ @Override
+ public Credentials getCredentials() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public JobID getJobID() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int getNumReduceTasks() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path getWorkingDirectory() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getOutputKeyClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getOutputValueClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getMapOutputKeyClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getMapOutputValueClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getJobName() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Partitioner<?, ?>> getPartitionerClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public RawComparator<?> getSortComparator() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getJar() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public RawComparator<?> getGroupingComparator() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getJobSetupCleanupNeeded() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getTaskCleanupNeeded() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getProfileEnabled() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getProfileParams() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public IntegerRanges getProfileTaskRange(boolean isMap) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getUser() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getSymlink() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getArchiveClassPaths() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public URI[] getCacheArchives() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public URI[] getCacheFiles() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getLocalCacheArchives() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getLocalCacheFiles() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getFileClassPaths() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String[] getArchiveTimestamps() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String[] getFileTimestamps() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int getMaxMapAttempts() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int getMaxReduceAttempts() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void progress() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public InputSplit getInputSplit() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public RawComparator<?> getCombinerKeyGroupingComparator() {
+ throw new NotImplementedException();
}
}
[24/42] kylin git commit: minor fix review bug
Posted by sh...@apache.org.
minor fix review bug
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76e321b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76e321b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76e321b2
Branch: refs/heads/sparkcubing-rebase
Commit: 76e321b2d95955959fe3fac030195bbd321f9659
Parents: bc1a49e
Author: zhongjian <ji...@163.com>
Authored: Wed Dec 28 15:22:02 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Wed Dec 28 15:22:02 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/cubes/cube_clone.html | 4 ++--
webapp/app/partials/models/model_clone.html | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/76e321b2/webapp/app/partials/cubes/cube_clone.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cube_clone.html b/webapp/app/partials/cubes/cube_clone.html
index f5a332c..a2d3ad1 100644
--- a/webapp/app/partials/cubes/cube_clone.html
+++ b/webapp/app/partials/cubes/cube_clone.html
@@ -39,8 +39,8 @@
<b>New Cube Name:</b>
<br/>
<input type="text" class="form-control" name="cube_name" ng-model="targetObj.cubeName" ng-pattern="/^\w+$/" required />
- <small class="help-block red" ng-show="forms.cube_clone_form.cube_name.$error.required && (forms.cube_clone_form.cube_name.$dirty||forms.cube_clone_form.$submitted)">Cube name is required.</small>
- <small class="help-block red" ng-show="!forms.cube_clone_form.cube_name.$error.required&&forms.cube_clone_form.cube_name.$invalid && (forms.cube_clone_form.cube_name.$dirty||forms.cube_clone_form.$submitted)">Cube name is invalid.</small>
+ <small class="help-block red" ng-show="forms.cube_clone_form.cube_name.$error.required && (forms.cube_clone_form.cube_name.$dirty)">Cube name is required.</small>
+ <small class="help-block red" ng-show="!forms.cube_clone_form.cube_name.$error.required&&forms.cube_clone_form.cube_name.$invalid && (forms.cube_clone_form.cube_name.$dirty)">Cube name is invalid.</small>
</div>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/76e321b2/webapp/app/partials/models/model_clone.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/models/model_clone.html b/webapp/app/partials/models/model_clone.html
index 484821a..2554633 100644
--- a/webapp/app/partials/models/model_clone.html
+++ b/webapp/app/partials/models/model_clone.html
@@ -50,8 +50,8 @@
<b>New Model Name:</b>
<br/>
<input type="text" name="model_name" class="form-control" ng-model="targetObj.modelName" ng-pattern="/^\w+$/" required />
- <small class="help-block red" ng-show="forms.model_clone_form.model_name.$error.required && (forms.model_clone_form.model_name.$dirty||forms.model_clone_form.$submitted)">Model name is required.</small>
- <small class="help-block red" ng-show="!forms.model_clone_form.model_name.$error.required&&forms.model_clone_form.model_name.$invalid && (forms.model_clone_form.model_name.$dirty||forms.model_clone_form.$submitted)">Model name is invalid.</small>
+ <small class="help-block red" ng-show="forms.model_clone_form.model_name.$error.required && (forms.model_clone_form.model_name.$dirty)">Model name is required.</small>
+ <small class="help-block red" ng-show="!forms.model_clone_form.model_name.$error.required&&forms.model_clone_form.model_name.$invalid && (forms.model_clone_form.model_name.$dirty)">Model name is invalid.</small>
</div>
</div>
</div>
[39/42] kylin git commit: KYLIN-2323 code review
Posted by sh...@apache.org.
KYLIN-2323 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a9ee6d33
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a9ee6d33
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a9ee6d33
Branch: refs/heads/sparkcubing-rebase
Commit: a9ee6d33eca53f67d9d4dd1da52b6fbfe98cb510
Parents: 5cbef32
Author: Yang Li <li...@apache.org>
Authored: Fri Dec 30 07:07:53 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Dec 30 07:07:53 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/util/StringUtil.java | 11 +++++++++++
.../apache/kylin/rest/controller/TableController.java | 3 ++-
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9ee6d33/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index bbc9448..12e7279 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -153,4 +153,15 @@ public class StringUtil {
}
}
+ public static String[] splitAndTrim(String str, String splitBy) {
+ String[] split = str.split(splitBy);
+ ArrayList<String> r = new ArrayList<>(split.length);
+ for (String s : split) {
+ s = s.trim();
+ if (!s.isEmpty())
+ r.add(s);
+ }
+ return (String[]) r.toArray(new String[r.size()]);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9ee6d33/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index eed5413..1ab6ba6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.exception.NotFoundException;
@@ -94,7 +95,7 @@ public class TableController extends BasicController {
public Map<String, String[]> loadHiveTables(@PathVariable String tables, @PathVariable String project, @RequestBody HiveTableRequest request) throws IOException {
String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
Map<String, String[]> result = new HashMap<String, String[]>();
- String[] tableNames = tables.split(",");
+ String[] tableNames = StringUtil.splitAndTrim(tables, ",");
try {
String[] loaded = tableService.loadHiveTablesToProject(tableNames, project);
result.put("result.loaded", loaded);