You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ka...@apache.org on 2017/04/14 12:54:51 UTC
[1/4] kylin git commit: Fix NPE in DistributedScheduler
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2506 0018fafc7 -> 55c0ae51e
Fix NPE in DistributedScheduler
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55c0ae51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55c0ae51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55c0ae51
Branch: refs/heads/KYLIN-2506
Commit: 55c0ae51e52089d73900ff7d29b172303aabedad
Parents: 5077b53
Author: kangkaisen <ka...@163.com>
Authored: Fri Apr 14 20:10:27 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Fri Apr 14 20:47:30 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/job/impl/threadpool/DistributedScheduler.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c0ae51/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index b99da7c..c5b03dc 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -294,11 +294,11 @@ public class DistributedScheduler implements Scheduler<AbstractExecutable>, Conn
jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS, new SynchronousQueue<Runnable>());
context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());
- resumeAllRunningJobs();
-
fetcher = new FetcherRunner();
fetcherPool.scheduleAtFixedRate(fetcher, 10, ExecutableConstants.DEFAULT_SCHEDULER_INTERVAL_SECONDS, TimeUnit.SECONDS);
hasStarted = true;
+
+ resumeAllRunningJobs();
}
private void resumeAllRunningJobs() {
[4/4] kylin git commit: KYLIN-2543 Still build dictionary for TopN
group
Posted by ka...@apache.org.
KYLIN-2543 Still build dictionary for TopN group
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca01e096
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca01e096
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca01e096
Branch: refs/heads/KYLIN-2506
Commit: ca01e096acc0338ac343a861253838e4591e01e0
Parents: 0018faf
Author: shaofengshi <sh...@apache.org>
Authored: Thu Apr 13 19:34:22 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Fri Apr 14 20:47:30 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca01e096/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index b7252a0..f90aab2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -221,7 +221,11 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
int start = (functionDesc.getParameter().isColumnType() == true) ? 1 : 0;
for (int i = start; i < allCols.size(); i++) {
TblColRef tblColRef = allCols.get(i);
- String encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getName());
+ String encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getIdentity());
+ if (StringUtils.isEmpty(encoding)) {
+ // for backward compatibility
+ encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getName());
+ }
if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) {
columnsNeedDict.add(tblColRef);
}
[3/4] kylin git commit: minor, fix beeline-params bug in sample.sh
Posted by ka...@apache.org.
minor, fix beeline-params bug in sample.sh
Signed-off-by: Billy Liu <bi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5077b53f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5077b53f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5077b53f
Branch: refs/heads/KYLIN-2506
Commit: 5077b53fd1aed94b126087ccc488ad8a03614d6c
Parents: 9846a82
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Apr 14 11:08:37 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Fri Apr 14 20:47:30 2017 +0800
----------------------------------------------------------------------
build/bin/sample.sh | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5077b53f/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index f4d63d8..617b671 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -44,8 +44,13 @@ if [ "${hive_client_mode}" == "beeline" ]
then
beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$sample_database
- hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'`
- beeline_params=${beeline_params//${hive2_url}/${hive2_url}/${sample_database}}
+ hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\/\)'`
+ if [ -z ${hive2_url} ]; then
+ hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'`
+ beeline_params=${beeline_params/${hive2_url}/${hive2_url}/${sample_database}}
+ else
+ beeline_params=${beeline_params/${hive2_url}/${hive2_url}${sample_database}}
+ fi
beeline ${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
else
hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database
[2/4] kylin git commit: KYLIN-2543 Still build dictionary for TopN
group, fix IT
Posted by ka...@apache.org.
KYLIN-2543 Still build dictionary for TopN group, fix IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9846a827
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9846a827
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9846a827
Branch: refs/heads/KYLIN-2506
Commit: 9846a827d1869f19eede53122feae71c2350253f
Parents: ca01e09
Author: shaofengshi <sh...@apache.org>
Authored: Fri Apr 14 10:32:48 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Fri Apr 14 20:47:30 2017 +0800
----------------------------------------------------------------------
.../metadata/measure/TopNMeasureTypeTest.java | 2 +-
.../kylin/measure/topn/TopNMeasureType.java | 32 +++++++++++++++-----
2 files changed, 26 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9846a827/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
index 593eb6e..f04c742 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
@@ -69,7 +69,7 @@ public class TopNMeasureTypeTest extends LocalFileMetadataTestCase {
assertTrue(colsNeedDict != null && colsNeedDict.size() == 1);
TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1);
- topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getName(), "int:6");
+ topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6");
colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());
assertTrue(colsNeedDict.size() == 0);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9846a827/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index f90aab2..f936cb8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -27,6 +27,7 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.dimension.DateDimEnc;
import org.apache.kylin.dimension.DictionaryDimEnc;
import org.apache.kylin.dimension.DimensionEncoding;
@@ -221,11 +222,7 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
int start = (functionDesc.getParameter().isColumnType() == true) ? 1 : 0;
for (int i = start; i < allCols.size(); i++) {
TblColRef tblColRef = allCols.get(i);
- String encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getIdentity());
- if (StringUtils.isEmpty(encoding)) {
- // for backward compatibility
- encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getName());
- }
+ String encoding = getEncoding(functionDesc, tblColRef).getFirst();
if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) {
columnsNeedDict.add(tblColRef);
}
@@ -415,8 +412,10 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
final DimensionEncoding[] dimensionEncodings = new DimensionEncoding[literalCols.size()];
for (int i = 0; i < literalCols.size(); i++) {
TblColRef colRef = literalCols.get(i);
- String encoding = function.getConfiguration().get(TopNMeasureType.CONFIG_ENCODING_PREFIX + colRef.getName());
- String encodingVersionStr = function.getConfiguration().get(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + colRef.getName());
+
+ Pair<String, String> topNEncoding = TopNMeasureType.getEncoding(function, colRef);
+ String encoding = topNEncoding.getFirst();
+ String encodingVersionStr = topNEncoding.getSecond();
if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) {
dimensionEncodings[i] = new DictionaryDimEnc(dictionaryMap.get(colRef));
} else {
@@ -460,4 +459,23 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
return FUNC_TOP_N.equalsIgnoreCase(functionDesc.getExpression());
}
+
+ /**
+ * Get the encoding name and version for the given col from Measure FunctionDesc
+ * @param functionDesc
+ * @param tblColRef
+ * @return a pair of the encoding name and encoding version
+ */
+ public static final Pair<String, String> getEncoding(FunctionDesc functionDesc, TblColRef tblColRef) {
+ String encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getIdentity());
+ String encodingVersion =functionDesc.getConfiguration().get(CONFIG_ENCODING_VERSION_PREFIX + tblColRef.getIdentity());
+ if (StringUtils.isEmpty(encoding)) {
+ // for backward compatibility
+ encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getName());
+ encodingVersion =functionDesc.getConfiguration().get(CONFIG_ENCODING_VERSION_PREFIX + tblColRef.getName());
+ }
+
+ return new Pair<>(encoding, encodingVersion);
+ }
+
}