You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/11/26 10:34:58 UTC
[kylin] branch 2.5.x updated (1a7789c -> 48a2bec)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 1a7789c KYLIN-3693 TopN incorrect in Spark engine
new b4280a1 KYLIN-3666 HDFS metadata url not be recognized
new 48a2bec KYLIN-3669, add log to indicate the case when using GTStreamAggregateScanner.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 1 +
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 15 +++++++++------
.../kylin/engine/mr/steps/UpdateDictionaryStep.java | 3 ++-
3 files changed, 12 insertions(+), 7 deletions(-)
[kylin] 02/02: KYLIN-3669,
add log to indicate the case when using GTStreamAggregateScanner.
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 48a2bec01caa7b764d55bfa74b3da5a4936c598d
Author: Jiatao Tao <24...@qq.com>
AuthorDate: Tue Nov 20 17:11:21 2018 +0800
KYLIN-3669, add log to indicate the case when using GTStreamAggregateScanner.
---
.../java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 6711664..7fef9f5 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -92,6 +92,7 @@ public class SegmentCubeTupleIterator implements ITupleIterator {
boolean hasMultiplePartitions = records instanceof SortMergedPartitionResultIterator;
if (hasMultiplePartitions && context.isStreamAggregateEnabled()) {
+ logger.info("Using GTStreamAggregateScanner to pre-aggregate storage partition.");
// input records are ordered, leverage stream aggregator to produce possibly fewer records
IGTScanner inputScanner = new IGTScanner() {
public GTInfo getInfo() {
[kylin] 01/02: KYLIN-3666 HDFS metadata url not be recognized
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b4280a13469865f847b6c985d19a98dcd9c59b44
Author: chao long <wa...@qq.com>
AuthorDate: Wed Nov 21 11:29:59 2018 +0800
KYLIN-3666 HDFS metadata url not be recognized
---
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 15 +++++++++------
.../kylin/engine/mr/steps/UpdateDictionaryStep.java | 3 ++-
2 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 8873f30..fe20643 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -490,7 +490,14 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
public static KylinConfig loadKylinConfigFromHdfs(SerializableConfiguration conf, String uri) {
HadoopUtil.setCurrentConfiguration(conf.get());
- return loadKylinConfigFromHdfs(uri);
+ KylinConfig config = loadKylinConfigFromHdfs(uri);
+
+ // This is a bad example where the thread local KylinConfig cannot be auto-closed due to
+ // limitation of MR API. It works because MR task runs its own process. Do not copy.
+ @SuppressWarnings("unused")
+ SetAndUnsetThreadLocalConfig shouldAutoClose = KylinConfig.setAndUnsetThreadLocalConfig(config);
+
+ return config;
}
public static KylinConfig loadKylinConfigFromHdfs(String uri) {
@@ -517,11 +524,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
} catch (IOException e) {
throw new RuntimeException(e);
}
-
- // This is a bad example where the thread local KylinConfig cannot be auto-closed due to
- // limitation of MR API. It works because MR task runs its own process. Do not copy.
- @SuppressWarnings("unused")
- SetAndUnsetThreadLocalConfig shouldAutoClose = KylinConfig.setAndUnsetThreadLocalConfig(config);
+
kylinConfigCache.put(uri, config);
return config;
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
index bcdb29f..28c567a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
@@ -43,6 +43,7 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -67,7 +68,7 @@ public class UpdateDictionaryStep extends AbstractExecutable {
final String metadataUrl = this.getParams().get(BatchConstants.ARG_META_URL);
final KylinConfig kylinConfHbase = cube.getConfig();
- final KylinConfig kylinConfHdfs = KylinConfig.createInstanceFromUri(metadataUrl);
+ final KylinConfig kylinConfHdfs = AbstractHadoopJob.loadKylinConfigFromHdfs(metadataUrl);
Collections.sort(mergingSegments);