You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/11/22 06:00:31 UTC
[1/2] kylin git commit: KYLIN-3032 Cube size shows 0 but actually it
is not empty
Repository: kylin
Updated Branches:
refs/heads/master e4b1d7e0b -> b02e7bd24
KYLIN-3032 Cube size shows 0 but actually it is not empty
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b02e7bd2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b02e7bd2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b02e7bd2
Branch: refs/heads/master
Commit: b02e7bd24cc724e03023706eac6389bfe71ebb3e
Parents: 2b9470c
Author: shaofengshi <sh...@apache.org>
Authored: Wed Nov 22 13:57:25 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Nov 22 13:58:11 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/engine/mr/common/HadoopCmdOutput.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b02e7bd2/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
index 11d6d2c..d82b988 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
@@ -22,11 +22,12 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.FileSystemCounter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskCounter;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.steps.FactDistinctColumnsMapper.RawDataCounter;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.slf4j.Logger;
@@ -99,9 +100,13 @@ public class HadoopCmdOutput {
mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue());
- String fsScheme = FileSystem.get(job.getConfiguration()).getScheme();
+ String outputFolder = job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir", KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
+ logger.debug("outputFolder is " + outputFolder);
+ Path outputPath = new Path(outputFolder);
+ String fsScheme = outputPath.getFileSystem(job.getConfiguration()).getScheme();
long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue();
if (bytesWritten == 0) {
+ logger.debug("Seems no counter found for " + fsScheme);
bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue();
}
hdfsBytesWritten = String.valueOf(bytesWritten);
[2/2] kylin git commit: KYLIN-3028 Build cube error when set S3 as
working-dir
Posted by sh...@apache.org.
KYLIN-3028 Build cube error when set S3 as working-dir
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2b9470c3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2b9470c3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2b9470c3
Branch: refs/heads/master
Commit: 2b9470c38243c4678fb5d975664c82e7a9a87805
Parents: e4b1d7e
Author: shaofengshi <sh...@apache.org>
Authored: Wed Nov 22 13:56:06 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Nov 22 13:58:11 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/storage/hbase/HBaseConnection.java | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2b9470c3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index ef26334..16bd20d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -154,6 +154,14 @@ public class HBaseConnection {
String hbaseClusterFs = kylinConf.getHBaseClusterFs();
if (StringUtils.isNotEmpty(hbaseClusterFs)) {
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, hbaseClusterFs);
+ } else {
+ try {
+ FileSystem fs = HadoopUtil.getWorkingFileSystem(HadoopUtil.getCurrentConfiguration());
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, fs.getUri().toString());
+ logger.debug("Using the working dir FS for HBase: " + fs.getUri().toString());
+ } catch (IOException e) {
+ logger.error("Fail to set working dir to HBase configuration", e);
+ }
}
// https://issues.apache.org/jira/browse/KYLIN-953