You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/07/27 13:53:27 UTC
kylin git commit: KYLIN-2757 support getting counters for non-hdfs
storage
Repository: kylin
Updated Branches:
refs/heads/2.1.x c8a0622d6 -> c40944e1a
KYLIN-2757 support getting counters for non-hdfs storage
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c40944e1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c40944e1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c40944e1
Branch: refs/heads/2.1.x
Commit: c40944e1a230ac9f9d699e65e9fae5f62e1f0647
Parents: c8a0622
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jul 27 21:53:12 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jul 27 21:53:12 2017 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/common/HadoopCmdOutput.java | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c40944e1/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
index 5da1947..11d6d2c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.FileSystemCounter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.kylin.engine.mr.steps.FactDistinctColumnsMapper.RawDataCounter;
@@ -94,17 +95,17 @@ public class HadoopCmdOutput {
}
this.output.append(counters.toString()).append("\n");
logger.debug(counters.toString());
-
- String bytsWrittenCounterName = "HDFS_BYTES_WRITTEN";
- String fsScheme = FileSystem.get(job.getConfiguration()).getScheme();
- if (("wasb").equalsIgnoreCase(fsScheme)) {
- // for Azure blob store
- bytsWrittenCounterName = "WASB_BYTES_WRITTEN";
- }
mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
- hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", bytsWrittenCounterName).getValue());
rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue());
+
+ String fsScheme = FileSystem.get(job.getConfiguration()).getScheme();
+ long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue();
+ if (bytesWritten == 0) {
+ bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue();
+ }
+ hdfsBytesWritten = String.valueOf(bytesWritten);
+
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
output.append(e.getLocalizedMessage());