You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/04/01 00:56:14 UTC
kylin git commit: support azure hdinsight counters
Repository: kylin
Updated Branches:
refs/heads/yang22 6617955b4 -> 09e754508
support azure hdinsight counters
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/09e75450
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/09e75450
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/09e75450
Branch: refs/heads/yang22
Commit: 09e75450879281b5921ffa24cd5fc6edc554ee73
Parents: 6617955
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 8 11:23:29 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Apr 1 08:56:09 2017 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/common/HadoopCmdOutput.java | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/09e75450/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
index 9d016cc..023717f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskCounter;
@@ -93,9 +94,18 @@ public class HadoopCmdOutput {
this.output.append(counters.toString()).append("\n");
logger.debug(counters.toString());
+ String bytsWrittenCounterName = "HDFS_BYTES_WRITTEN";
+ String bytsReadCounterName = "HDFS_BYTES_READ";
+ String fsScheme = FileSystem.get(job.getConfiguration()).getScheme();
+ if (("wasb").equalsIgnoreCase(fsScheme)) {
+ // for Azure blob store
+ bytsWrittenCounterName = "WASB_BYTES_WRITTEN";
+ bytsReadCounterName = "WASB_BYTES_READ";
+ }
+
mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
- hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue());
- hdfsBytesRead = String.valueOf(counters.findCounter("FileSystemCounters", "HDFS_BYTES_READ").getValue());
+ hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", bytsWrittenCounterName).getValue());
+ hdfsBytesRead = String.valueOf(counters.findCounter("FileSystemCounters", bytsReadCounterName).getValue());
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
output.append(e.getLocalizedMessage());