You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by yo...@apache.org on 2016/09/07 17:42:22 UTC
[26/52] [abbrv] incubator-eagle git commit: [EAGLE-512] add job
count(total&fail) for mr history feeder
[EAGLE-512] add job count(total&fail) for mr history feeder
Author: wujinhu <wu...@126.com>
Closes #406 from wujinhu/EAGLE-512.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4aa5b452
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4aa5b452
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4aa5b452
Branch: refs/heads/master
Commit: 4aa5b45260e9bf4ea029a84dd04ed87eae4dc38b
Parents: da7f552
Author: wujinhu <wu...@126.com>
Authored: Tue Aug 30 20:23:38 2016 +0800
Committer: Qingwen Zhao <qi...@gmail.com>
Committed: Tue Aug 30 20:23:38 2016 +0800
----------------------------------------------------------------------
.../mr/historyentity/JPAEntityRepository.java | 1 +
.../jpm/mr/historyentity/JobCountEntity.java | 56 ++++++
.../history/crawler/JHFCrawlerDriverImpl.java | 51 ++++-
.../JobEntityCreationEagleServiceListener.java | 20 ++
.../jpm/mr/history/storm/JobHistorySpout.java | 4 +-
.../mr/history/zkres/JobHistoryZKStateLCM.java | 6 +
.../history/zkres/JobHistoryZKStateManager.java | 46 +++++
.../org/apache/eagle/jpm/util/Constants.java | 1 +
.../src/main/resources/JobCounter.conf | 187 +++++++++++++++++++
9 files changed, 370 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4aa5b452/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java
index 49095ed..2303869 100644
--- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java
+++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java
@@ -36,5 +36,6 @@ public class JPAEntityRepository extends EntityRepository {
entitySet.add(TaskFailureCountAPIEntity.class);
entitySet.add(TaskAttemptCounterAPIEntity.class);
entitySet.add(JobProcessTimeStampEntity.class);
+ entitySet.add(JobCountEntity.class);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4aa5b452/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java
new file mode 100644
index 0000000..f18188a
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.mr.historyentity;
+
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
+import org.apache.eagle.log.entity.meta.*;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@Table("eaglejcount")
+@ColumnFamily("f")
+@Prefix("jcount")
+@Service(Constants.JPA_JOB_COUNT_SERVICE_NAME)
+@TimeSeries(true)
+@Partition({"site"})
+public class JobCountEntity extends TaggedLogAPIEntity {
+ @Column("a")
+ private int total;
+ @Column("b")
+ private int fail;
+
+ public int getTotal() {
+ return total;
+ }
+
+ public void setTotal(int total) {
+ this.total = total;
+ valueChanged("total");
+ }
+
+ public int getFail() {
+ return fail;
+ }
+
+ public void setFail(int fail) {
+ this.fail = fail;
+ valueChanged("fail");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4aa5b452/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
index 278599d..e16ecce 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
@@ -19,9 +19,13 @@
package org.apache.eagle.jpm.mr.history.crawler;
import org.apache.eagle.jpm.mr.history.MRHistoryJobConfig;
+import org.apache.eagle.jpm.mr.history.parser.EagleJobStatus;
import org.apache.eagle.jpm.mr.history.zkres.JobHistoryZKStateLCM;
+import org.apache.eagle.jpm.mr.historyentity.JobCountEntity;
import org.apache.eagle.jpm.util.JobIdFilter;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
@@ -59,11 +63,16 @@ public class JHFCrawlerDriverImpl implements JHFCrawlerDriver {
private JobIdFilter jobFilter;
private int partitionId;
private TimeZone timeZone;
+ private MRHistoryJobConfig.EagleServiceConfig eagleServiceConfig;
+ private MRHistoryJobConfig.JobExtractorConfig jobExtractorConfig;
- public JHFCrawlerDriverImpl(MRHistoryJobConfig.JobHistoryEndpointConfig jobHistoryConfig,
+ public JHFCrawlerDriverImpl(MRHistoryJobConfig.EagleServiceConfig eagleServiceConfig,
+ MRHistoryJobConfig.JobExtractorConfig jobExtractorConfig,
MRHistoryJobConfig.ControlConfig controlConfig, JHFInputStreamCallback reader,
JobHistoryZKStateLCM zkStateLCM,
JobHistoryLCM historyLCM, JobIdFilter jobFilter, int partitionId) throws Exception {
+ this.eagleServiceConfig = eagleServiceConfig;
+ this.jobExtractorConfig = jobExtractorConfig;
this.zeroBasedMonth = controlConfig.zeroBasedMonth;
this.dryRun = controlConfig.dryRun;
if (this.dryRun) {
@@ -185,6 +194,7 @@ public class JHFCrawlerDriverImpl implements JHFCrawlerDriver {
jobHistoryFile);
processedJobFileNames.add(jobHistoryFile);
+ flushJobCount();
Long modifiedTime = item.getLeft();
return modifiedTime;
}
@@ -230,7 +240,46 @@ public class JHFCrawlerDriverImpl implements JHFCrawlerDriver {
}
}
+ private void flushJobCount() throws Exception {
+ List<Pair<String, String>> jobs = zkStateLcm.getProcessedJobs(
+ String.format(FORMAT_JOB_PROCESS_DATE, this.processDate.year, this.processDate.month + 1, this.processDate.day)
+ );
+ JobCountEntity entity = new JobCountEntity();
+ entity.setTotal(jobs.size());
+ entity.setFail(0);
+ jobs.stream().filter(job -> !job.getRight().equals(EagleJobStatus.SUCCESS.toString())).forEach(
+ job -> entity.setFail(1 + entity.getFail())
+ );
+
+ IEagleServiceClient client = new EagleServiceClientImpl(
+ eagleServiceConfig.eagleServiceHost,
+ eagleServiceConfig.eagleServicePort,
+ eagleServiceConfig.username,
+ eagleServiceConfig.password);
+
+
+ GregorianCalendar cal = new GregorianCalendar(this.processDate.year, this.processDate.month, this.processDate.day, 0, 0, 0);
+ cal.setTimeZone(timeZone);
+ entity.setTimestamp(cal.getTimeInMillis());
+ @SuppressWarnings("serial")
+ Map<String, String> baseTags = new HashMap<String, String>() {
+ {
+ put("site", jobExtractorConfig.site);
+ }
+ };
+ entity.setTags(baseTags);
+ List<JobCountEntity> entities = new ArrayList<>();
+ entities.add(entity);
+
+ LOG.info("start flushing entities of total number " + entities.size());
+ client.create(entities);
+ LOG.info("finish flushing entities of total number " + entities.size());
+ client.getJerseyClient().destroy();
+ client.close();
+ }
+
private void advanceOneDay() throws Exception {
+ //flushJobCount();
GregorianCalendar cal = new GregorianCalendar(timeZone);
cal.set(this.processDate.year, this.processDate.month, this.processDate.day, 0, 0, 0);
cal.add(Calendar.DATE, 1);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4aa5b452/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationEagleServiceListener.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationEagleServiceListener.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationEagleServiceListener.java
index e7b8a6b..a681aca 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationEagleServiceListener.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationEagleServiceListener.java
@@ -19,7 +19,9 @@
package org.apache.eagle.jpm.mr.history.parser;
import org.apache.eagle.jpm.mr.history.MRHistoryJobConfig;
+import org.apache.eagle.jpm.mr.history.zkres.JobHistoryZKStateManager;
import org.apache.eagle.jpm.mr.historyentity.*;
+import org.apache.eagle.jpm.util.MRJobTagName;
import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
import org.apache.eagle.service.client.IEagleServiceClient;
import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
@@ -27,7 +29,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.GregorianCalendar;
import java.util.List;
+import java.util.TimeZone;
public class JobEntityCreationEagleServiceListener implements HistoryJobEntityCreationListener {
private static final Logger logger = LoggerFactory.getLogger(JobEntityCreationEagleServiceListener.class);
@@ -39,6 +43,8 @@ public class JobEntityCreationEagleServiceListener implements HistoryJobEntityCr
List<JobEventAPIEntity> jobEvents = new ArrayList<>();
List<TaskExecutionAPIEntity> taskExecs = new ArrayList<>();
List<TaskAttemptExecutionAPIEntity> taskAttemptExecs = new ArrayList<>();
+ private JobHistoryZKStateManager zkState;
+ private TimeZone timeZone;
public JobEntityCreationEagleServiceListener(MRHistoryJobConfig configManager) {
this(configManager, BATCH_SIZE);
@@ -50,6 +56,8 @@ public class JobEntityCreationEagleServiceListener implements HistoryJobEntityCr
throw new IllegalArgumentException("batchSize must be greater than 0 when it is provided");
}
this.batchSize = batchSize;
+ zkState = new JobHistoryZKStateManager(configManager.getZkStateConfig());
+ timeZone = TimeZone.getTimeZone(configManager.getControlConfig().timeZone);
}
@Override
@@ -61,6 +69,15 @@ public class JobEntityCreationEagleServiceListener implements HistoryJobEntityCr
}
}
+ private String timeStamp2Date(long timeStamp) {
+ GregorianCalendar cal = new GregorianCalendar(timeZone);
+ cal.setTimeInMillis(timeStamp);
+ return String.format("%4d%02d%02d",
+ cal.get(GregorianCalendar.YEAR),
+ cal.get(GregorianCalendar.MONTH) + 1,
+ cal.get(GregorianCalendar.DAY_OF_MONTH));
+ }
+
/**
* We need save network bandwidth as well.
*/
@@ -80,6 +97,9 @@ public class JobEntityCreationEagleServiceListener implements HistoryJobEntityCr
JobBaseAPIEntity entity = list.get(i);
if (entity instanceof JobExecutionAPIEntity) {
jobs.add((JobExecutionAPIEntity) entity);
+ this.zkState.updateProcessedJob(timeStamp2Date(entity.getTimestamp()),
+ entity.getTags().get(MRJobTagName.JOB_ID.toString()),
+ ((JobExecutionAPIEntity) entity).getCurrentState());
} else if (entity instanceof JobEventAPIEntity) {
jobEvents.add((JobEventAPIEntity) entity);
} else if (entity instanceof TaskExecutionAPIEntity) {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4aa5b452/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
index ca4a94f..402f93e 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
@@ -149,7 +149,9 @@ public class JobHistorySpout extends BaseRichSpout {
try {
jhfLCM = new JobHistoryDAOImpl(configManager.getJobHistoryEndpointConfig());
- driver = new JHFCrawlerDriverImpl(configManager.getJobHistoryEndpointConfig(),
+ driver = new JHFCrawlerDriverImpl(
+ configManager.getEagleServiceConfig(),
+ configManager.getJobExtractorConfig(),
configManager.getControlConfig(),
callback,
zkState,
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4aa5b452/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/zkres/JobHistoryZKStateLCM.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/zkres/JobHistoryZKStateLCM.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/zkres/JobHistoryZKStateLCM.java
index cbde88c..0adcffe 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/zkres/JobHistoryZKStateLCM.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/zkres/JobHistoryZKStateLCM.java
@@ -18,6 +18,8 @@
package org.apache.eagle.jpm.mr.history.zkres;
+import org.apache.commons.lang3.tuple.Pair;
+
import java.util.List;
public interface JobHistoryZKStateLCM {
@@ -31,6 +33,8 @@ public interface JobHistoryZKStateLCM {
void addProcessedJob(String date, String jobId);
+ void updateProcessedJob(String date, String jobId, String status);
+
void truncateProcessedJob(String date);
void truncateEverything();
@@ -38,4 +42,6 @@ public interface JobHistoryZKStateLCM {
long readProcessedTimeStamp(int partitionId);
void updateProcessedTimeStamp(int partitionId, long timeStamp);
+
+ List<Pair<String, String>> getProcessedJobs(String date);
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4aa5b452/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/zkres/JobHistoryZKStateManager.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/zkres/JobHistoryZKStateManager.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/zkres/JobHistoryZKStateManager.java
index feb896e..c61d05a 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/zkres/JobHistoryZKStateManager.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/zkres/JobHistoryZKStateManager.java
@@ -18,6 +18,7 @@
package org.apache.eagle.jpm.mr.history.zkres;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.eagle.jpm.mr.history.MRHistoryJobConfig.ZKStateConfig;
import org.apache.curator.framework.CuratorFramework;
@@ -29,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
@@ -40,6 +42,7 @@ public class JobHistoryZKStateManager implements JobHistoryZKStateLCM {
public static final String ZNODE_FORCE_START_FROM = "forceStartFrom";
public static final String ZNODE_PARTITIONS = "partitions";
public static final String ZNODE_JOBS = "jobs";
+ public static final String ZNODE_JOB_IDS = "jobIds";
public static final String ZNODE_TIMESTAMPS = "timeStamps";
public static final int BACKOFF_DAYS = 0;
@@ -268,6 +271,12 @@ public class JobHistoryZKStateManager implements JobHistoryZKStateLCM {
_curator.delete().deletingChildrenIfNeeded().forPath(path);
LOG.info("really truncated all data for day " + date);
}
+
+ String jobIdPath = zkRoot + "/" + ZNODE_JOB_IDS + "/" + date;
+ if (_curator.checkExists().forPath(jobIdPath) != null) {
+ _curator.delete().deletingChildrenIfNeeded().forPath(jobIdPath);
+ LOG.info("really truncated all jobIds for day " + date);
+ }
} catch (Exception e) {
LOG.error("fail truncating processed jobs", e);
throw new RuntimeException(e);
@@ -345,4 +354,41 @@ public class JobHistoryZKStateManager implements JobHistoryZKStateLCM {
throw new RuntimeException(e);
}
}
+
+ @Override
+ public List<Pair<String, String>> getProcessedJobs(String date) {
+ List<Pair<String, String>> result = new ArrayList<>();
+ String path = zkRoot + "/" + ZNODE_JOB_IDS + "/" + date;
+ try {
+ if (_curator.checkExists().forPath(path) != null) {
+ List<String> jobs = _curator.getChildren().forPath(path);
+ for (String job : jobs) {
+ String jobPath = path + "/" + job;
+ String status = new String(_curator.getData().forPath(jobPath), "UTF-8");
+ result.add(Pair.of(job, status));
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("fail read processed jobs", e);
+ throw new RuntimeException(e);
+ }
+ return result;
+ }
+
+ @Override
+ public void updateProcessedJob(String date, String jobId, String status) {
+ String path = zkRoot + "/" + ZNODE_JOB_IDS + "/" + date + "/" + jobId;
+ try {
+ if (_curator.checkExists().forPath(path) == null) {
+ _curator.create()
+ .creatingParentsIfNeeded()
+ .withMode(CreateMode.PERSISTENT)
+ .forPath(path);
+ }
+ _curator.setData().forPath(path, status.getBytes("UTF-8"));
+ } catch (Exception e) {
+ LOG.error("fail adding processed jobs", e);
+ throw new RuntimeException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4aa5b452/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
index 1a6c09c..7dce0a2 100644
--- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
+++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
@@ -101,6 +101,7 @@ public class Constants {
public static final String JPA_JOB_CONFIG_SERVICE_NAME = "JobConfigService";
public static final String JPA_JOB_EVENT_SERVICE_NAME = "JobEventService";
public static final String JPA_JOB_EXECUTION_SERVICE_NAME = "JobExecutionService";
+ public static final String JPA_JOB_COUNT_SERVICE_NAME = "JobCountService";
public static final String JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME = "RunningJobExecutionService";
public static final String JPA_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "TaskAttemptExecutionService";
public static final String JPA_TASK_FAILURE_COUNT_SERVICE_NAME = "TaskFailureCountService";
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4aa5b452/eagle-jpm/eagle-jpm-util/src/main/resources/JobCounter.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-util/src/main/resources/JobCounter.conf b/eagle-jpm/eagle-jpm-util/src/main/resources/JobCounter.conf
new file mode 100644
index 0000000..1524e61
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-util/src/main/resources/JobCounter.conf
@@ -0,0 +1,187 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#### Sample configuration:
+## counter.group0.name = groupname1
+## counter.group0.counter0.names = counterName1,counterName2,...
+## counter.group0.counter0.description = counter description...
+
+counter.group0.name = org.apache.hadoop.mapreduce.FileSystemCounter
+counter.group0.description = File System Counters
+counter.group0.counter0.names = FILE_BYTES_READ
+counter.group0.counter0.description = FILE: Number of bytes read
+counter.group0.counter1.names = FILE_BYTES_WRITTEN
+counter.group0.counter1.description = FILE: Number of bytes written
+counter.group0.counter2.names = FILE_READ_OPS
+counter.group0.counter2.description = FILE: Number of read operations
+counter.group0.counter3.names = FILE_LARGE_READ_OPS
+counter.group0.counter3.description = FILE: Number of large read operations
+counter.group0.counter4.names = FILE_WRITE_OPS
+counter.group0.counter4.description = FILE: Number of write operations
+counter.group0.counter5.names = HDFS_BYTES_READ
+counter.group0.counter5.description = HDFS: Number of bytes read
+counter.group0.counter6.names = HDFS_BYTES_WRITTEN
+counter.group0.counter6.description = HDFS: Number of bytes written
+counter.group0.counter7.names = HDFS_READ_OPS
+counter.group0.counter7.description = HDFS: Number of read operations
+counter.group0.counter8.names = HDFS_LARGE_READ_OPS
+counter.group0.counter8.description = HDFS: Number of large read operations
+counter.group0.counter9.names = HDFS_WRITE_OPS
+counter.group0.counter9.description = HDFS: Number of write operations
+
+counter.group1.name = org.apache.hadoop.mapreduce.TaskCounter
+counter.group1.description = Map-Reduce Framework
+counter.group1.counter0.names = MAP_INPUT_RECORDS
+counter.group1.counter0.description = Map input records
+counter.group1.counter1.names = MAP_OUTPUT_RECORDS
+counter.group1.counter1.description = Map output records
+counter.group1.counter2.names = SPLIT_RAW_BYTES
+counter.group1.counter2.description = Input split bytes
+counter.group1.counter3.names = SPILLED_RECORDS
+counter.group1.counter3.description = Spilled Records
+counter.group1.counter4.names = CPU_MILLISECONDS
+counter.group1.counter4.description = CPU time spent (ms)
+counter.group1.counter5.names = PHYSICAL_MEMORY_BYTES
+counter.group1.counter5.description = Physical memory (bytes) snapshot
+counter.group1.counter6.names = VIRTUAL_MEMORY_BYTES
+counter.group1.counter6.description = Virtual memory (bytes) snapshot
+counter.group1.counter7.names = COMMITTED_HEAP_BYTES
+counter.group1.counter7.description = Total committed heap usage (bytes)
+counter.group1.counter8.names = REDUCE_SHUFFLE_BYTES
+counter.group1.counter8.description = Reduce shuffle bytes (bytes)
+counter.group1.counter9.names = GC_TIME_MILLIS
+counter.group1.counter9.description = GC time milliseconds
+counter.group1.counter10.names = MAP_OUTPUT_BYTES
+counter.group1.counter10.description = map output bytes
+counter.group1.counter11.names = REDUCE_INPUT_RECORDS
+counter.group1.counter11.description = reduce input records
+counter.group1.counter12.names = COMBINE_INPUT_RECORDS
+counter.group1.counter12.description = combine input records
+counter.group1.counter13.names = COMBINE_OUTPUT_RECORDS
+counter.group1.counter13.description = combine output records
+counter.group1.counter14.names = REDUCE_INPUT_GROUPS
+counter.group1.counter14.description = reduce input groups
+counter.group1.counter15.names = REDUCE_OUTPUT_RECORDS
+counter.group1.counter15.description = reduce output records
+counter.group1.counter16.names = SHUFFLED_MAPS
+counter.group1.counter16.description = shuffled maps
+counter.group1.counter17.names = MAP_OUTPUT_MATERIALIZED_BYTES
+counter.group1.counter17.description = MAP_OUTPUT_MATERIALIZED_BYTES
+counter.group1.counter18.names = MERGED_MAP_OUTPUTS
+counter.group1.counter18.description = MERGED_MAP_OUTPUTS
+counter.group1.counter19.names = FAILED_SHUFFLE
+counter.group1.counter19.description = FAILED_SHUFFLE
+
+counter.group2.name = org.apache.hadoop.mapreduce.JobCounter
+counter.group2.description = Map-Reduce Job Counter
+counter.group2.counter0.names = MB_MILLIS_MAPS
+counter.group2.counter0.description = Total megabyte-seconds taken by all map tasks
+counter.group2.counter1.names = MB_MILLIS_REDUCES
+counter.group2.counter1.description = Total megabyte-seconds taken by all reduce tasks
+counter.group2.counter2.names = VCORES_MILLIS_MAPS
+counter.group2.counter2.description = Total vcore-seconds taken by all map tasks
+counter.group2.counter3.names = VCORES_MILLIS_REDUCES
+counter.group2.counter3.description = Total vcore-seconds taken by all reduce tasks
+counter.group2.counter4.names = OTHER_LOCAL_MAPS
+counter.group2.counter4.description = Total vcore-seconds taken by all reduce tasks
+counter.group2.counter5.names = DATA_LOCAL_MAPS
+counter.group2.counter5.description = Total vcore-seconds taken by all reduce tasks
+counter.group2.counter6.names = MILLIS_MAPS
+counter.group2.counter6.description = Total vcore-seconds taken by all reduce tasks
+counter.group2.counter7.names = MILLIS_REDUCES
+counter.group2.counter7.description = Total vcore-seconds taken by all reduce tasks
+counter.group2.counter8.names = TOTAL_LAUNCHED_MAPS
+counter.group2.counter8.description = Total vcore-seconds taken by all reduce tasks
+counter.group2.counter9.names = TOTAL_LAUNCHED_REDUCES
+counter.group2.counter9.description = Total vcore-seconds taken by all reduce tasks
+counter.group2.counter10.names = SLOTS_MILLIS_MAPS
+counter.group2.counter10.description = Total vcore-seconds taken by all reduce tasks
+counter.group2.counter11.names = SLOTS_MILLIS_REDUCES
+counter.group2.counter11.description = Total vcore-seconds taken by all reduce tasks
+counter.group2.counter12.names = RACK_LOCAL_MAPS
+counter.group2.counter12.description = Total vcore-seconds taken by all reduce tasks
+
+counter.group3.name = MapTaskAttemptCounter
+counter.group3.description = Reduce Task Attempt Counter Aggregation
+counter.group3.counter0.names = MAP_OUTPUT_MATERIALIZED_BYTES
+counter.group3.counter1.names = MAP_INPUT_RECORDS
+counter.group3.counter2.names = MERGED_MAP_OUTPUTS
+counter.group3.counter3.names = SPILLED_RECORDS
+counter.group3.counter4.names = MAP_OUTPUT_BYTES
+counter.group3.counter5.names = COMMITTED_HEAP_BYTES
+counter.group3.counter6.names = FAILED_SHUFFLE
+counter.group3.counter7.names = CPU_MILLISECONDS
+counter.group3.counter8.names = SPLIT_RAW_BYTES
+counter.group3.counter9.names = COMBINE_INPUT_RECORDS
+counter.group3.counter10.names = PHYSICAL_MEMORY_BYTES
+counter.group3.counter11.names = TASK_ATTEMPT_DURATION
+counter.group3.counter12.names = VIRTUAL_MEMORY_BYTES
+counter.group3.counter13.names = MAP_OUTPUT_RECORDS
+counter.group3.counter14.names = GC_TIME_MILLIS
+counter.group3.counter15.names = COMBINE_OUTPUT_RECORDS
+counter.group3.counter16.names = REDUCE_INPUT_GROUPS
+counter.group3.counter17.names = REDUCE_INPUT_RECORDS
+counter.group3.counter18.names = REDUCE_OUTPUT_RECORDS
+counter.group3.counter19.names = REDUCE_SHUFFLE_BYTES
+counter.group3.counter20.names = SHUFFLED_MAPS
+
+counter.group4.name = ReduceTaskAttemptCounter
+counter.group4.description = Reduce Task Attempt Counter Aggregation
+counter.group4.counter0.names = MAP_OUTPUT_MATERIALIZED_BYTES
+counter.group4.counter1.names = MAP_INPUT_RECORDS
+counter.group4.counter2.names = MERGED_MAP_OUTPUTS
+counter.group4.counter3.names = SPILLED_RECORDS
+counter.group4.counter4.names = MAP_OUTPUT_BYTES
+counter.group4.counter5.names = COMMITTED_HEAP_BYTES
+counter.group4.counter6.names = FAILED_SHUFFLE
+counter.group4.counter7.names = CPU_MILLISECONDS
+counter.group4.counter8.names = SPLIT_RAW_BYTES
+counter.group4.counter9.names = COMBINE_INPUT_RECORDS
+counter.group4.counter10.names = PHYSICAL_MEMORY_BYTES
+counter.group4.counter11.names = TASK_ATTEMPT_DURATION
+counter.group4.counter12.names = VIRTUAL_MEMORY_BYTES
+counter.group4.counter13.names = MAP_OUTPUT_RECORDS
+counter.group4.counter14.names = GC_TIME_MILLIS
+counter.group4.counter15.names = COMBINE_OUTPUT_RECORDS
+counter.group4.counter16.names = REDUCE_INPUT_GROUPS
+counter.group4.counter17.names = REDUCE_INPUT_RECORDS
+counter.group4.counter18.names = REDUCE_OUTPUT_RECORDS
+counter.group4.counter19.names = REDUCE_SHUFFLE_BYTES
+counter.group4.counter20.names = SHUFFLED_MAPS
+
+counter.group5.name = MapTaskAttemptFileSystemCounter
+counter.group5.description = Map Task Attempt File System Counter Aggregation
+counter.group5.counter0.names = FILE_READ_OPS
+counter.group5.counter1.names = FILE_WRITE_OPS
+counter.group5.counter2.names = FILE_BYTES_READ
+counter.group5.counter3.names = FILE_LARGE_READ_OPS
+counter.group5.counter4.names = HDFS_BYTES_READ
+counter.group5.counter5.names = FILE_BYTES_WRITTEN
+counter.group5.counter6.names = HDFS_LARGE_READ_OPS
+counter.group5.counter7.names = HDFS_BYTES_WRITTEN
+counter.group5.counter8.names = HDFS_READ_OPS
+
+counter.group6.name = ReduceTaskAttemptFileSystemCounter
+counter.group6.description = Reduce Task Attempt File System Counter Aggregation
+counter.group6.description = Map-Reduce Job Counter
+counter.group6.counter0.names = FILE_READ_OPS
+counter.group6.counter1.names = FILE_WRITE_OPS
+counter.group6.counter2.names = FILE_BYTES_READ
+counter.group6.counter3.names = FILE_LARGE_READ_OPS
+counter.group6.counter4.names = HDFS_BYTES_READ
+counter.group6.counter5.names = FILE_BYTES_WRITTEN
+counter.group6.counter6.names = HDFS_LARGE_READ_OPS
+counter.group6.counter7.names = HDFS_BYTES_WRITTEN
+counter.group6.counter8.names = HDFS_READ_OPS
\ No newline at end of file