You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ge...@apache.org on 2020/01/08 12:37:21 UTC
[oozie] branch master updated: OOZIE-3578 MapReduce counters cannot
be used over 120 (dionusos via pbacsko, gezapeti)
This is an automated email from the ASF dual-hosted git repository.
gezapeti pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new f0af3a0 OOZIE-3578 MapReduce counters cannot be used over 120 (dionusos via pbacsko, gezapeti)
f0af3a0 is described below
commit f0af3a01c68e373cba3122ac7a005b987d84c729
Author: Gezapeti Cseh <ge...@apache.org>
AuthorDate: Wed Jan 8 13:36:40 2020 +0100
OOZIE-3578 MapReduce counters cannot be used over 120 (dionusos via pbacsko, gezapeti)
---
.../oozie/service/HadoopAccessorService.java | 12 ++++++++++
.../oozie/service/TestHadoopAccessorService.java | 13 +++++++++++
core/src/test/resources/test-mapred-site.xml | 27 ++++++++++++++++++++++
release-log.txt | 1 +
4 files changed, 53 insertions(+)
diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
index 396ad29..ceb05ef 100644
--- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
+++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest;
@@ -185,6 +186,17 @@ public class HadoopAccessorService implements Service {
}
setConfigForHadoopSecurityUtil(conf);
+ initializeMRLimits(conf);
+ }
+
+ /**
+ * This method initializes the MapReduce's Limits class so the user can define more than 120 counters
+ * in their map reduce action. For more details please see OOZIE-3578.
+ *
+ * @param conf the loaded hadoop configurations including mapred-site.xml
+ */
+ private void initializeMRLimits(Configuration conf) {
+ Limits.init(conf);
}
private void setConfigForHadoopSecurityUtil(Configuration conf) {
diff --git a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
index d3cb97a..582ad08 100644
--- a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
@@ -20,6 +20,7 @@ package org.apache.oozie.service;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -35,6 +36,7 @@ import org.apache.oozie.util.IOUtils;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
@@ -389,6 +391,17 @@ public class TestHadoopAccessorService extends XFsTestCase {
assertSame("The two configuration object shall be the same", base, result);
}
+ public void testIfMRLimitsIsInitialized() throws IOException, ServiceException {
+ InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("test-mapred-site.xml");
+ OutputStream os = new FileOutputStream(new File(getTestCaseConfDir() + "/hadoop-confx", "mapred-site.xml"));
+ IOUtils.copyStream(is, os);
+ HadoopAccessorService has = new HadoopAccessorService();
+ has.init(Services.get());
+ Assert.assertEquals("Limits class shall not use default value for number of counters.",
+ 500,
+ has.createConfiguration("jt").getInt(MRJobConfig.COUNTERS_MAX_KEY, 120));
+ }
+
public FileSystem createFileSystemWithCustomProperties(
final String uri, final String fsKey, final String commaSeparatedKeyValues) throws Exception {
final HadoopAccessorService has = new HadoopAccessorService();
diff --git a/core/src/test/resources/test-mapred-site.xml b/core/src/test/resources/test-mapred-site.xml
new file mode 100644
index 0000000..53472e0
--- /dev/null
+++ b/core/src/test/resources/test-mapred-site.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+ <property>
+ <name>mapreduce.job.counters.max</name>
+ <value>500</value>
+ </property>
+
+</configuration>
diff --git a/release-log.txt b/release-log.txt
index 46c1da8..5877766 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.3.0 release (trunk - unreleased)
+OOZIE-3578 MapReduce counters cannot be used over 120 (dionusos via pbacsko, gezapeti)
OOZIE-3573 Fix oozie-default.xml descriptions (arvaiadam via asalamon74)
OOZIE-3571 Wrong log string substitution in CredentialsProviderFactory class (arvaiadam via asalamon74)
OOZIE-3566 Wrong numbers provided by Oozie monitoring webservices.requests sampler (cas via asalamon74)