You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2016/08/24 14:42:34 UTC
hadoop git commit: MAPREDUCE-6761. Regression when handling providers
- invalid configuration ServiceConfiguration causes Cluster initialization
failure. Contributed by Peter Vary (cherry picked from commit
6fa9bf4407a0b09be8ce4e52d2fde89adba34f1a)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a30f6a68f -> 5cdc4ba9f
MAPREDUCE-6761. Regression when handling providers - invalid configuration ServiceConfiguration causes Cluster initialization failure. Contributed by Peter Vary
(cherry picked from commit 6fa9bf4407a0b09be8ce4e52d2fde89adba34f1a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cdc4ba9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cdc4ba9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cdc4ba9
Branch: refs/heads/branch-2
Commit: 5cdc4ba9f4047a34e21fa91fa255ddea3c391e66
Parents: a30f6a6
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Aug 24 14:39:28 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Aug 24 14:42:06 2016 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/mapreduce/Cluster.java | 18 ++++-
.../apache/hadoop/mapreduce/TestCluster.java | 81 ++++++++++++++++++++
2 files changed, 95 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdc4ba9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
index dcce2aa..3a2f982 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
@@ -24,8 +24,10 @@ import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
+import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -53,7 +55,7 @@ public class Cluster {
@InterfaceStability.Evolving
public static enum JobTrackerStatus {INITIALIZING, RUNNING};
-
+
private ClientProtocolProvider clientProtocolProvider;
private ClientProtocol client;
private UserGroupInformation ugi;
@@ -64,7 +66,8 @@ public class Cluster {
private Path jobHistoryDir = null;
private static final Log LOG = LogFactory.getLog(Cluster.class);
- private static ServiceLoader<ClientProtocolProvider> frameworkLoader =
+ @VisibleForTesting
+ static Iterable<ClientProtocolProvider> frameworkLoader =
ServiceLoader.load(ClientProtocolProvider.class);
private volatile List<ClientProtocolProvider> providerList = null;
@@ -74,8 +77,15 @@ public class Cluster {
if (providerList == null) {
List<ClientProtocolProvider> localProviderList =
new ArrayList<ClientProtocolProvider>();
- for (ClientProtocolProvider provider : frameworkLoader) {
- localProviderList.add(provider);
+ try {
+ for (ClientProtocolProvider provider : frameworkLoader) {
+ localProviderList.add(provider);
+ }
+ } catch(ServiceConfigurationError e) {
+ LOG.info("Failed to instantiate ClientProtocolProvider, please "
+ + "check the /META-INF/services/org.apache."
+ + "hadoop.mapreduce.protocol.ClientProtocolProvider "
+ + "files on the classpath", e);
}
providerList = localProviderList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdc4ba9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestCluster.java
new file mode 100644
index 0000000..3aa0ff1
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestCluster.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce;
+
+import org.apache.hadoop.conf.Configuration;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.times;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
+import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Iterator;
+import java.util.ServiceConfigurationError;
+
+/**
+ * Testing the Cluster initialization.
+ */
+public class TestCluster {
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testProtocolProviderCreation() throws Exception {
+ Iterator iterator = mock(Iterator.class);
+ when(iterator.hasNext()).thenReturn(true, true, true, true);
+ when(iterator.next()).thenReturn(getClientProtocolProvider())
+ .thenThrow(new ServiceConfigurationError("Test error"))
+ .thenReturn(getClientProtocolProvider());
+
+ Iterable frameworkLoader = mock(Iterable.class);
+ when(frameworkLoader.iterator()).thenReturn(iterator);
+
+ Cluster.frameworkLoader = frameworkLoader;
+ Cluster testCluster = new Cluster(new Configuration());
+
+ // Check that we get the acceptable client, even after
+ // failure in instantiation.
+ assertNotNull("ClientProtocol is expected", testCluster.getClient());
+ // Check if we do not try to load the providers after a failure.
+ verify(iterator, times(2)).next();
+ }
+
+ public ClientProtocolProvider getClientProtocolProvider() {
+ return new ClientProtocolProvider() {
+ @Override
+ public ClientProtocol create(Configuration conf) throws IOException {
+ return mock(ClientProtocol.class);
+ }
+
+ @Override
+ public ClientProtocol create(InetSocketAddress addr, Configuration
+ conf) throws IOException {
+ return mock(ClientProtocol.class);
+ }
+
+ @Override
+ public void close(ClientProtocol clientProtocol) throws IOException {
+ }
+ };
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org