You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/04/04 19:56:25 UTC
[01/50] [abbrv] hadoop git commit: YARN-6050. AMs can't be scheduled
on racks or nodes (rkanter)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 38dddad74 -> adc651044
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
new file mode 100644
index 0000000..078b8fd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
@@ -0,0 +1,297 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class TestRMServerUtils {
+ @Test
+ public void testGetApplicableNodeCountForAMLocality() throws Exception {
+ List<NodeId> rack1Nodes = new ArrayList<>();
+ for (int i = 0; i < 29; i++) {
+ rack1Nodes.add(NodeId.newInstance("host" + i, 1234));
+ }
+ NodeId node1 = NodeId.newInstance("node1", 1234);
+ NodeId node2 = NodeId.newInstance("node2", 1234);
+ rack1Nodes.add(node2);
+
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, false);
+ ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+ Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100);
+ Mockito.when(scheduler.getNodeIds("/rack1")).thenReturn(rack1Nodes);
+ Mockito.when(scheduler.getNodeIds("node1"))
+ .thenReturn(Collections.singletonList(node1));
+ Mockito.when(scheduler.getNodeIds("node2"))
+ .thenReturn(Collections.singletonList(node2));
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+
+ ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY,
+ true, null);
+ List<ResourceRequest> reqs = new ArrayList<>();
+ reqs.add(anyReq);
+ Assert.assertEquals(100,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ ResourceRequest rackReq = createResourceRequest("/rack1", true, null);
+ reqs.add(rackReq);
+ Assert.assertEquals(30,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ anyReq.setRelaxLocality(false);
+ Assert.assertEquals(30,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(false);
+ Assert.assertEquals(100,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ ResourceRequest node1Req = createResourceRequest("node1", false, null);
+ reqs.add(node1Req);
+ Assert.assertEquals(100,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node1Req.setRelaxLocality(true);
+ Assert.assertEquals(1,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(true);
+ Assert.assertEquals(31,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ ResourceRequest node2Req = createResourceRequest("node2", false, null);
+ reqs.add(node2Req);
+ Assert.assertEquals(31,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node2Req.setRelaxLocality(true);
+ Assert.assertEquals(31,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(false);
+ Assert.assertEquals(2,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node1Req.setRelaxLocality(false);
+ Assert.assertEquals(1,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node2Req.setRelaxLocality(false);
+ Assert.assertEquals(100,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ }
+
+ @Test
+ public void testGetApplicableNodeCountForAMLabels() throws Exception {
+ Set<NodeId> noLabelNodes = new HashSet<>();
+ for (int i = 0; i < 80; i++) {
+ noLabelNodes.add(NodeId.newInstance("host" + i, 1234));
+ }
+ Set<NodeId> label1Nodes = new HashSet<>();
+ for (int i = 80; i < 90; i++) {
+ label1Nodes.add(NodeId.newInstance("host" + i, 1234));
+ }
+ label1Nodes.add(NodeId.newInstance("host101", 0));
+ label1Nodes.add(NodeId.newInstance("host102", 0));
+ Map<String, Set<NodeId>> label1NodesMap = new HashMap<>();
+ label1NodesMap.put("label1", label1Nodes);
+
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+ Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100);
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+ RMNodeLabelsManager labMan = Mockito.mock(RMNodeLabelsManager.class);
+ Mockito.when(labMan.getNodesWithoutALabel()).thenReturn(noLabelNodes);
+ Mockito.when(labMan.getLabelsToNodes(Collections.singleton("label1")))
+ .thenReturn(label1NodesMap);
+ Mockito.when(rmContext.getNodeLabelManager()).thenReturn(labMan);
+
+ ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY,
+ true, null);
+ List<ResourceRequest> reqs = new ArrayList<>();
+ reqs.add(anyReq);
+ Assert.assertEquals(80,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ anyReq.setNodeLabelExpression("label1");
+ Assert.assertEquals(10,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ }
+
+ @Test
+ public void testGetApplicableNodeCountForAMLocalityAndLabels()
+ throws Exception {
+ List<NodeId> rack1Nodes = new ArrayList<>();
+ for (int i = 0; i < 29; i++) {
+ rack1Nodes.add(NodeId.newInstance("host" + i, 1234));
+ }
+ NodeId node1 = NodeId.newInstance("node1", 1234);
+ NodeId node2 = NodeId.newInstance("node2", 1234);
+ rack1Nodes.add(node2);
+ Set<NodeId> noLabelNodes = new HashSet<>();
+ for (int i = 0; i < 19; i++) {
+ noLabelNodes.add(rack1Nodes.get(i));
+ }
+ noLabelNodes.add(node2);
+ for (int i = 29; i < 89; i++) {
+ noLabelNodes.add(NodeId.newInstance("host" + i, 1234));
+ }
+ Set<NodeId> label1Nodes = new HashSet<>();
+ label1Nodes.add(node1);
+ for (int i = 89; i < 93; i++) {
+ label1Nodes.add(NodeId.newInstance("host" + i, 1234));
+ }
+ for (int i = 19; i < 29; i++) {
+ label1Nodes.add(rack1Nodes.get(i));
+ }
+ label1Nodes.add(NodeId.newInstance("host101", 0));
+ label1Nodes.add(NodeId.newInstance("host102", 0));
+ Map<String, Set<NodeId>> label1NodesMap = new HashMap<>();
+ label1NodesMap.put("label1", label1Nodes);
+
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+ Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100);
+ Mockito.when(scheduler.getNodeIds("/rack1")).thenReturn(rack1Nodes);
+ Mockito.when(scheduler.getNodeIds("node1"))
+ .thenReturn(Collections.singletonList(node1));
+ Mockito.when(scheduler.getNodeIds("node2"))
+ .thenReturn(Collections.singletonList(node2));
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+ RMNodeLabelsManager labMan = Mockito.mock(RMNodeLabelsManager.class);
+ Mockito.when(labMan.getNodesWithoutALabel()).thenReturn(noLabelNodes);
+ Mockito.when(labMan.getLabelsToNodes(Collections.singleton("label1")))
+ .thenReturn(label1NodesMap);
+ Mockito.when(rmContext.getNodeLabelManager()).thenReturn(labMan);
+
+ ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY,
+ true, null);
+ List<ResourceRequest> reqs = new ArrayList<>();
+ reqs.add(anyReq);
+ Assert.assertEquals(80,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ ResourceRequest rackReq = createResourceRequest("/rack1", true, null);
+ reqs.add(rackReq);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ anyReq.setRelaxLocality(false);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(false);
+ Assert.assertEquals(80,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ ResourceRequest node1Req = createResourceRequest("node1", false, null);
+ reqs.add(node1Req);
+ Assert.assertEquals(80,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node1Req.setRelaxLocality(true);
+ Assert.assertEquals(0,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(true);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ ResourceRequest node2Req = createResourceRequest("node2", false, null);
+ reqs.add(node2Req);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node2Req.setRelaxLocality(true);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(false);
+ Assert.assertEquals(1,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node1Req.setRelaxLocality(false);
+ Assert.assertEquals(1,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node2Req.setRelaxLocality(false);
+ Assert.assertEquals(80,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ anyReq.setNodeLabelExpression("label1");
+ rackReq.setNodeLabelExpression("label1");
+ node1Req.setNodeLabelExpression("label1");
+ node2Req.setNodeLabelExpression("label1");
+ anyReq.setRelaxLocality(true);
+ reqs = new ArrayList<>();
+ reqs.add(anyReq);
+ Assert.assertEquals(15,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ rackReq.setRelaxLocality(true);
+ reqs.add(rackReq);
+ Assert.assertEquals(10,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ anyReq.setRelaxLocality(false);
+ Assert.assertEquals(10,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(false);
+ Assert.assertEquals(15,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ node1Req.setRelaxLocality(false);
+ reqs.add(node1Req);
+ Assert.assertEquals(15,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node1Req.setRelaxLocality(true);
+ Assert.assertEquals(1,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(true);
+ Assert.assertEquals(11,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ node2Req.setRelaxLocality(false);
+ reqs.add(node2Req);
+ Assert.assertEquals(11,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node2Req.setRelaxLocality(true);
+ Assert.assertEquals(11,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(false);
+ Assert.assertEquals(1,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node1Req.setRelaxLocality(false);
+ Assert.assertEquals(0,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ node2Req.setRelaxLocality(false);
+ Assert.assertEquals(15,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ }
+
+ private ResourceRequest createResourceRequest(String resource,
+ boolean relaxLocality, String nodeLabel) {
+ return ResourceRequest.newInstance(Priority.newInstance(0),
+ resource, Resource.newInstance(1, 1), 1, relaxLocality, nodeLabel);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 9be52c6..5246eb7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -57,7 +57,7 @@ import com.google.common.collect.Lists;
public abstract class MockAsm extends MockApps {
public static class ApplicationBase implements RMApp {
- ResourceRequest amReq;
+ List<ResourceRequest> amReqs;
@Override
public String getUser() {
throw new UnsupportedOperationException("Not supported yet.");
@@ -204,8 +204,8 @@ public abstract class MockAsm extends MockApps {
}
@Override
- public ResourceRequest getAMResourceRequest() {
- return this.amReq;
+ public List<ResourceRequest> getAMResourceRequests() {
+ return this.amReqs;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 55e93c1..7005bca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -526,7 +526,8 @@ public class TestSystemMetricsPublisher {
when(app.getAppNodeLabelExpression()).thenCallRealMethod();
ResourceRequest amReq = mock(ResourceRequest.class);
when(amReq.getNodeLabelExpression()).thenReturn("high-mem");
- when(app.getAMResourceRequest()).thenReturn(amReq);
+ when(app.getAMResourceRequests())
+ .thenReturn(Collections.singletonList(amReq));
when(app.getAmNodeLabelExpression()).thenCallRealMethod();
when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10));
when(app.getCallerContext())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 118b6bc..9290ff8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -62,14 +63,14 @@ public class MockRMApp implements RMApp {
StringBuilder diagnostics = new StringBuilder();
RMAppAttempt attempt;
int maxAppAttempts = 1;
- ResourceRequest amReq;
+ List<ResourceRequest> amReqs;
public MockRMApp(int newid, long time, RMAppState newState) {
finish = time;
id = MockApps.newAppID(newid);
state = newState;
- amReq = ResourceRequest.newInstance(Priority.UNDEFINED, "0.0.0.0",
- Resource.newInstance(0, 0), 1);
+ amReqs = Collections.singletonList(ResourceRequest.newInstance(
+ Priority.UNDEFINED, "0.0.0.0", Resource.newInstance(0, 0), 1));
}
public MockRMApp(int newid, long time, RMAppState newState, String userName) {
@@ -276,8 +277,8 @@ public class MockRMApp implements RMApp {
}
@Override
- public ResourceRequest getAMResourceRequest() {
- return this.amReq;
+ public List<ResourceRequest> getAMResourceRequests() {
+ return this.amReqs;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 4884851..5aa7af9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -30,8 +30,10 @@ import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -271,7 +273,8 @@ public class TestRMAppTransitions {
submissionContext.setAMContainerSpec(mock(ContainerLaunchContext.class));
RMApp application = new RMAppImpl(applicationId, rmContext, conf, name,
user, queue, submissionContext, scheduler, masterService,
- System.currentTimeMillis(), "YARN", null, mock(ResourceRequest.class));
+ System.currentTimeMillis(), "YARN", null,
+ new ArrayList<ResourceRequest>());
testAppStartState(applicationId, user, name, queue, application);
this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
@@ -1024,9 +1027,9 @@ public class TestRMAppTransitions {
submissionContext.getQueue(), submissionContext, scheduler, null,
appState.getSubmitTime(), submissionContext.getApplicationType(),
submissionContext.getApplicationTags(),
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1)));
Assert.assertEquals(RMAppState.NEW, application.getState());
RMAppEvent recoverEvent =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index ced5bd9..9a4b6dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -328,9 +328,9 @@ public class TestRMAppAttemptTransitions {
applicationAttempt =
new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler,
masterService, submissionContext, new Configuration(),
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1), application);
+ submissionContext.getResource(), 1)), application);
when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
when(application.getApplicationId()).thenReturn(applicationId);
@@ -1108,9 +1108,9 @@ public class TestRMAppAttemptTransitions {
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
spyRMContext, scheduler,masterService,
submissionContext, myConf,
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1), application);
+ submissionContext.getResource(), 1)), application);
//submit, schedule and allocate app attempt
myApplicationAttempt.handle(
@@ -1584,9 +1584,9 @@ public class TestRMAppAttemptTransitions {
applicationAttempt =
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext,
scheduler, masterService, submissionContext, new Configuration(),
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1), application);
+ submissionContext.getResource(), 1)), application);
when(submissionContext.getKeepContainersAcrossApplicationAttempts())
.thenReturn(true);
when(submissionContext.getMaxAppAttempts()).thenReturn(1);
@@ -1645,9 +1645,10 @@ public class TestRMAppAttemptTransitions {
applicationAttempt =
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
spyRMContext, scheduler, masterService, submissionContext,
- new Configuration(), ResourceRequest.newInstance(
- Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3,
- false, "label-expression"), application);
+ new Configuration(), Collections.singletonList(
+ ResourceRequest.newInstance(Priority.UNDEFINED, "host1",
+ Resource.newInstance(3333, 1), 3,
+ false, "label-expression")), application);
new RMAppAttemptImpl.ScheduleTransition().transition(
(RMAppAttemptImpl) applicationAttempt, null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index bb0a123..8aca235 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -613,7 +614,8 @@ public class TestApplicationLimits {
ResourceRequest amResourceRequest = mock(ResourceRequest.class);
Resource amResource = Resources.createResource(0, 0);
when(amResourceRequest.getCapability()).thenReturn(amResource);
- when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+ when(rmApp.getAMResourceRequests()).thenReturn(
+ Collections.singletonList(amResourceRequest));
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index b70a359..0aac2ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -639,7 +640,8 @@ public class TestApplicationLimitsByPartition {
ResourceRequest amResourceRequest = mock(ResourceRequest.class);
Resource amResource = Resources.createResource(0, 0);
when(amResourceRequest.getCapability()).thenReturn(amResource);
- when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+ when(rmApp.getAMResourceRequests()).thenReturn(
+ Collections.singletonList(amResourceRequest));
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 293bac2..ff0f7cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -3205,7 +3205,7 @@ public class TestCapacityScheduler {
RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName);
assertEquals("RMApp does not containes minimum allocation",
- minAllocResource, rmApp.getAMResourceRequest().getCapability());
+ minAllocResource, rmApp.getAMResourceRequests().get(0).getCapability());
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
LeafQueue queueA =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 3fbbae3..1162b9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -151,7 +151,8 @@ public class TestLeafQueue {
amResourceRequest = mock(ResourceRequest.class);
when(amResourceRequest.getCapability()).thenReturn(
Resources.createResource(0, 0));
- when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+ when(rmApp.getAMResourceRequests()).thenReturn(
+ Collections.singletonList(amResourceRequest));
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index baf7434..9bf79f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3204,6 +3204,84 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals(1, app.getLiveContainers().size());
}
+ @Test
+ public void testAMStrictLocalityRack() throws IOException {
+ testAMStrictLocality(false, false);
+ }
+
+ @Test
+ public void testAMStrictLocalityNode() throws IOException {
+ testAMStrictLocality(true, false);
+ }
+
+ @Test
+ public void testAMStrictLocalityRackInvalid() throws IOException {
+ testAMStrictLocality(false, true);
+ }
+
+ @Test
+ public void testAMStrictLocalityNodeInvalid() throws IOException {
+ testAMStrictLocality(true, true);
+ }
+
+ private void testAMStrictLocality(boolean node, boolean invalid)
+ throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ RMNode node2 = MockNodes.newNodeInfo(2, Resources.createResource(1024), 2,
+ "127.0.0.2");
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+
+ List<ResourceRequest> reqs = new ArrayList<>();
+ ResourceRequest nodeRequest = createResourceRequest(1024,
+ node2.getHostName(), 1, 1, true);
+ if (node && invalid) {
+ nodeRequest.setResourceName("invalid");
+ }
+ ResourceRequest rackRequest = createResourceRequest(1024,
+ node2.getRackName(), 1, 1, !node);
+ if (!node && invalid) {
+ rackRequest.setResourceName("invalid");
+ }
+ ResourceRequest anyRequest = createResourceRequest(1024,
+ ResourceRequest.ANY, 1, 1, false);
+ reqs.add(anyRequest);
+ reqs.add(rackRequest);
+ if (node) {
+ reqs.add(nodeRequest);
+ }
+
+ ApplicationAttemptId attId1 =
+ createSchedulingRequest("queue1", "user1", reqs);
+
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent node2UpdateEvent =
+ new NodeUpdateSchedulerEvent(node2);
+
+ FSAppAttempt app = scheduler.getSchedulerApp(attId1);
+
+ // node2 should get the container
+ scheduler.handle(node2UpdateEvent);
+ if (invalid) {
+ assertEquals(0, app.getLiveContainers().size());
+ assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers());
+ assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ } else {
+ assertEquals(1, app.getLiveContainers().size());
+ assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers());
+ assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ }
+ }
+
/**
* Strict locality requests shouldn't reserve resources on another node.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 30f25e9..fb9e8ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1213,8 +1213,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
assertEquals(app1.getApplicationId().toString(), appInfo.getAppId());
assertEquals(app1.getName(), appInfo.getName());
assertEquals(app1.createApplicationState(), appInfo.getState());
- assertEquals(app1.getAMResourceRequest().getCapability().getMemorySize(),
- appInfo.getAllocatedMB());
+ assertEquals(app1.getAMResourceRequests().get(0).getCapability()
+ .getMemorySize(), appInfo.getAllocatedMB());
rm.stop();
}
@@ -1427,7 +1427,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
expectedNumberOfElements++;
appNodeLabelExpression = info.getString("appNodeLabelExpression");
}
- if (app.getAMResourceRequest().getNodeLabelExpression() != null) {
+ if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
expectedNumberOfElements++;
amNodeLabelExpression = info.getString("amNodeLabelExpression");
}
@@ -1534,7 +1534,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
app.getApplicationSubmissionContext().getNodeLabelExpression(),
appNodeLabelExpression);
assertEquals("unmanagedApplication doesn't match",
- app.getAMResourceRequest().getNodeLabelExpression(),
+ app.getAMResourceRequests().get(0).getNodeLabelExpression(),
amNodeLabelExpression);
assertEquals("amRPCAddress",
AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
@@ -1561,7 +1561,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
String nodeLabelExpression, int numContainers, boolean relaxLocality,
int priority, String resourceName, long memory, long vCores,
String executionType, boolean enforceExecutionType) {
- ResourceRequest request = app.getAMResourceRequest();
+ ResourceRequest request = app.getAMResourceRequests().get(0);
assertEquals("nodeLabelExpression doesn't match",
request.getNodeLabelExpression(), nodeLabelExpression);
assertEquals("numContainers doesn't match", request.getNumContainers(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HADOOP-14247.
FileContextMainOperationsBaseTest should clean up test root path. Contributed
by Mingliang Liu
Posted by ae...@apache.org.
HADOOP-14247. FileContextMainOperationsBaseTest should clean up test root path. Contributed by Mingliang Liu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13c766b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13c766b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13c766b6
Branch: refs/heads/HDFS-7240
Commit: 13c766b62c48ae07849f7f1762ef6883239aef54
Parents: 15e3873
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Mar 27 16:37:39 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed Mar 29 11:13:39 2017 -0700
----------------------------------------------------------------------
.../fs/FileContextMainOperationsBaseTest.java | 21 +++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13c766b6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index 2b3ab2a..ece96f8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -36,6 +36,8 @@ import org.junit.Assert;
import static org.junit.Assert.*;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.hadoop.fs.FileContextTestHelper.*;
import static org.apache.hadoop.fs.CreateFlag.*;
@@ -60,6 +62,9 @@ import static org.apache.hadoop.fs.CreateFlag.*;
* </p>
*/
public abstract class FileContextMainOperationsBaseTest {
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(FileContextMainOperationsBaseTest.class);
private static String TEST_DIR_AAA2 = "test/hadoop2/aaa";
private static String TEST_DIR_AAA = "test/hadoop/aaa";
@@ -111,9 +116,19 @@ public abstract class FileContextMainOperationsBaseTest {
@After
public void tearDown() throws Exception {
if (fc != null) {
- boolean del = fc.delete(new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test")), true);
- assertTrue(del);
- fc.delete(localFsRootPath, true);
+ final Path testRoot = fileContextTestHelper.getAbsoluteTestRootPath(fc);
+ LOG.info("Deleting test root path {}", testRoot);
+ try {
+ fc.delete(testRoot, true);
+ } catch (Exception e) {
+ LOG.error("Error when deleting test root path " + testRoot, e);
+ }
+
+ try {
+ fc.delete(localFsRootPath, true);
+ } catch (Exception e) {
+ LOG.error("Error when deleting localFsRootPath " + localFsRootPath, e);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDFS-11551. Handle SlowDiskReport
from DataNode at the NameNode. Contributed by Hanisha Koneru.
Posted by ae...@apache.org.
HDFS-11551. Handle SlowDiskReport from DataNode at the NameNode. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28cdc5a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28cdc5a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28cdc5a8
Branch: refs/heads/HDFS-7240
Commit: 28cdc5a8dc37ade1f45bda3aede589ee8593945e
Parents: bf3fb58
Author: Hanisha Koneru <ha...@apache.org>
Authored: Thu Mar 30 22:41:26 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Mar 30 22:41:26 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/protocol/SlowDiskReports.java | 28 +-
.../server/blockmanagement/DatanodeManager.java | 34 +-
.../server/blockmanagement/SlowDiskTracker.java | 291 ++++++++++++
.../datanode/metrics/DataNodeDiskMetrics.java | 35 +-
.../blockmanagement/TestSlowDiskTracker.java | 448 +++++++++++++++++++
5 files changed, 812 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28cdc5a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java
index ef4d09e..8095c2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java
@@ -48,7 +48,7 @@ public final class SlowDiskReports {
private final Map<String, Map<DiskOp, Double>> slowDisks;
/**
- * An object representing a SlowPeerReports with no entries. Should
+ * An object representing a SlowDiskReports with no entries. Should
* be used instead of null or creating new objects when there are
* no slow peers to report.
*/
@@ -119,8 +119,28 @@ public final class SlowDiskReports {
* Lists the types of operations on which disk latencies are measured.
*/
public enum DiskOp {
- METADATA,
- READ,
- WRITE
+ METADATA("MetadataOp"),
+ READ("ReadIO"),
+ WRITE("WriteIO");
+
+ private final String value;
+
+ DiskOp(final String v) {
+ this.value = v;
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
+
+ public static DiskOp fromValue(final String value) {
+ for (DiskOp as : DiskOp.values()) {
+ if (as.value.equals(value)) {
+ return as;
+ }
+ }
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28cdc5a8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index e22b7af..18135a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList;
+import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
@@ -180,9 +181,15 @@ public class DatanodeManager {
* True if we should process latency metrics from downstream peers.
*/
private final boolean dataNodePeerStatsEnabled;
+ /**
+ * True if we should process latency metrics from individual DN disks.
+ */
+ private final boolean dataNodeDiskStatsEnabled;
@Nullable
private final SlowPeerTracker slowPeerTracker;
+ @Nullable
+ private final SlowDiskTracker slowDiskTracker;
/**
* The minimum time between resending caching directives to Datanodes,
@@ -208,9 +215,16 @@ public class DatanodeManager {
this.dataNodePeerStatsEnabled = conf.getBoolean(
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
+ this.dataNodeDiskStatsEnabled = Util.isDiskStatsEnabled(conf.getDouble(
+ DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
+ DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT));
+ final Timer timer = new Timer();
this.slowPeerTracker = dataNodePeerStatsEnabled ?
- new SlowPeerTracker(conf, new Timer()) : null;
+ new SlowPeerTracker(conf, timer) : null;
+
+ this.slowDiskTracker = dataNodeDiskStatsEnabled ?
+ new SlowDiskTracker(conf, timer) : null;
this.defaultXferPort = NetUtils.createSocketAddr(
conf.getTrimmed(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY,
@@ -1664,6 +1678,16 @@ public class DatanodeManager {
}
}
+ if (slowDiskTracker != null) {
+ if (!slowDisks.getSlowDisks().isEmpty()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("DataNode " + nodeReg + " reported slow disks: " +
+ slowDisks.getSlowDisks());
+ }
+ slowDiskTracker.addSlowDiskReport(nodeReg.getIpcAddr(false), slowDisks);
+ }
+ }
+
if (!cmds.isEmpty()) {
return cmds.toArray(new DatanodeCommand[cmds.size()]);
}
@@ -1875,5 +1899,13 @@ public class DatanodeManager {
public String getSlowPeersReport() {
return slowPeerTracker != null ? slowPeerTracker.getJson() : null;
}
+
+ /**
+ * Use only for testing.
+ */
+ @VisibleForTesting
+ public SlowDiskTracker getSlowDiskTracker() {
+ return slowDiskTracker;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28cdc5a8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
new file mode 100644
index 0000000..25920a2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Doubles;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
+import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports.DiskOp;
+import org.apache.hadoop.util.Timer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * This class aggregates information from {@link SlowDiskReports} received via
+ * heartbeats.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class SlowDiskTracker {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(SlowPeerTracker.class);
+
+ /**
+ * Time duration after which a report is considered stale. This is
+ * set to DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY * 3 i.e.
+ * maintained for at least two successive reports.
+ */
+ private long reportValidityMs;
+
+ /**
+ * Timer object for querying the current time. Separated out for
+ * unit testing.
+ */
+ private final Timer timer;
+
+ /**
+ * Number of disks to include in JSON report per operation. We will return
+ * disks with the highest latency.
+ */
+ private static final int MAX_DISKS_TO_REPORT = 5;
+ private static final String DATANODE_DISK_SEPARATOR = ":";
+ private final long reportGenerationIntervalMs;
+
+ private volatile long lastUpdateTime;
+ private AtomicBoolean isUpdateInProgress = new AtomicBoolean(false);
+
+ /**
+ * Information about disks that have been reported as being slow.
+ * It is map of (Slow Disk ID) -> (DiskLatency). The DiskLatency contains
+ * the disk ID, the latencies reported and the timestamp when the report
+ * was received.
+ */
+ private final Map<String, DiskLatency> diskIDLatencyMap;
+
+ /**
+ * Map of slow disk -> diskOperations it has been reported slow in.
+ */
+ private volatile ArrayList<DiskLatency> slowDisksReport =
+ Lists.newArrayList();
+ private volatile ArrayList<DiskLatency> oldSlowDisksCheck;
+
+ public SlowDiskTracker(Configuration conf, Timer timer) {
+ this.timer = timer;
+ this.lastUpdateTime = timer.monotonicNow();
+ this.diskIDLatencyMap = new ConcurrentHashMap<>();
+ this.reportGenerationIntervalMs = conf.getTimeDuration(
+ DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ this.reportValidityMs = reportGenerationIntervalMs * 3;
+ }
+
+ @VisibleForTesting
+ public static String getSlowDiskIDForReport(String datanodeID,
+ String slowDisk) {
+ return datanodeID + DATANODE_DISK_SEPARATOR + slowDisk;
+ }
+
+ public void addSlowDiskReport(String dataNodeID,
+ SlowDiskReports dnSlowDiskReport) {
+ Map<String, Map<DiskOp, Double>> slowDisks =
+ dnSlowDiskReport.getSlowDisks();
+
+ long now = timer.monotonicNow();
+
+ for (Map.Entry<String, Map<DiskOp, Double>> slowDiskEntry :
+ slowDisks.entrySet()) {
+
+ String diskID = getSlowDiskIDForReport(dataNodeID,
+ slowDiskEntry.getKey());
+
+ Map<DiskOp, Double> latencies = slowDiskEntry.getValue();
+
+ DiskLatency diskLatency = new DiskLatency(diskID, latencies, now);
+ diskIDLatencyMap.put(diskID, diskLatency);
+ }
+
+ checkAndUpdateReportIfNecessary();
+ }
+
+ private void checkAndUpdateReportIfNecessary() {
+ // Check if it is time for update
+ long now = timer.monotonicNow();
+ if (now - lastUpdateTime > reportGenerationIntervalMs) {
+ updateSlowDiskReportAsync(now);
+ }
+ }
+
+ @VisibleForTesting
+ public void updateSlowDiskReportAsync(long now) {
+ if (isUpdateInProgress.compareAndSet(false, true)) {
+ lastUpdateTime = now;
+ new Thread(new Runnable() {
+ @Override
+ public void run() {
+ slowDisksReport = getSlowDisks(diskIDLatencyMap,
+ MAX_DISKS_TO_REPORT, now);
+
+ cleanUpOldReports(now);
+
+ isUpdateInProgress.set(false);
+ }
+ }).start();
+ }
+ }
+
+ /**
+ * This structure is a thin wrapper over disk latencies.
+ */
+ public static class DiskLatency {
+ @JsonProperty("SlowDiskID")
+ final private String slowDiskID;
+ @JsonProperty("Latencies")
+ final private Map<DiskOp, Double> latencyMap;
+ @JsonIgnore
+ private long timestamp;
+
+ /**
+ * Constructor needed by Jackson for Object mapping.
+ */
+ public DiskLatency(
+ @JsonProperty("SlowDiskID") String slowDiskID,
+ @JsonProperty("Latencies") Map<DiskOp, Double> latencyMap) {
+ this.slowDiskID = slowDiskID;
+ this.latencyMap = latencyMap;
+ }
+
+ public DiskLatency(String slowDiskID, Map<DiskOp, Double> latencyMap,
+ long timestamp) {
+ this.slowDiskID = slowDiskID;
+ this.latencyMap = latencyMap;
+ this.timestamp = timestamp;
+ }
+
+ String getSlowDiskID() {
+ return this.slowDiskID;
+ }
+
+ double getMaxLatency() {
+ double maxLatency = 0;
+ for (double latency : latencyMap.values()) {
+ if (latency > maxLatency) {
+ maxLatency = latency;
+ }
+ }
+ return maxLatency;
+ }
+
+ Double getLatency(DiskOp op) {
+ return this.latencyMap.get(op);
+ }
+ }
+
+ /**
+ * Retrieve a list of stop low disks i.e disks with the highest max latencies.
+ * @param numDisks number of disks to return. This is to limit the size of
+ * the generated JSON.
+ */
+ private ArrayList<DiskLatency> getSlowDisks(
+ Map<String, DiskLatency> reports, int numDisks, long now) {
+ if (reports.isEmpty()) {
+ return new ArrayList(ImmutableList.of());
+ }
+
+ final PriorityQueue<DiskLatency> topNReports = new PriorityQueue<>(
+ reports.size(),
+ new Comparator<DiskLatency>() {
+ @Override
+ public int compare(DiskLatency o1, DiskLatency o2) {
+ return Doubles.compare(
+ o1.getMaxLatency(), o2.getMaxLatency());
+ }
+ });
+
+ ArrayList<DiskLatency> oldSlowDiskIDs = Lists.newArrayList();
+
+ for (Map.Entry<String, DiskLatency> entry : reports.entrySet()) {
+ DiskLatency diskLatency = entry.getValue();
+ if (now - diskLatency.timestamp < reportValidityMs) {
+ if (topNReports.size() < numDisks) {
+ topNReports.add(diskLatency);
+ } else if (topNReports.peek().getMaxLatency() <
+ diskLatency.getMaxLatency()) {
+ topNReports.poll();
+ topNReports.add(diskLatency);
+ }
+ } else {
+ oldSlowDiskIDs.add(diskLatency);
+ }
+ }
+
+ oldSlowDisksCheck = oldSlowDiskIDs;
+
+ return Lists.newArrayList(topNReports);
+ }
+
+ /**
+ * Retrieve all valid reports as a JSON string.
+ * @return serialized representation of valid reports. null if
+ * serialization failed.
+ */
+ public String getSlowDiskReportAsJsonString() {
+ ObjectMapper objectMapper = new ObjectMapper();
+ try {
+ return objectMapper.writeValueAsString(slowDisksReport);
+ } catch (JsonProcessingException e) {
+ // Failed to serialize. Don't log the exception call stack.
+ LOG.debug("Failed to serialize statistics" + e);
+ return null;
+ }
+ }
+
+ private void cleanUpOldReports(long now) {
+ if (oldSlowDisksCheck != null) {
+ for (DiskLatency oldDiskLatency : oldSlowDisksCheck) {
+ diskIDLatencyMap.remove(oldDiskLatency.getSlowDiskID(), oldDiskLatency);
+ }
+ }
+ // Replace oldSlowDiskIDsCheck with an empty ArrayList
+ oldSlowDisksCheck = null;
+ }
+
+ @VisibleForTesting
+ ArrayList<DiskLatency> getSlowDisksReport() {
+ return this.slowDisksReport;
+ }
+
+ @VisibleForTesting
+ long getReportValidityMs() {
+ return reportValidityMs;
+ }
+
+ @VisibleForTesting
+ void setReportValidityMs(long reportValidityMs) {
+ this.reportValidityMs = reportValidityMs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28cdc5a8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
index 2602b01..f2954e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.datanode.metrics;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
@@ -33,9 +32,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
-import java.util.Set;
/**
* This class detects and maintains DataNode disk outliers and their
@@ -122,43 +121,41 @@ public class DataNodeDiskMetrics {
private void detectAndUpdateDiskOutliers(Map<String, Double> metadataOpStats,
Map<String, Double> readIoStats, Map<String, Double> writeIoStats) {
- Set<String> diskOutliersSet = Sets.newHashSet();
+ Map<String, Map<DiskOp, Double>> diskStats = Maps.newHashMap();
// Get MetadataOp Outliers
Map<String, Double> metadataOpOutliers = slowDiskDetector
.getOutliers(metadataOpStats);
- if (!metadataOpOutliers.isEmpty()) {
- diskOutliersSet.addAll(metadataOpOutliers.keySet());
+ for (Map.Entry<String, Double> entry : metadataOpOutliers.entrySet()) {
+ addDiskStat(diskStats, entry.getKey(), DiskOp.METADATA, entry.getValue());
}
// Get ReadIo Outliers
Map<String, Double> readIoOutliers = slowDiskDetector
.getOutliers(readIoStats);
- if (!readIoOutliers.isEmpty()) {
- diskOutliersSet.addAll(readIoOutliers.keySet());
+ for (Map.Entry<String, Double> entry : readIoOutliers.entrySet()) {
+ addDiskStat(diskStats, entry.getKey(), DiskOp.READ, entry.getValue());
}
// Get WriteIo Outliers
Map<String, Double> writeIoOutliers = slowDiskDetector
.getOutliers(writeIoStats);
- if (!readIoOutliers.isEmpty()) {
- diskOutliersSet.addAll(writeIoOutliers.keySet());
- }
-
- Map<String, Map<DiskOp, Double>> diskStats =
- Maps.newHashMap();
- for (String disk : diskOutliersSet) {
- Map<DiskOp, Double> diskStat = Maps.newHashMap();
- diskStat.put(DiskOp.METADATA, metadataOpStats.get(disk));
- diskStat.put(DiskOp.READ, readIoStats.get(disk));
- diskStat.put(DiskOp.WRITE, writeIoStats.get(disk));
- diskStats.put(disk, diskStat);
+ for (Map.Entry<String, Double> entry : writeIoOutliers.entrySet()) {
+ addDiskStat(diskStats, entry.getKey(), DiskOp.WRITE, entry.getValue());
}
diskOutliersStats = diskStats;
LOG.debug("Updated disk outliers.");
}
+ private void addDiskStat(Map<String, Map<DiskOp, Double>> diskStats,
+ String disk, DiskOp diskOp, double latency) {
+ if (!diskStats.containsKey(disk)) {
+ diskStats.put(disk, new HashMap<>());
+ }
+ diskStats.get(disk).put(diskOp, latency);
+ }
+
public Map<String, Map<DiskOp, Double>> getDiskOutliersStats() {
return diskOutliersStats;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28cdc5a8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java
new file mode 100644
index 0000000..e96b96a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java
@@ -0,0 +1,448 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.hdfs.DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys
+ .DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
+import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports.DiskOp;
+import org.apache.hadoop.hdfs.server.blockmanagement.SlowDiskTracker
+ .DiskLatency;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.FakeTimer;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.Maps;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tests for {@link SlowDiskTracker}.
+ */
+public class TestSlowDiskTracker {
+ public static final Logger LOG = LoggerFactory.getLogger(
+ TestSlowDiskTracker.class);
+
+ /**
+ * Set a timeout for every test case.
+ */
+ @Rule
+ public Timeout testTimeout = new Timeout(300_000);
+
+ private static Configuration conf;
+ private SlowDiskTracker tracker;
+ private FakeTimer timer;
+ private long reportValidityMs;
+ private static final long OUTLIERS_REPORT_INTERVAL = 1000;
+
+ static {
+ conf = new HdfsConfiguration();
+ conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ conf.setDouble(DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
+ conf.setTimeDuration(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
+ OUTLIERS_REPORT_INTERVAL, TimeUnit.MILLISECONDS);
+ }
+ @Before
+ public void setup() {
+ timer = new FakeTimer();
+ tracker = new SlowDiskTracker(conf, timer);
+ reportValidityMs = tracker.getReportValidityMs();
+ }
+
+ @Test
+ public void testDataNodeHeartbeatSlowDiskReport() throws Exception {
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
+ .build();
+ try {
+ DataNode dn1 = cluster.getDataNodes().get(0);
+ DataNode dn2 = cluster.getDataNodes().get(1);
+ NameNode nn = cluster.getNameNode(0);
+
+ DatanodeManager datanodeManager = nn.getNamesystem().getBlockManager()
+ .getDatanodeManager();
+ SlowDiskTracker slowDiskTracker = datanodeManager.getSlowDiskTracker();
+ slowDiskTracker.setReportValidityMs(OUTLIERS_REPORT_INTERVAL * 100);
+
+ dn1.getDiskMetrics().addSlowDiskForTesting("disk1", ImmutableMap.of(
+ DiskOp.WRITE, 1.3));
+ dn1.getDiskMetrics().addSlowDiskForTesting("disk2", ImmutableMap.of(
+ DiskOp.READ, 1.6, DiskOp.WRITE, 1.1));
+ dn2.getDiskMetrics().addSlowDiskForTesting("disk1", ImmutableMap.of(
+ DiskOp.METADATA, 0.8));
+ dn2.getDiskMetrics().addSlowDiskForTesting("disk2", ImmutableMap.of(
+ DiskOp.WRITE, 1.3));
+
+ String dn1ID = dn1.getDatanodeId().getIpcAddr(false);
+ String dn2ID = dn2.getDatanodeId().getIpcAddr(false);
+
+ // Advance the timer and wait for NN to receive reports from DataNodes.
+ Thread.sleep(OUTLIERS_REPORT_INTERVAL);
+
+ // Wait for NN to receive reports from all DNs
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return (slowDiskTracker.getSlowDisksReport().size() == 4);
+ }
+ }, 1000, 100000);
+
+ Map<String, DiskLatency> slowDisksReport = getSlowDisksReportForTesting(
+ slowDiskTracker);
+
+ assertThat(slowDisksReport.size(), is(4));
+ assertTrue(Math.abs(slowDisksReport.get(dn1ID + ":disk1")
+ .getLatency(DiskOp.WRITE) - 1.3) < 0.0000001);
+ assertTrue(Math.abs(slowDisksReport.get(dn1ID + ":disk2")
+ .getLatency(DiskOp.READ) - 1.6) < 0.0000001);
+ assertTrue(Math.abs(slowDisksReport.get(dn1ID + ":disk2")
+ .getLatency(DiskOp.WRITE) - 1.1) < 0.0000001);
+ assertTrue(Math.abs(slowDisksReport.get(dn2ID + ":disk1")
+ .getLatency(DiskOp.METADATA) - 0.8) < 0.0000001);
+ assertTrue(Math.abs(slowDisksReport.get(dn2ID + ":disk2")
+ .getLatency(DiskOp.WRITE) - 1.3) < 0.0000001);
+
+ // Test the slow disk report JSON string
+ ArrayList<DiskLatency> jsonReport = getAndDeserializeJson(
+ slowDiskTracker.getSlowDiskReportAsJsonString());
+
+ assertThat(jsonReport.size(), is(4));
+ assertTrue(isDiskInReports(jsonReport, dn1ID, "disk1", DiskOp.WRITE, 1.3));
+ assertTrue(isDiskInReports(jsonReport, dn1ID, "disk2", DiskOp.READ, 1.6));
+ assertTrue(isDiskInReports(jsonReport, dn1ID, "disk2", DiskOp.WRITE, 1.1));
+ assertTrue(isDiskInReports(jsonReport, dn2ID, "disk1", DiskOp.METADATA,
+ 0.8));
+ assertTrue(isDiskInReports(jsonReport, dn2ID, "disk2", DiskOp.WRITE, 1.3));
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Edge case, there are no reports to retrieve.
+ */
+ @Test
+ public void testEmptyReports() {
+ tracker.updateSlowDiskReportAsync(timer.monotonicNow());
+ assertTrue(getSlowDisksReportForTesting(tracker).isEmpty());
+ }
+
+ @Test
+ public void testReportsAreRetrieved() throws Exception {
+ addSlowDiskForTesting("dn1", "disk1",
+ ImmutableMap.of(DiskOp.METADATA, 1.1, DiskOp.READ, 1.8));
+ addSlowDiskForTesting("dn1", "disk2",
+ ImmutableMap.of(DiskOp.READ, 1.3));
+ addSlowDiskForTesting("dn2", "disk2",
+ ImmutableMap.of(DiskOp.READ, 1.1));
+
+ tracker.updateSlowDiskReportAsync(timer.monotonicNow());
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return !tracker.getSlowDisksReport().isEmpty();
+ }
+ }, 500, 5000);
+
+ Map<String, DiskLatency> reports = getSlowDisksReportForTesting(tracker);
+
+ assertThat(reports.size(), is(3));
+ assertTrue(Math.abs(reports.get("dn1:disk1")
+ .getLatency(DiskOp.METADATA) - 1.1) < 0.0000001);
+ assertTrue(Math.abs(reports.get("dn1:disk1")
+ .getLatency(DiskOp.READ) - 1.8) < 0.0000001);
+ assertTrue(Math.abs(reports.get("dn1:disk2")
+ .getLatency(DiskOp.READ) - 1.3) < 0.0000001);
+ assertTrue(Math.abs(reports.get("dn2:disk2")
+ .getLatency(DiskOp.READ) - 1.1) < 0.0000001);
+ }
+
+ /**
+ * Test that when all reports are expired, we get back nothing.
+ */
+ @Test
+ public void testAllReportsAreExpired() throws Exception {
+ addSlowDiskForTesting("dn1", "disk1",
+ ImmutableMap.of(DiskOp.METADATA, 1.1, DiskOp.READ, 1.8));
+ addSlowDiskForTesting("dn1", "disk2",
+ ImmutableMap.of(DiskOp.READ, 1.3));
+ addSlowDiskForTesting("dn2", "disk2",
+ ImmutableMap.of(DiskOp.WRITE, 1.1));
+
+ // No reports should expire after 1ms.
+ timer.advance(1);
+ tracker.updateSlowDiskReportAsync(timer.monotonicNow());
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return !tracker.getSlowDisksReport().isEmpty();
+ }
+ }, 500, 5000);
+
+ Map<String, DiskLatency> reports = getSlowDisksReportForTesting(tracker);
+
+ assertThat(reports.size(), is(3));
+ assertTrue(Math.abs(reports.get("dn1:disk1")
+ .getLatency(DiskOp.METADATA) - 1.1) < 0.0000001);
+ assertTrue(Math.abs(reports.get("dn1:disk1")
+ .getLatency(DiskOp.READ) - 1.8) < 0.0000001);
+ assertTrue(Math.abs(reports.get("dn1:disk2")
+ .getLatency(DiskOp.READ) - 1.3) < 0.0000001);
+ assertTrue(Math.abs(reports.get("dn2:disk2")
+ .getLatency(DiskOp.WRITE) - 1.1) < 0.0000001);
+
+ // All reports should expire after REPORT_VALIDITY_MS.
+ timer.advance(reportValidityMs);
+ tracker.updateSlowDiskReportAsync(timer.monotonicNow());
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return tracker.getSlowDisksReport().isEmpty();
+ }
+ }, 500, 3000);
+
+ reports = getSlowDisksReportForTesting(tracker);
+
+ assertThat(reports.size(), is(0));
+ }
+
+ /**
+ * Test the case when a subset of reports has expired.
+ * Ensure that we only get back non-expired reports.
+ */
+ @Test
+ public void testSomeReportsAreExpired() throws Exception {
+ addSlowDiskForTesting("dn1", "disk1",
+ ImmutableMap.of(DiskOp.METADATA, 1.1, DiskOp.READ, 1.8));
+ addSlowDiskForTesting("dn1", "disk2",
+ ImmutableMap.of(DiskOp.READ, 1.3));
+ timer.advance(reportValidityMs);
+ addSlowDiskForTesting("dn2", "disk2",
+ ImmutableMap.of(DiskOp.WRITE, 1.1));
+
+ tracker.updateSlowDiskReportAsync(timer.monotonicNow());
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return !tracker.getSlowDisksReport().isEmpty();
+ }
+ }, 500, 5000);
+
+ Map<String, DiskLatency> reports = getSlowDisksReportForTesting(tracker);
+
+ assertThat(reports.size(), is(1));
+ assertTrue(Math.abs(reports.get("dn2:disk2")
+ .getLatency(DiskOp.WRITE) - 1.1) < 0.0000001);
+ }
+
+ /**
+ * Test the case when an expired report is replaced by a valid one.
+ */
+ @Test
+ public void testReplacement() throws Exception {
+ addSlowDiskForTesting("dn1", "disk1",
+ ImmutableMap.of(DiskOp.METADATA, 1.1, DiskOp.READ, 1.8));
+ timer.advance(reportValidityMs);
+ addSlowDiskForTesting("dn1", "disk1",
+ ImmutableMap.of(DiskOp.READ, 1.4));
+
+ tracker.updateSlowDiskReportAsync(timer.monotonicNow());
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return !tracker.getSlowDisksReport().isEmpty();
+ }
+ }, 500, 5000);
+
+ Map<String, DiskLatency> reports = getSlowDisksReportForTesting(tracker);
+
+ assertThat(reports.size(), is(1));
+ assertTrue(reports.get("dn1:disk1").getLatency(DiskOp.METADATA) == null);
+ assertTrue(Math.abs(reports.get("dn1:disk1")
+ .getLatency(DiskOp.READ) - 1.4) < 0.0000001);
+ }
+
+ @Test
+ public void testGetJson() throws Exception {
+ addSlowDiskForTesting("dn1", "disk1",
+ ImmutableMap.of(DiskOp.METADATA, 1.1, DiskOp.READ, 1.8));
+ addSlowDiskForTesting("dn1", "disk2",
+ ImmutableMap.of(DiskOp.READ, 1.3));
+ addSlowDiskForTesting("dn2", "disk2",
+ ImmutableMap.of(DiskOp.WRITE, 1.1));
+ addSlowDiskForTesting("dn3", "disk1",
+ ImmutableMap.of(DiskOp.WRITE, 1.1));
+
+ tracker.updateSlowDiskReportAsync(timer.monotonicNow());
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return tracker.getSlowDiskReportAsJsonString() != null;
+ }
+ }, 500, 5000);
+
+ ArrayList<DiskLatency> jsonReport = getAndDeserializeJson(
+ tracker.getSlowDiskReportAsJsonString());
+
+ // And ensure its contents are what we expect.
+ assertThat(jsonReport.size(), is(4));
+ assertTrue(isDiskInReports(jsonReport, "dn1", "disk1", DiskOp.METADATA,
+ 1.1));
+ assertTrue(isDiskInReports(jsonReport, "dn1", "disk1", DiskOp.READ, 1.8));
+ assertTrue(isDiskInReports(jsonReport, "dn1", "disk2", DiskOp.READ, 1.3));
+ assertTrue(isDiskInReports(jsonReport, "dn2", "disk2", DiskOp.WRITE, 1.1));
+ assertTrue(isDiskInReports(jsonReport, "dn3", "disk1", DiskOp.WRITE, 1.1));
+ }
+
+ @Test
+ public void testGetJsonSizeIsLimited() throws Exception {
+ addSlowDiskForTesting("dn1", "disk1",
+ ImmutableMap.of(DiskOp.READ, 1.1));
+ addSlowDiskForTesting("dn1", "disk2",
+ ImmutableMap.of(DiskOp.READ, 1.2));
+ addSlowDiskForTesting("dn1", "disk3",
+ ImmutableMap.of(DiskOp.READ, 1.3));
+ addSlowDiskForTesting("dn2", "disk1",
+ ImmutableMap.of(DiskOp.READ, 1.4));
+ addSlowDiskForTesting("dn2", "disk2",
+ ImmutableMap.of(DiskOp.READ, 1.5));
+ addSlowDiskForTesting("dn3", "disk1",
+ ImmutableMap.of(DiskOp.WRITE, 1.6));
+ addSlowDiskForTesting("dn3", "disk2",
+ ImmutableMap.of(DiskOp.READ, 1.7));
+ addSlowDiskForTesting("dn3", "disk3",
+ ImmutableMap.of(DiskOp.READ, 1.2));
+
+ tracker.updateSlowDiskReportAsync(timer.monotonicNow());
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return tracker.getSlowDiskReportAsJsonString() != null;
+ }
+ }, 500, 5000);
+
+ ArrayList<DiskLatency> jsonReport = getAndDeserializeJson(
+ tracker.getSlowDiskReportAsJsonString());
+
+ // Ensure that only the top 5 highest latencies are in the report.
+ assertThat(jsonReport.size(), is(5));
+ assertTrue(isDiskInReports(jsonReport, "dn3", "disk2", DiskOp.READ, 1.7));
+ assertTrue(isDiskInReports(jsonReport, "dn3", "disk1", DiskOp.WRITE, 1.6));
+ assertTrue(isDiskInReports(jsonReport, "dn2", "disk2", DiskOp.READ, 1.5));
+ assertTrue(isDiskInReports(jsonReport, "dn2", "disk1", DiskOp.READ, 1.4));
+ assertTrue(isDiskInReports(jsonReport, "dn1", "disk3", DiskOp.READ, 1.3));
+
+ // Remaining nodes should be in the list.
+ assertFalse(isDiskInReports(jsonReport, "dn1", "disk1", DiskOp.READ, 1.1));
+ assertFalse(isDiskInReports(jsonReport, "dn1", "disk2", DiskOp.READ, 1.2));
+ assertFalse(isDiskInReports(jsonReport, "dn3", "disk3", DiskOp.READ, 1.2));
+ }
+
+ @Test
+ public void testEmptyReport() throws Exception {
+ addSlowDiskForTesting("dn1", "disk1",
+ ImmutableMap.of(DiskOp.READ, 1.1));
+ timer.advance(reportValidityMs);
+
+ tracker.updateSlowDiskReportAsync(timer.monotonicNow());
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return tracker.getSlowDiskReportAsJsonString() != null;
+ }
+ }, 500, 5000);
+
+ ArrayList<DiskLatency> jsonReport = getAndDeserializeJson(
+ tracker.getSlowDiskReportAsJsonString());
+
+ assertTrue(jsonReport.isEmpty());
+ }
+
+ private boolean isDiskInReports(ArrayList<DiskLatency> reports,
+ String dataNodeID, String disk, DiskOp diskOp, double latency) {
+ String diskID = SlowDiskTracker.getSlowDiskIDForReport(dataNodeID, disk);
+ for (DiskLatency diskLatency : reports) {
+ if (diskLatency.getSlowDiskID().equals(diskID)) {
+ if (diskLatency.getLatency(diskOp) == null) {
+ return false;
+ }
+ if (Math.abs(diskLatency.getLatency(diskOp) - latency) < 0.0000001) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private ArrayList<DiskLatency> getAndDeserializeJson(
+ final String json) throws IOException {
+ return (new ObjectMapper()).readValue(json,
+ new TypeReference<ArrayList<DiskLatency>>() {});
+ }
+
+ private void addSlowDiskForTesting(String dnID, String disk,
+ Map<DiskOp, Double> latencies) {
+ Map<String, Map<DiskOp, Double>> slowDisk = Maps.newHashMap();
+ slowDisk.put(disk, latencies);
+ SlowDiskReports slowDiskReport = SlowDiskReports.create(slowDisk);
+ tracker.addSlowDiskReport(dnID, slowDiskReport);
+ }
+
+ Map<String, DiskLatency> getSlowDisksReportForTesting(
+ SlowDiskTracker slowDiskTracker) {
+ Map<String, DiskLatency> slowDisksMap = Maps.newHashMap();
+ for (DiskLatency diskLatency : slowDiskTracker.getSlowDisksReport()) {
+ slowDisksMap.put(diskLatency.getSlowDiskID(), diskLatency);
+ }
+ return slowDisksMap;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDFS-11515. -du throws
ConcurrentModificationException. Contributed by Istvan Fajth, Wei-Chiu Chuang.
Posted by ae...@apache.org.
HDFS-11515. -du throws ConcurrentModificationException. Contributed by Istvan Fajth, Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bc7aff7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bc7aff7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bc7aff7c
Branch: refs/heads/HDFS-7240
Commit: bc7aff7cec07bbc3fed63a00c8f1584c34670998
Parents: 845529b
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Apr 3 07:32:27 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Apr 3 07:35:09 2017 -0700
----------------------------------------------------------------------
.../snapshot/DirectoryWithSnapshotFeature.java | 5 ++
.../snapshot/TestRenameWithSnapshots.java | 6 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 75 ++++++++++++++++++++
3 files changed, 84 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc7aff7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
index 9addbfa..9840679 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
@@ -633,6 +633,11 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
for(DirectoryDiff d : diffs) {
for(INode deletedNode : d.getChildrenDiff().getList(ListType.DELETED)) {
context.reportDeletedSnapshottedNode(deletedNode);
+ if (deletedNode.isDirectory()){
+ DirectoryWithSnapshotFeature sf =
+ deletedNode.asDirectory().getDirectoryWithSnapshotFeature();
+ sf.computeContentSummary4Snapshot(context);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc7aff7c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
index d1b3aa6..d06c384 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
@@ -26,6 +26,7 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
+import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
import java.io.File;
import java.io.IOException;
@@ -2429,7 +2430,7 @@ public class TestRenameWithSnapshots {
*/
@Test (timeout=300000)
public void testDu() throws Exception {
- File tempFile = File.createTempFile("testDu-", ".tmp");
+ File tempFile = File.createTempFile("testDu-", ".tmp", getTestDir());
tempFile.deleteOnExit();
final FileSystem localfs = FileSystem.getLocal(conf);
@@ -2539,7 +2540,8 @@ public class TestRenameWithSnapshots {
*/
@Test (timeout=300000)
public void testDuMultipleDirs() throws Exception {
- File tempFile = File.createTempFile("testDuMultipleDirs-", "" + ".tmp");
+ File tempFile = File.createTempFile("testDuMultipleDirs-", ".tmp",
+ getTestDir());
tempFile.deleteOnExit();
final FileSystem localfs = FileSystem.getLocal(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc7aff7c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
index ca53788..7926e44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
@@ -27,6 +27,7 @@ import java.io.PrintStream;
import java.security.PrivilegedAction;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
@@ -1232,4 +1233,78 @@ public class TestSnapshotDeletion {
// make sure bar has been cleaned from inodeMap
Assert.assertNull(fsdir.getInode(fileId));
}
+
+ /**
+ * Test for HDFS-11515.
+ * In a scenario where a directory with subdirectories is removed from the
+ * file system after taking a snapshot on one of its ancestors, du command
+ * fails with a ConcurrentModificationException until a new snapshot is taken,
+ * or the old snapshots are removed.
+ * This test is testing this scenario with checks on the space consumed
+ * calculation.
+ *
+ * @throws Exception
+ */
+ @Test(timeout = 180000)
+ public void testDuWithRmdirInSnapshots() throws Exception {
+ final Path parent = new Path("/testDuWithRmdirInSnapshots");
+ final Path snapshotDir = new Path(parent, "snapshotDir");
+ final Path dir1 = new Path(snapshotDir, "d1"); //snapshotDir/d1
+ final Path dir2 = new Path(snapshotDir, "d2"); //snapshotDir/d2
+ final Path dir4 = new Path(dir2, "d4"); //snapshotDir/d2/d4
+ final Path dir3 = new Path(snapshotDir, "d3"); //snapshotDir/d3
+ final Path dir5 = new Path(dir3, "d5"); //snapshotDir/d3/d5
+ final Path aFileOutsideSnapshots = new Path(parent, "aFile");
+ final Path aFileInsideSnapshots = new Path(dir5, "aFile");
+
+ final String snapshotName = "s1";
+ final String snapshotName2 = "s2";
+
+ final long spaceConsumed = BLOCKSIZE * REPLICATION;
+ final long spaceConsumed2 = 2 * spaceConsumed;
+ ContentSummary summary = null;
+
+ DFSTestUtil.createFile(hdfs, aFileOutsideSnapshots,
+ BLOCKSIZE, REPLICATION, 0);
+ summary = hdfs.getContentSummary(parent);
+ assertEquals("Du is wrong even with one file without further ado.",
+ spaceConsumed, summary.getSpaceConsumed());
+
+ hdfs.mkdirs(snapshotDir);
+ hdfs.allowSnapshot(snapshotDir);
+ hdfs.mkdirs(dir1);
+
+ hdfs.createSnapshot(snapshotDir, snapshotName);
+
+ hdfs.mkdirs(dir4);
+ hdfs.mkdirs(dir5);
+ DFSTestUtil.createFile(hdfs, aFileInsideSnapshots,
+ BLOCKSIZE, REPLICATION, 0);
+ summary = hdfs.getContentSummary(parent);
+ assertEquals("Du is wrong with 2 files added to the file system.",
+ spaceConsumed2, summary.getSpaceConsumed());
+
+ hdfs.createSnapshot(snapshotDir, snapshotName2);
+
+ hdfs.delete(dir2, true);
+ hdfs.delete(dir3, true);
+
+ summary = hdfs.getContentSummary(parent);
+ assertEquals("Snapshot file count is not matching expected value.",
+ 1, summary.getSnapshotFileCount());
+ assertEquals("Snapshot directory count is not matching expected value.",
+ 4, summary.getSnapshotDirectoryCount());
+ assertEquals("Consumed space does not matching expected value.",
+ spaceConsumed, summary.getSnapshotSpaceConsumed());
+ assertEquals("Snapshot length is not matching expected value.",
+ BLOCKSIZE, summary.getSnapshotLength());
+ assertEquals("File count is not matching expected value.",
+ 2, summary.getFileCount());
+ assertEquals("Directory count is not matching expected value.",
+ 7, summary.getDirectoryCount());
+ assertEquals("Consumed space is not matching expected value.",
+ spaceConsumed2, summary.getSpaceConsumed());
+ assertEquals("Length is not matching expected value.",
+ 2 * BLOCKSIZE, summary.getLength());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HADOOP-14268. Fix markdown
itemization in hadoop-aws documents. Contributed by Akira Ajisaka
Posted by ae...@apache.org.
HADOOP-14268. Fix markdown itemization in hadoop-aws documents. Contributed by Akira Ajisaka
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5faa949b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5faa949b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5faa949b
Branch: refs/heads/HDFS-7240
Commit: 5faa949b782be48ef400d2eb1695f420455de764
Parents: bbd6847
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Apr 3 11:07:14 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Apr 3 11:07:14 2017 -0700
----------------------------------------------------------------------
.../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 2 ++
.../hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md | 1 +
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5faa949b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 82c3588..18c0ceb 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -41,6 +41,7 @@ The specifics of using these filesystems are documented in this section.
See also:
+
* [Testing](testing.html)
* [Troubleshooting S3a](troubleshooting_s3a.html)
@@ -99,6 +100,7 @@ access to the data. Anyone with the credentials can not only read your datasets
\u2014they can delete them.
Do not inadvertently share these credentials through means such as
+
1. Checking in to SCM any configuration files containing the secrets.
1. Logging them to a console, as they invariably end up being seen.
1. Defining filesystem URIs with the credentials in the URL, such as
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5faa949b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index 79551a3..39ca8f4 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -643,6 +643,7 @@ located.
New tests are always welcome. Bear in mind that we need to keep costs
and test time down, which is done by
+
* Not duplicating tests.
* Being efficient in your use of Hadoop API calls.
* Isolating large/slow tests into the "scale" test group.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDFS-11592. Closing a file has a
wasteful preconditions in NameNode. Contributed by Eric Badger
Posted by ae...@apache.org.
HDFS-11592. Closing a file has a wasteful preconditions in NameNode. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5f0622a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5f0622a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5f0622a
Branch: refs/heads/HDFS-7240
Commit: e5f0622a6f40706d360d45200c8f259c79046438
Parents: b58777a
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Mar 30 15:44:06 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Mar 30 15:44:21 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/namenode/INodeFile.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5f0622a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 2b0e0ad..3da6aa7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -328,9 +328,11 @@ public class INodeFile extends INodeWithAdditionalFields
for (int i = 0; i < blocks.length; i++) {
final String err = checkBlockComplete(blocks, i, numCommittedAllowed,
minReplication);
- Preconditions.checkState(err == null,
- "Unexpected block state: %s, file=%s (%s), blocks=%s (i=%s)",
- err, this, getClass().getSimpleName(), Arrays.asList(blocks), i);
+ if(err != null) {
+ throw new IllegalStateException(String.format("Unexpected block state: " +
+ "%s, file=%s (%s), blocks=%s (i=%s)", err, this,
+ getClass().getSimpleName(), Arrays.asList(blocks), i));
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HADOOP-14271. Correct spelling of
'occurred' and variants. Contributed by Yeliang Cang
Posted by ae...@apache.org.
HADOOP-14271. Correct spelling of 'occurred' and variants. Contributed by Yeliang Cang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6eba7923
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6eba7923
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6eba7923
Branch: refs/heads/HDFS-7240
Commit: 6eba79232f36b36e0196163adc8fe4219a6b6bf9
Parents: 5faa949
Author: Chris Douglas <cd...@apache.org>
Authored: Mon Apr 3 20:13:14 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Mon Apr 3 20:13:14 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/util/Progressable.java | 2 +-
.../apache/hadoop/util/UTF8ByteArrayUtils.java | 4 +--
.../src/main/native/gtest/gtest-all.cc | 2 +-
.../src/main/native/gtest/include/gtest/gtest.h | 2 +-
.../hadoop/test/MultithreadedTestUtil.java | 2 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 2 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 2 +-
.../hadoop/fs/http/server/FSOperations.java | 34 ++++++++++----------
.../datanode/fsdataset/impl/BlockPoolSlice.java | 2 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 2 +-
.../mapreduce/lib/jobcontrol/JobControl.java | 2 +-
.../hadoop/fs/azure/BlockBlobAppendStream.java | 2 +-
.../hadoop/streaming/StreamKeyValUtil.java | 8 ++---
.../distributedshell/TestDistributedShell.java | 30 ++++++++---------
.../launcher/TestContainerLaunch.java | 12 +++----
.../scheduler/capacity/TestParentQueue.java | 18 +++++------
16 files changed, 63 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progressable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progressable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progressable.java
index 495ca82..201ee5c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progressable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progressable.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* to explicitly report progress to the Hadoop framework. This is especially
* important for operations which take significant amount of time since,
* in-lieu of the reported progress, the framework has to assume that an error
- * has occured and time-out the operation.</p>
+ * has occurred and time-out the operation.</p>
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/UTF8ByteArrayUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/UTF8ByteArrayUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/UTF8ByteArrayUtils.java
index 2a804c6..069494f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/UTF8ByteArrayUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/UTF8ByteArrayUtils.java
@@ -30,7 +30,7 @@ public class UTF8ByteArrayUtils {
* @param start starting offset
* @param end ending position
* @param b the byte to find
- * @return position that first byte occures otherwise -1
+ * @return position that first byte occurs, otherwise -1
*/
public static int findByte(byte [] utf, int start, int end, byte b) {
for(int i=start; i<end; i++) {
@@ -47,7 +47,7 @@ public class UTF8ByteArrayUtils {
* @param start starting offset
* @param end ending position
* @param b the bytes to find
- * @return position that first byte occures otherwise -1
+ * @return position that first byte occurs, otherwise -1
*/
public static int findBytes(byte [] utf, int start, int end, byte[] b) {
int matchEnd = end - b.length;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-common-project/hadoop-common/src/main/native/gtest/gtest-all.cc
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/gtest/gtest-all.cc b/hadoop-common-project/hadoop-common/src/main/native/gtest/gtest-all.cc
index 4f8c08a..fb6ddbc 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/gtest/gtest-all.cc
+++ b/hadoop-common-project/hadoop-common/src/main/native/gtest/gtest-all.cc
@@ -8298,7 +8298,7 @@ FilePath FilePath::RemoveExtension(const char* extension) const {
return *this;
}
-// Returns a pointer to the last occurence of a valid path separator in
+// Returns a pointer to the last occurrence of a valid path separator in
// the FilePath. On Windows, for example, both '/' and '\' are valid path
// separators. Returns NULL if no path separator was found.
const char* FilePath::FindLastPathSeparator() const {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-common-project/hadoop-common/src/main/native/gtest/include/gtest/gtest.h
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/gtest/include/gtest/gtest.h b/hadoop-common-project/hadoop-common/src/main/native/gtest/include/gtest/gtest.h
index c04205d..3d795bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/gtest/include/gtest/gtest.h
+++ b/hadoop-common-project/hadoop-common/src/main/native/gtest/include/gtest/gtest.h
@@ -4457,7 +4457,7 @@ class GTEST_API_ FilePath {
void Normalize();
- // Returns a pointer to the last occurence of a valid path separator in
+ // Returns a pointer to the last occurrence of a valid path separator in
// the FilePath. On Windows, for example, both '/' and '\' are valid path
// separators. Returns NULL if no path separator was found.
const char* FindLastPathSeparator() const;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MultithreadedTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MultithreadedTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MultithreadedTestUtil.java
index e0bc136..b51329f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MultithreadedTestUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MultithreadedTestUtil.java
@@ -225,7 +225,7 @@ public abstract class MultithreadedTestUtil {
/**
* User method for any code to test repeating behavior of (as threads).
- * @throws Exception throw an exception if a failure has occured.
+ * @throws Exception throw an exception if a failure has occurred.
*/
public abstract void doAnAction() throws Exception;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 39d0eed..d388d00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -777,7 +777,7 @@ public class DFSInputStream extends FSInputStream
}
} finally {
// Check if need to report block replicas corruption either read
- // was successful or ChecksumException occured.
+ // was successful or ChecksumException occurred.
reportCheckSumFailure(corruptedBlocks,
currentLocatedBlock.getLocations().length, false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 07e6609..75ad022 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -394,7 +394,7 @@ public class DFSStripedInputStream extends DFSInputStream {
return result;
} finally {
// Check if need to report block replicas corruption either read
- // was successful or ChecksumException occured.
+ // was successful or ChecksumException occurred.
reportCheckSumFailure(corruptedBlocks,
currentLocatedBlock.getLocations().length, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index 5ff42a4..3373582 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -333,7 +333,7 @@ public class FSOperations {
*
* @return void.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public Void execute(FileSystem fs) throws IOException {
@@ -377,7 +377,7 @@ public class FSOperations {
*
* @return void.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public Void execute(FileSystem fs) throws IOException {
@@ -418,7 +418,7 @@ public class FSOperations {
* wait for it to complete before proceeding with further file
* updates.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public JSONObject execute(FileSystem fs) throws IOException {
@@ -452,7 +452,7 @@ public class FSOperations {
*
* @return a Map object (JSON friendly) with the content-summary.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public Map execute(FileSystem fs) throws IOException {
@@ -501,7 +501,7 @@ public class FSOperations {
*
* @return The URI of the created file.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public Void execute(FileSystem fs) throws IOException {
@@ -549,7 +549,7 @@ public class FSOperations {
* @return <code>true</code> if the delete operation was successful,
* <code>false</code> otherwise.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public JSONObject execute(FileSystem fs) throws IOException {
@@ -583,7 +583,7 @@ public class FSOperations {
*
* @return a Map object (JSON friendly) with the file checksum.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public Map execute(FileSystem fs) throws IOException {
@@ -640,7 +640,7 @@ public class FSOperations {
*
* @return a JSON object with the user home directory.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
@SuppressWarnings("unchecked")
@@ -765,7 +765,7 @@ public class FSOperations {
* @return <code>true</code> if the mkdirs operation was successful,
* <code>false</code> otherwise.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public JSONObject execute(FileSystem fs) throws IOException {
@@ -799,7 +799,7 @@ public class FSOperations {
*
* @return The inputstream of the file.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public InputStream execute(FileSystem fs) throws IOException {
@@ -837,7 +837,7 @@ public class FSOperations {
* @return <code>true</code> if the rename operation was successful,
* <code>false</code> otherwise.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public JSONObject execute(FileSystem fs) throws IOException {
@@ -876,7 +876,7 @@ public class FSOperations {
*
* @return void.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public Void execute(FileSystem fs) throws IOException {
@@ -913,7 +913,7 @@ public class FSOperations {
*
* @return void.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public Void execute(FileSystem fs) throws IOException {
@@ -1186,7 +1186,7 @@ public class FSOperations {
* @return <code>true</code> if the replication value was set,
* <code>false</code> otherwise.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
@SuppressWarnings("unchecked")
@@ -1228,7 +1228,7 @@ public class FSOperations {
*
* @return void.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public Void execute(FileSystem fs) throws IOException {
@@ -1314,7 +1314,7 @@ public class FSOperations {
*
* @return Map a map object (JSON friendly) with the xattr names.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public Map execute(FileSystem fs) throws IOException {
@@ -1353,7 +1353,7 @@ public class FSOperations {
*
* @return Map a map object (JSON friendly) with the xattrs.
*
- * @throws IOException thrown if an IO error occured.
+ * @throws IOException thrown if an IO error occurred.
*/
@Override
public Map execute(FileSystem fs) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index c8df300..c17ef36 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -822,7 +822,7 @@ class BlockPoolSlice {
} catch (Exception e) {
// Any exception we need to revert back to read from disk
// Log the error and return false
- LOG.info("Exception occured while reading the replicas cache file: "
+ LOG.info("Exception occurred while reading the replicas cache file: "
+ replicaFile.getPath(), e );
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index f6e6a59..b948fb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -1323,7 +1323,7 @@ public class FsVolumeImpl implements FsVolumeSpi {
fileNames = fileIoProvider.listDirectory(
this, dir, BlockDirFilter.INSTANCE);
} catch (IOException ioe) {
- LOG.warn("Exception occured while compiling report: ", ioe);
+ LOG.warn("Exception occurred while compiling report: ", ioe);
// Volume error check moved to FileIoProvider.
// Ignore this directory and proceed.
return report;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java
index b0b7a3c..e5399b5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java
@@ -276,7 +276,7 @@ public class JobControl implements Runnable {
}
synchronized private void failAllJobs(Throwable t) {
- String message = "Unexpected System Error Occured: "+
+ String message = "Unexpected System Error Occurred: "+
StringUtils.stringifyException(t);
Iterator<ControlledJob> it = jobsInProgress.iterator();
while(it.hasNext()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
index a7e286c..afb9379 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
@@ -346,7 +346,7 @@ public class BlockBlobAppendStream extends OutputStream {
try {
if (!ioThreadPool.awaitTermination(10, TimeUnit.MINUTES)) {
- LOG.error("Time out occured while waiting for IO request to finish in append"
+ LOG.error("Time out occurred while waiting for IO request to finish in append"
+ " for blob : {}", key);
NativeAzureFileSystemHelper.logAllLiveStackTraces();
throw new IOException("Timed out waiting for IO requests to finish");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamKeyValUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamKeyValUtil.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamKeyValUtil.java
index 75e05dc..fba45b1 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamKeyValUtil.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamKeyValUtil.java
@@ -26,11 +26,11 @@ import org.apache.hadoop.util.LineReader;
public class StreamKeyValUtil {
/**
- * Find the first occured tab in a UTF-8 encoded string
+ * Find the first occurred tab in a UTF-8 encoded string
* @param utf a byte array containing a UTF-8 encoded string
* @param start starting offset
* @param length no. of bytes
- * @return position that first tab occures otherwise -1
+ * @return position that first tab occurres otherwise -1
*/
public static int findTab(byte [] utf, int start, int length) {
for(int i=start; i<(start+length); i++) {
@@ -41,9 +41,9 @@ public class StreamKeyValUtil {
return -1;
}
/**
- * Find the first occured tab in a UTF-8 encoded string
+ * Find the first occurred tab in a UTF-8 encoded string
* @param utf a byte array containing a UTF-8 encoded string
- * @return position that first tab occures otherwise -1
+ * @return position that first tab occurres otherwise -1
*/
public static int findTab(byte [] utf) {
return org.apache.hadoop.util.UTF8ByteArrayUtils.findNthByte(utf, 0,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 300ea67..ef21c87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -544,17 +544,17 @@ public class TestDistributedShell {
Assert.assertEquals(
"Container created event needs to be published atleast once",
1,
- getNumOfStringOccurences(containerEntityFile,
+ getNumOfStringOccurrences(containerEntityFile,
ContainerMetricsConstants.CREATED_EVENT_TYPE));
// to avoid race condition of testcase, atleast check 4 times with sleep
// of 500ms
- long numOfContainerFinishedOccurences = 0;
+ long numOfContainerFinishedOccurrences = 0;
for (int i = 0; i < 4; i++) {
- numOfContainerFinishedOccurences =
- getNumOfStringOccurences(containerEntityFile,
+ numOfContainerFinishedOccurrences =
+ getNumOfStringOccurrences(containerEntityFile,
ContainerMetricsConstants.FINISHED_EVENT_TYPE);
- if (numOfContainerFinishedOccurences > 0) {
+ if (numOfContainerFinishedOccurrences > 0) {
break;
} else {
Thread.sleep(500L);
@@ -563,7 +563,7 @@ public class TestDistributedShell {
Assert.assertEquals(
"Container finished event needs to be published atleast once",
1,
- numOfContainerFinishedOccurences);
+ numOfContainerFinishedOccurrences);
// Verify RM posting Application life cycle Events are getting published
String appMetricsTimestampFileName =
@@ -576,17 +576,17 @@ public class TestDistributedShell {
Assert.assertEquals(
"Application created event should be published atleast once",
1,
- getNumOfStringOccurences(appEntityFile,
+ getNumOfStringOccurrences(appEntityFile,
ApplicationMetricsConstants.CREATED_EVENT_TYPE));
// to avoid race condition of testcase, atleast check 4 times with sleep
// of 500ms
- long numOfStringOccurences = 0;
+ long numOfStringOccurrences = 0;
for (int i = 0; i < 4; i++) {
- numOfStringOccurences =
- getNumOfStringOccurences(appEntityFile,
+ numOfStringOccurrences =
+ getNumOfStringOccurrences(appEntityFile,
ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- if (numOfStringOccurences > 0) {
+ if (numOfStringOccurrences > 0) {
break;
} else {
Thread.sleep(500L);
@@ -595,7 +595,7 @@ public class TestDistributedShell {
Assert.assertEquals(
"Application finished event should be published atleast once",
1,
- numOfStringOccurences);
+ numOfStringOccurrences);
// Verify RM posting AppAttempt life cycle Events are getting published
String appAttemptMetricsTimestampFileName =
@@ -609,13 +609,13 @@ public class TestDistributedShell {
Assert.assertEquals(
"AppAttempt register event should be published atleast once",
1,
- getNumOfStringOccurences(appAttemptEntityFile,
+ getNumOfStringOccurrences(appAttemptEntityFile,
AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE));
Assert.assertEquals(
"AppAttempt finished event should be published atleast once",
1,
- getNumOfStringOccurences(appAttemptEntityFile,
+ getNumOfStringOccurrences(appAttemptEntityFile,
AppAttemptMetricsConstants.FINISHED_EVENT_TYPE));
} finally {
FileUtils.deleteDirectory(tmpRootFolder.getParentFile());
@@ -636,7 +636,7 @@ public class TestDistributedShell {
return entityFile;
}
- private long getNumOfStringOccurences(File entityFile, String searchString)
+ private long getNumOfStringOccurrences(File entityFile, String searchString)
throws IOException {
BufferedReader reader = null;
String strLine;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 23b99d9..8dcf4be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -641,8 +641,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
ContainerLaunch launch = new ContainerLaunch(context, conf, dispatcher,
exec, app, container, dirsHandler, containerManager);
launch.call();
- Assert.assertTrue("ContainerExitEvent should have occured",
- eventHandler.isContainerExitEventOccured());
+ Assert.assertTrue("ContainerExitEvent should have occurred",
+ eventHandler.isContainerExitEventOccurred());
}
private static class ContainerExitHandler implements EventHandler<Event> {
@@ -652,15 +652,15 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
this.testForMultiFile = testForMultiFile;
}
- boolean containerExitEventOccured = false;
+ boolean containerExitEventOccurred = false;
- public boolean isContainerExitEventOccured() {
- return containerExitEventOccured;
+ public boolean isContainerExitEventOccurred() {
+ return containerExitEventOccurred;
}
public void handle(Event event) {
if (event instanceof ContainerExitEvent) {
- containerExitEventOccured = true;
+ containerExitEventOccurred = true;
ContainerExitEvent exitEvent = (ContainerExitEvent) event;
Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
exitEvent.getType());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eba7923/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index c4b7a0d..cdbbc51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -343,43 +343,43 @@ public class TestParentQueue {
csConf.setCapacity(Q_B, 70.5F);
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
- boolean exceptionOccured = false;
+ boolean exceptionOccurred = false;
try {
CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
TestUtils.spyHook);
} catch (IllegalArgumentException ie) {
- exceptionOccured = true;
+ exceptionOccurred = true;
}
- if (!exceptionOccured) {
+ if (!exceptionOccurred) {
Assert.fail("Capacity is more then 100% so should be failed.");
}
csConf.setCapacity(Q_A, 30);
csConf.setCapacity(Q_B, 70);
- exceptionOccured = false;
+ exceptionOccurred = false;
queues.clear();
try {
CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
TestUtils.spyHook);
} catch (IllegalArgumentException ie) {
- exceptionOccured = true;
+ exceptionOccurred = true;
}
- if (exceptionOccured) {
+ if (exceptionOccurred) {
Assert.fail("Capacity is 100% so should not be failed.");
}
csConf.setCapacity(Q_A, 30);
csConf.setCapacity(Q_B, 70.005F);
- exceptionOccured = false;
+ exceptionOccurred = false;
queues.clear();
try {
CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
TestUtils.spyHook);
} catch (IllegalArgumentException ie) {
- exceptionOccured = true;
+ exceptionOccurred = true;
}
- if (exceptionOccured) {
+ if (exceptionOccurred) {
Assert
.fail("Capacity is under PRECISION which is .05% so should not be failed.");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: Revert "HADOOP-11794. Enable
distcp to copy blocks in parallel. Contributed by Yongjun Zhang,
Wei-Chiu Chuang, Xiao Chen."
Posted by ae...@apache.org.
Revert "HADOOP-11794. Enable distcp to copy blocks in parallel. Contributed by Yongjun Zhang, Wei-Chiu Chuang, Xiao Chen."
This reverts commit 064c8b25eca9bc825dc07a54d9147d65c9290a03.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/144f1cf7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/144f1cf7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/144f1cf7
Branch: refs/heads/HDFS-7240
Commit: 144f1cf76527e6c75aec77ef683a898580f3cc8d
Parents: 064c8b2
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Thu Mar 30 17:38:18 2017 -0700
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Thu Mar 30 17:38:18 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 22 +-
.../org/apache/hadoop/tools/CopyListing.java | 37 +-
.../hadoop/tools/CopyListingFileStatus.java | 87 +----
.../java/org/apache/hadoop/tools/DistCp.java | 52 ---
.../apache/hadoop/tools/DistCpOptionSwitch.java | 10 -
.../org/apache/hadoop/tools/DistCpOptions.java | 22 +-
.../org/apache/hadoop/tools/OptionsParser.java | 36 +-
.../apache/hadoop/tools/SimpleCopyListing.java | 83 ++---
.../hadoop/tools/mapred/CopyCommitter.java | 174 +--------
.../apache/hadoop/tools/mapred/CopyMapper.java | 40 +-
.../tools/mapred/RetriableFileCopyCommand.java | 26 +-
.../tools/mapred/UniformSizeInputFormat.java | 5 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 111 +-----
.../src/site/markdown/DistCp.md.vm | 1 -
.../apache/hadoop/tools/TestDistCpSystem.java | 368 ++-----------------
.../apache/hadoop/tools/TestOptionsParser.java | 2 +-
.../hadoop/tools/mapred/TestCopyCommitter.java | 5 +-
17 files changed, 110 insertions(+), 971 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 9b782f3..1329195 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -862,27 +862,7 @@ public class DFSTestUtil {
out.write(toAppend);
}
}
-
- /**
- * Append specified length of bytes to a given file, starting with new block.
- * @param fs The file system
- * @param p Path of the file to append
- * @param length Length of bytes to append to the file
- * @throws IOException
- */
- public static void appendFileNewBlock(DistributedFileSystem fs,
- Path p, int length) throws IOException {
- assert fs.exists(p);
- assert length >= 0;
- byte[] toAppend = new byte[length];
- Random random = new Random();
- random.nextBytes(toAppend);
- try (FSDataOutputStream out = fs.append(p,
- EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null)) {
- out.write(toAppend);
- }
- }
-
+
/**
* @return url content as string (UTF-8 encoding assumed)
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
index 9ebf9d2..481aa61 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
@@ -145,22 +145,12 @@ public abstract class CopyListing extends Configured {
Configuration config = getConf();
FileSystem fs = pathToListFile.getFileSystem(config);
- final boolean splitLargeFile = options.splitLargeFile();
-
- // When splitLargeFile is enabled, we don't randomize the copylist
- // earlier, so we don't do the sorting here. For a file that has
- // multiple entries due to split, we check here that their
- // <chunkOffset, chunkLength> is continuous.
- //
- Path checkPath = splitLargeFile?
- pathToListFile : DistCpUtils.sortListing(fs, config, pathToListFile);
+ Path sortedList = DistCpUtils.sortListing(fs, config, pathToListFile);
SequenceFile.Reader reader = new SequenceFile.Reader(
- config, SequenceFile.Reader.file(checkPath));
+ config, SequenceFile.Reader.file(sortedList));
try {
Text lastKey = new Text("*"); //source relative path can never hold *
- long lastChunkOffset = -1;
- long lastChunkLength = -1;
CopyListingFileStatus lastFileStatus = new CopyListingFileStatus();
Text currentKey = new Text();
@@ -171,21 +161,8 @@ public abstract class CopyListing extends Configured {
if (currentKey.equals(lastKey)) {
CopyListingFileStatus currentFileStatus = new CopyListingFileStatus();
reader.getCurrentValue(currentFileStatus);
- if (!splitLargeFile) {
- throw new DuplicateFileException("File " + lastFileStatus.getPath()
- + " and " + currentFileStatus.getPath()
- + " would cause duplicates. Aborting");
- } else {
- if (lastChunkOffset + lastChunkLength !=
- currentFileStatus.getChunkOffset()) {
- throw new InvalidInputException("File " + lastFileStatus.getPath()
- + " " + lastChunkOffset + "," + lastChunkLength
- + " and " + currentFileStatus.getPath()
- + " " + currentFileStatus.getChunkOffset() + ","
- + currentFileStatus.getChunkLength()
- + " are not continuous. Aborting");
- }
- }
+ throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " +
+ currentFileStatus.getPath() + " would cause duplicates. Aborting");
}
reader.getCurrentValue(lastFileStatus);
if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
@@ -204,12 +181,8 @@ public abstract class CopyListing extends Configured {
xAttrSupportCheckFsSet.add(lastFsUri);
}
}
-
lastKey.set(currentKey);
- if (splitLargeFile) {
- lastChunkOffset = lastFileStatus.getChunkOffset();
- lastChunkLength = lastFileStatus.getChunkLength();
- }
+
if (options.shouldUseDiff() && LOG.isDebugEnabled()) {
LOG.debug("Copy list entry " + idx + ": " +
lastFileStatus.getPath().toUri().getPath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
index 29c59ac..00d4b32 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
@@ -74,14 +74,6 @@ public final class CopyListingFileStatus implements Writable {
private List<AclEntry> aclEntries;
private Map<String, byte[]> xAttrs;
- // <chunkOffset, chunkLength> represents the offset and length of a file
- // chunk in number of bytes.
- // used when splitting a large file to chunks to copy in parallel.
- // If a file is not large enough to split, chunkOffset would be 0 and
- // chunkLength would be the length of the file.
- private long chunkOffset = 0;
- private long chunkLength = Long.MAX_VALUE;
-
/**
* Default constructor.
*/
@@ -104,32 +96,11 @@ public final class CopyListingFileStatus implements Writable {
fileStatus.getPath());
}
- public CopyListingFileStatus(FileStatus fileStatus,
- long chunkOffset, long chunkLength) {
- this(fileStatus.getLen(), fileStatus.isDirectory(),
- fileStatus.getReplication(), fileStatus.getBlockSize(),
- fileStatus.getModificationTime(), fileStatus.getAccessTime(),
- fileStatus.getPermission(), fileStatus.getOwner(),
- fileStatus.getGroup(),
- fileStatus.getPath());
- this.chunkOffset = chunkOffset;
- this.chunkLength = chunkLength;
- }
-
@SuppressWarnings("checkstyle:parameternumber")
public CopyListingFileStatus(long length, boolean isdir,
int blockReplication, long blocksize, long modificationTime,
long accessTime, FsPermission permission, String owner, String group,
Path path) {
- this(length, isdir, blockReplication, blocksize, modificationTime,
- accessTime, permission, owner, group, path, 0, Long.MAX_VALUE);
- }
-
- @SuppressWarnings("checkstyle:parameternumber")
- public CopyListingFileStatus(long length, boolean isdir,
- int blockReplication, long blocksize, long modificationTime,
- long accessTime, FsPermission permission, String owner, String group,
- Path path, long chunkOffset, long chunkLength) {
this.length = length;
this.isdir = isdir;
this.blockReplication = (short)blockReplication;
@@ -146,23 +117,6 @@ public final class CopyListingFileStatus implements Writable {
this.owner = (owner == null) ? "" : owner;
this.group = (group == null) ? "" : group;
this.path = path;
- this.chunkOffset = chunkOffset;
- this.chunkLength = chunkLength;
- }
-
- public CopyListingFileStatus(CopyListingFileStatus other) {
- this.length = other.length;
- this.isdir = other.isdir;
- this.blockReplication = other.blockReplication;
- this.blocksize = other.blocksize;
- this.modificationTime = other.modificationTime;
- this.accessTime = other.accessTime;
- this.permission = other.permission;
- this.owner = other.owner;
- this.group = other.group;
- this.path = new Path(other.path.toUri());
- this.chunkOffset = other.chunkOffset;
- this.chunkLength = other.chunkLength;
}
public Path getPath() {
@@ -246,31 +200,6 @@ public final class CopyListingFileStatus implements Writable {
this.xAttrs = xAttrs;
}
- public long getChunkOffset() {
- return chunkOffset;
- }
-
- public void setChunkOffset(long offset) {
- this.chunkOffset = offset;
- }
-
- public long getChunkLength() {
- return chunkLength;
- }
-
- public void setChunkLength(long chunkLength) {
- this.chunkLength = chunkLength;
- }
-
- public boolean isSplit() {
- return getChunkLength() != Long.MAX_VALUE &&
- getChunkLength() != getLen();
- }
-
- public long getSizeToCopy() {
- return isSplit()? getChunkLength() : getLen();
- }
-
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN);
@@ -315,9 +244,6 @@ public final class CopyListingFileStatus implements Writable {
} else {
out.writeInt(NO_XATTRS);
}
-
- out.writeLong(chunkOffset);
- out.writeLong(chunkLength);
}
@Override
@@ -366,9 +292,6 @@ public final class CopyListingFileStatus implements Writable {
} else {
xAttrs = null;
}
-
- chunkOffset = in.readLong();
- chunkLength = in.readLong();
}
@Override
@@ -394,14 +317,8 @@ public final class CopyListingFileStatus implements Writable {
public String toString() {
StringBuilder sb = new StringBuilder(super.toString());
sb.append('{');
- sb.append(this.getPath().toString());
- sb.append(" length = ").append(this.getLen());
- sb.append(" aclEntries = ").append(aclEntries);
- sb.append(", xAttrs = ").append(xAttrs);
- if (isSplit()) {
- sb.append(", chunkOffset = ").append(this.getChunkOffset());
- sb.append(", chunkLength = ").append(this.getChunkLength());
- }
+ sb.append("aclEntries = " + aclEntries);
+ sb.append(", xAttrs = " + xAttrs);
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 8c2fa24..ab58e9c 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.mapreduce.Cluster;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
-import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.CopyListing.*;
import org.apache.hadoop.tools.mapred.CopyMapper;
import org.apache.hadoop.tools.mapred.CopyOutputFormat;
@@ -135,7 +134,6 @@ public class DistCp extends Configured implements Tool {
try {
inputOptions = (OptionsParser.parse(argv));
- setOptionsForSplitLargeFile();
setTargetPathExists();
LOG.info("Input Options: " + inputOptions);
} catch (Throwable e) {
@@ -237,56 +235,6 @@ public class DistCp extends Configured implements Tool {
getConf().setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS,
targetExists);
}
-
- /**
- * Check if concat is supported by fs.
- * Throws UnsupportedOperationException if not.
- */
- private void checkConcatSupport(FileSystem fs) {
- try {
- Path[] src = null;
- Path tgt = null;
- fs.concat(tgt, src);
- } catch (UnsupportedOperationException use) {
- throw new UnsupportedOperationException(
- DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() +
- " is not supported since the target file system doesn't" +
- " support concat.", use);
- } catch (Exception e) {
- // Ignore other exception
- }
- }
-
- /**
- * Set up needed options for splitting large files.
- */
- private void setOptionsForSplitLargeFile() throws IOException {
- if (!inputOptions.splitLargeFile()) {
- return;
- }
- Path target = inputOptions.getTargetPath();
- FileSystem targetFS = target.getFileSystem(getConf());
- checkConcatSupport(targetFS);
-
- LOG.info("Enabling preserving blocksize since "
- + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " is passed.");
- inputOptions.preserve(FileAttribute.BLOCKSIZE);
-
- LOG.info("Set " +
- DistCpOptionSwitch.APPEND.getSwitch()
- + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()
- + " is passed.");
- inputOptions.setAppend(false);
-
- LOG.info("Set " +
- DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES
- + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()
- + " is passed.");
- getConf().setBoolean(
- DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false);
- }
-
-
/**
* Create Job object for submitting it, with all the configuration
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index ced9b54..fb47d76 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -169,16 +169,6 @@ public enum DistCpOptionSwitch {
new Option("sizelimit", true, "(Deprecated!) Limit number of files " +
"copied to <= n bytes")),
- BLOCKS_PER_CHUNK("",
- new Option("blocksperchunk", true, "If set to a positive value, files"
- + "with more blocks than this value will be split into chunks of "
- + "<blocksperchunk> blocks to be transferred in parallel, and "
- + "reassembled on the destination. By default, <blocksperchunk> is "
- + "0 and the files will be transmitted in their entirety without "
- + "splitting. This switch is only applicable when the source file "
- + "system implements getBlockLocations method and the target file "
- + "system implements concat method")),
-
/**
* Specify bandwidth per map in MB, accepts bandwidth as a fraction
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index 9822d83..8c37ff3 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -97,11 +97,7 @@ public class DistCpOptions {
// targetPathExist is a derived field, it's initialized in the
// beginning of distcp.
private boolean targetPathExists = true;
-
- // Size of chunk in number of blocks when splitting large file into chunks
- // to copy in parallel. Default is 0 and file are not splitted.
- private int blocksPerChunk = 0;
-
+
public static enum FileAttribute{
REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES;
@@ -170,7 +166,6 @@ public class DistCpOptions {
this.targetPath = that.getTargetPath();
this.targetPathExists = that.getTargetPathExists();
this.filtersFile = that.getFiltersFile();
- this.blocksPerChunk = that.blocksPerChunk;
}
}
@@ -583,18 +578,6 @@ public class DistCpOptions {
this.filtersFile = filtersFilename;
}
- public final void setBlocksPerChunk(int csize) {
- this.blocksPerChunk = csize;
- }
-
- public final int getBlocksPerChunk() {
- return blocksPerChunk;
- }
-
- public final boolean splitLargeFile() {
- return blocksPerChunk > 0;
- }
-
void validate() {
if ((useDiff || useRdiff) && deleteMissing) {
// -delete and -diff/-rdiff are mutually exclusive. For backward
@@ -686,8 +669,6 @@ public class DistCpOptions {
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.FILTERS,
filtersFile);
}
- DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BLOCKS_PER_CHUNK,
- String.valueOf(blocksPerChunk));
}
/**
@@ -723,7 +704,6 @@ public class DistCpOptions {
", targetPath=" + targetPath +
", targetPathExists=" + targetPathExists +
", filtersFile='" + filtersFile + '\'' +
- ", blocksPerChunk=" + blocksPerChunk +
'}';
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
index 8881264..d0f82ca 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
@@ -172,44 +172,11 @@ public class OptionsParser {
DistCpOptionSwitch.FILTERS.getSwitch()));
}
- parseBlocksPerChunk(command, option);
-
option.validate();
return option;
}
-
- /**
- * A helper method to parse chunk size in number of blocks.
- * Used when breaking large file into chunks to copy in parallel.
- *
- * @param command command line arguments
- */
- private static void parseBlocksPerChunk(CommandLine command,
- DistCpOptions option) {
- boolean hasOption =
- command.hasOption(DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch());
- LOG.info("parseChunkSize: " +
- DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " " + hasOption);
- if (hasOption) {
- String chunkSizeString = getVal(command,
- DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch().trim());
- try {
- int csize = Integer.parseInt(chunkSizeString);
- if (csize < 0) {
- csize = 0;
- }
- LOG.info("Set distcp blocksPerChunk to " + csize);
- option.setBlocksPerChunk(csize);
- }
- catch (NumberFormatException e) {
- throw new IllegalArgumentException("blocksPerChunk is invalid: "
- + chunkSizeString, e);
- }
- }
- }
-
/**
* parseSizeLimit is a helper method for parsing the deprecated
* argument SIZE_LIMIT.
@@ -244,7 +211,8 @@ public class OptionsParser {
DistCpOptionSwitch.FILE_LIMIT.getSwitch().trim());
try {
Integer.parseInt(fileLimitString);
- } catch (NumberFormatException e) {
+ }
+ catch (NumberFormatException e) {
throw new IllegalArgumentException("File-limit is invalid: "
+ fileLimitString, e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
index af91347..105e4f2 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.tools;
import com.google.common.collect.Lists;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
@@ -48,7 +47,6 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
-import java.util.LinkedList;
import static org.apache.hadoop.tools.DistCpConstants
.HDFS_RESERVED_RAW_DIRECTORY_NAME;
@@ -242,10 +240,10 @@ public class SimpleCopyListing extends CopyListing {
final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs();
- LinkedList<CopyListingFileStatus> fileCopyListingStatus =
+ CopyListingFileStatus fileCopyListingStatus =
DistCpUtils.toCopyListingFileStatus(sourceFS, fileStatus,
- preserveAcls, preserveXAttrs, preserveRawXAttrs,
- options.getBlocksPerChunk());
+ preserveAcls, preserveXAttrs, preserveRawXAttrs);
+
writeToFileListingRoot(fileListWriter, fileCopyListingStatus,
sourceRoot, options);
}
@@ -350,10 +348,9 @@ public class SimpleCopyListing extends CopyListing {
FileStatus[] sourceFiles = sourceFS.listStatus(path);
boolean explore = (sourceFiles != null && sourceFiles.length > 0);
if (!explore || rootStatus.isDirectory()) {
- LinkedList<CopyListingFileStatus> rootCopyListingStatus =
- DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus,
- preserveAcls, preserveXAttrs, preserveRawXAttrs,
- options.getBlocksPerChunk());
+ CopyListingFileStatus rootCopyListingStatus =
+ DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus,
+ preserveAcls, preserveXAttrs, preserveRawXAttrs);
writeToFileListingRoot(fileListWriter, rootCopyListingStatus,
sourcePathRoot, options);
}
@@ -363,20 +360,20 @@ public class SimpleCopyListing extends CopyListing {
if (LOG.isDebugEnabled()) {
LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy.");
}
- LinkedList<CopyListingFileStatus> sourceCopyListingStatus =
- DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus,
- preserveAcls && sourceStatus.isDirectory(),
- preserveXAttrs && sourceStatus.isDirectory(),
- preserveRawXAttrs && sourceStatus.isDirectory(),
- options.getBlocksPerChunk());
- for (CopyListingFileStatus fs : sourceCopyListingStatus) {
- if (randomizeFileListing) {
- addToFileListing(statusList,
- new FileStatusInfo(fs, sourcePathRoot), fileListWriter);
- } else {
- writeToFileListing(fileListWriter, fs, sourcePathRoot);
- }
+ CopyListingFileStatus sourceCopyListingStatus =
+ DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus,
+ preserveAcls && sourceStatus.isDirectory(),
+ preserveXAttrs && sourceStatus.isDirectory(),
+ preserveRawXAttrs && sourceStatus.isDirectory());
+ if (randomizeFileListing) {
+ addToFileListing(statusList,
+ new FileStatusInfo(sourceCopyListingStatus, sourcePathRoot),
+ fileListWriter);
+ } else {
+ writeToFileListing(fileListWriter, sourceCopyListingStatus,
+ sourcePathRoot);
}
+
if (sourceStatus.isDirectory()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding source dir for traverse: " + sourceStatus.getPath());
@@ -644,20 +641,18 @@ public class SimpleCopyListing extends CopyListing {
LOG.debug("Recording source-path: " + child.getPath() + " for copy.");
}
if (workResult.getSuccess()) {
- LinkedList<CopyListingFileStatus> childCopyListingStatus =
+ CopyListingFileStatus childCopyListingStatus =
DistCpUtils.toCopyListingFileStatus(sourceFS, child,
preserveAcls && child.isDirectory(),
preserveXAttrs && child.isDirectory(),
- preserveRawXattrs && child.isDirectory(),
- options.getBlocksPerChunk());
-
- for (CopyListingFileStatus fs : childCopyListingStatus) {
- if (randomizeFileListing) {
- addToFileListing(fileStatuses,
- new FileStatusInfo(fs, sourcePathRoot), fileListWriter);
- } else {
- writeToFileListing(fileListWriter, fs, sourcePathRoot);
- }
+ preserveRawXattrs && child.isDirectory());
+ if (randomizeFileListing) {
+ addToFileListing(fileStatuses,
+ new FileStatusInfo(childCopyListingStatus, sourcePathRoot),
+ fileListWriter);
+ } else {
+ writeToFileListing(fileListWriter, childCopyListingStatus,
+ sourcePathRoot);
}
}
if (retry < maxRetries) {
@@ -680,21 +675,19 @@ public class SimpleCopyListing extends CopyListing {
}
private void writeToFileListingRoot(SequenceFile.Writer fileListWriter,
- LinkedList<CopyListingFileStatus> fileStatus, Path sourcePathRoot,
+ CopyListingFileStatus fileStatus, Path sourcePathRoot,
DistCpOptions options) throws IOException {
boolean syncOrOverwrite = options.shouldSyncFolder() ||
options.shouldOverwrite();
- for (CopyListingFileStatus fs : fileStatus) {
- if (fs.getPath().equals(sourcePathRoot) &&
- fs.isDirectory() && syncOrOverwrite) {
- // Skip the root-paths when syncOrOverwrite
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skip " + fs.getPath());
- }
- return;
- }
- writeToFileListing(fileListWriter, fs, sourcePathRoot);
+ if (fileStatus.getPath().equals(sourcePathRoot) &&
+ fileStatus.isDirectory() && syncOrOverwrite) {
+ // Skip the root-paths when syncOrOverwrite
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skip " + fileStatus.getPath());
+ }
+ return;
}
+ writeToFileListing(fileListWriter, fileStatus, sourcePathRoot);
}
private void writeToFileListing(SequenceFile.Writer fileListWriter,
@@ -714,7 +707,7 @@ public class SimpleCopyListing extends CopyListing {
fileListWriter.sync();
if (!fileStatus.isDirectory()) {
- totalBytesToCopy += fileStatus.getSizeToCopy();
+ totalBytesToCopy += fileStatus.getLen();
} else {
totalDirs++;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
index 6ddaab9..75cefb4 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -35,17 +34,14 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.tools.CopyListing;
import org.apache.hadoop.tools.CopyListingFileStatus;
import org.apache.hadoop.tools.DistCpConstants;
-import org.apache.hadoop.tools.DistCpOptionSwitch;
import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.GlobbedCopyListing;
import org.apache.hadoop.tools.util.DistCpUtils;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
-import java.util.LinkedList;
import java.util.List;
/**
@@ -67,8 +63,7 @@ public class CopyCommitter extends FileOutputCommitter {
private boolean syncFolder = false;
private boolean overwrite = false;
private boolean targetPathExists = true;
- private boolean ignoreFailures = false;
-
+
/**
* Create a output committer
*
@@ -87,13 +82,8 @@ public class CopyCommitter extends FileOutputCommitter {
Configuration conf = jobContext.getConfiguration();
syncFolder = conf.getBoolean(DistCpConstants.CONF_LABEL_SYNC_FOLDERS, false);
overwrite = conf.getBoolean(DistCpConstants.CONF_LABEL_OVERWRITE, false);
- targetPathExists = conf.getBoolean(
- DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true);
- ignoreFailures = conf.getBoolean(
- DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
-
- concatFileChunks(conf);
-
+ targetPathExists = conf.getBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true);
+
super.commitJob(jobContext);
cleanupTempFiles(jobContext);
@@ -179,112 +169,9 @@ public class CopyCommitter extends FileOutputCommitter {
}
}
- private boolean isFileNotFoundException(IOException e) {
- if (e instanceof FileNotFoundException) {
- return true;
- }
-
- if (e instanceof RemoteException) {
- return ((RemoteException)e).unwrapRemoteException()
- instanceof FileNotFoundException;
- }
-
- return false;
- }
-
- /**
- * Concat chunk files for the same file into one.
- * Iterate through copy listing, identify chunk files for the same file,
- * concat them into one.
- */
- private void concatFileChunks(Configuration conf) throws IOException {
-
- LOG.info("concat file chunks ...");
-
- String spath = conf.get(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH);
- if (spath == null || spath.isEmpty()) {
- return;
- }
- Path sourceListing = new Path(spath);
- SequenceFile.Reader sourceReader = new SequenceFile.Reader(conf,
- SequenceFile.Reader.file(sourceListing));
- Path targetRoot =
- new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
-
- try {
- CopyListingFileStatus srcFileStatus = new CopyListingFileStatus();
- Text srcRelPath = new Text();
- CopyListingFileStatus lastFileStatus = null;
- LinkedList<Path> allChunkPaths = new LinkedList<Path>();
-
- // Iterate over every source path that was copied.
- while (sourceReader.next(srcRelPath, srcFileStatus)) {
- if (srcFileStatus.isDirectory()) {
- continue;
- }
- Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath);
- Path targetFileChunkPath =
- DistCpUtils.getSplitChunkPath(targetFile, srcFileStatus);
- if (LOG.isDebugEnabled()) {
- LOG.debug(" add " + targetFileChunkPath + " to concat.");
- }
- allChunkPaths.add(targetFileChunkPath);
- if (srcFileStatus.getChunkOffset() + srcFileStatus.getChunkLength()
- == srcFileStatus.getLen()) {
- // This is the last chunk of the splits, consolidate allChunkPaths
- try {
- concatFileChunks(conf, targetFile, allChunkPaths);
- } catch (IOException e) {
- // If the concat failed because a chunk file doesn't exist,
- // then we assume that the CopyMapper has skipped copying this
- // file, and we ignore the exception here.
- // If a chunk file should have been created but it was not, then
- // the CopyMapper would have failed.
- if (!isFileNotFoundException(e)) {
- String emsg = "Failed to concat chunk files for " + targetFile;
- if (!ignoreFailures) {
- throw new IOException(emsg, e);
- } else {
- LOG.warn(emsg, e);
- }
- }
- }
- allChunkPaths.clear();
- lastFileStatus = null;
- } else {
- if (lastFileStatus == null) {
- lastFileStatus = new CopyListingFileStatus(srcFileStatus);
- } else {
- // Two neighboring chunks have to be consecutive ones for the same
- // file, for them to be merged
- if (!srcFileStatus.getPath().equals(lastFileStatus.getPath()) ||
- srcFileStatus.getChunkOffset() !=
- (lastFileStatus.getChunkOffset() +
- lastFileStatus.getChunkLength())) {
- String emsg = "Inconsistent sequence file: current " +
- "chunk file " + srcFileStatus + " doesnt match prior " +
- "entry " + lastFileStatus;
- if (!ignoreFailures) {
- throw new IOException(emsg);
- } else {
- LOG.warn(emsg + ", skipping concat this set.");
- }
- } else {
- lastFileStatus.setChunkOffset(srcFileStatus.getChunkOffset());
- lastFileStatus.setChunkLength(srcFileStatus.getChunkLength());
- }
- }
- }
- }
- } finally {
- IOUtils.closeStream(sourceReader);
- }
- }
-
// This method changes the target-directories' file-attributes (owner,
// user/group permissions, etc.) based on the corresponding source directories.
- private void preserveFileAttributesForDirectories(Configuration conf)
- throws IOException {
+ private void preserveFileAttributesForDirectories(Configuration conf) throws IOException {
String attrSymbols = conf.get(DistCpConstants.CONF_LABEL_PRESERVE_STATUS);
final boolean syncOrOverwrite = syncFolder || overwrite;
@@ -438,57 +325,4 @@ public class CopyCommitter extends FileOutputCommitter {
", Unable to move to " + finalDir);
}
}
-
- /**
- * Concat the passed chunk files into one and rename it the targetFile.
- */
- private void concatFileChunks(Configuration conf, Path targetFile,
- LinkedList<Path> allChunkPaths) throws IOException {
- if (allChunkPaths.size() == 1) {
- return;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("concat " + targetFile + " allChunkSize+ "
- + allChunkPaths.size());
- }
- FileSystem dstfs = targetFile.getFileSystem(conf);
-
- Path firstChunkFile = allChunkPaths.removeFirst();
- Path[] restChunkFiles = new Path[allChunkPaths.size()];
- allChunkPaths.toArray(restChunkFiles);
- if (LOG.isDebugEnabled()) {
- LOG.debug("concat: firstchunk: " + dstfs.getFileStatus(firstChunkFile));
- int i = 0;
- for (Path f : restChunkFiles) {
- LOG.debug("concat: other chunk: " + i + ": " + dstfs.getFileStatus(f));
- ++i;
- }
- }
- dstfs.concat(firstChunkFile, restChunkFiles);
- if (LOG.isDebugEnabled()) {
- LOG.debug("concat: result: " + dstfs.getFileStatus(firstChunkFile));
- }
- rename(dstfs, firstChunkFile, targetFile);
- }
-
- /**
- * Rename tmp to dst on destFileSys.
- * @param destFileSys the file ssystem
- * @param tmp the source path
- * @param dst the destination path
- * @throws IOException if renaming failed
- */
- private static void rename(FileSystem destFileSys, Path tmp, Path dst)
- throws IOException {
- try {
- if (destFileSys.exists(dst)) {
- destFileSys.delete(dst, true);
- }
- destFileSys.rename(tmp, dst);
- } catch (IOException ioe) {
- throw new IOException("Fail to rename tmp file (=" + tmp
- + ") to destination file (=" + dst + ")", ioe);
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
index d6b3ba8..e1873f1 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
@@ -156,12 +156,10 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
sourceFS = sourcePath.getFileSystem(conf);
final boolean preserveXAttrs =
fileAttributes.contains(FileAttribute.XATTR);
- sourceCurrStatus = DistCpUtils.toCopyListingFileStatusHelper(sourceFS,
- sourceFS.getFileStatus(sourcePath),
- fileAttributes.contains(FileAttribute.ACL),
- preserveXAttrs, preserveRawXattrs,
- sourceFileStatus.getChunkOffset(),
- sourceFileStatus.getChunkLength());
+ sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS,
+ sourceFS.getFileStatus(sourcePath),
+ fileAttributes.contains(FileAttribute.ACL),
+ preserveXAttrs, preserveRawXattrs);
} catch (FileNotFoundException e) {
throw new IOException(new RetriableFileCopyCommand.CopyReadException(e));
}
@@ -175,8 +173,7 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
LOG.debug("Path could not be found: " + target, ignore);
}
- if (targetStatus != null &&
- (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) {
+ if (targetStatus != null && (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) {
throw new IOException("Can't replace " + target + ". Target is " +
getFileType(targetStatus) + ", Source is " + getFileType(sourceCurrStatus));
}
@@ -186,28 +183,19 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
return;
}
- FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target,
- targetStatus);
-
- Path tmpTarget = target;
+ FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, targetStatus);
if (action == FileAction.SKIP) {
LOG.info("Skipping copy of " + sourceCurrStatus.getPath()
+ " to " + target);
updateSkipCounters(context, sourceCurrStatus);
context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath()));
-
} else {
- if (sourceCurrStatus.isSplit()) {
- tmpTarget = DistCpUtils.getSplitChunkPath(target, sourceCurrStatus);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("copying " + sourceCurrStatus + " " + tmpTarget);
- }
- copyFileWithRetry(description, sourceCurrStatus, tmpTarget, context,
+ copyFileWithRetry(description, sourceCurrStatus, target, context,
action, fileAttributes);
}
- DistCpUtils.preserve(target.getFileSystem(conf), tmpTarget,
- sourceCurrStatus, fileAttributes, preserveRawXattrs);
+
+ DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus,
+ fileAttributes, preserveRawXattrs);
} catch (IOException exception) {
handleFailures(exception, sourceFileStatus, target, context);
}
@@ -273,12 +261,8 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
private void handleFailures(IOException exception,
CopyListingFileStatus sourceFileStatus, Path target, Context context)
throws IOException, InterruptedException {
- LOG.error("Failure in copying " + sourceFileStatus.getPath() +
- (sourceFileStatus.isSplit()? ","
- + " offset=" + sourceFileStatus.getChunkOffset()
- + " chunkLength=" + sourceFileStatus.getChunkLength()
- : "") +
- " to " + target, exception);
+ LOG.error("Failure in copying " + sourceFileStatus.getPath() + " to " +
+ target, exception);
if (ignoreFailures &&
ExceptionUtils.indexOfType(exception, CopyReadException.class) != -1) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
index 2c17fef..06acd78 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
@@ -118,21 +118,17 @@ public class RetriableFileCopyCommand extends RetriableCommand {
.contains(FileAttribute.CHECKSUMTYPE) ? sourceFS
.getFileChecksum(sourcePath) : null;
- long offset = (action == FileAction.APPEND) ?
- targetFS.getFileStatus(target).getLen() : source.getChunkOffset();
+ final long offset = action == FileAction.APPEND ? targetFS.getFileStatus(
+ target).getLen() : 0;
long bytesRead = copyToFile(targetPath, targetFS, source,
offset, context, fileAttributes, sourceChecksum);
- if (!source.isSplit()) {
- compareFileLengths(source, targetPath, configuration, bytesRead
- + offset);
- }
+ compareFileLengths(source, targetPath, configuration, bytesRead
+ + offset);
//At this point, src&dest lengths are same. if length==0, we skip checksum
if ((bytesRead != 0) && (!skipCrc)) {
- if (!source.isSplit()) {
- compareCheckSums(sourceFS, source.getPath(), sourceChecksum,
- targetFS, targetPath);
- }
+ compareCheckSums(sourceFS, source.getPath(), sourceChecksum,
+ targetFS, targetPath);
}
// it's not append case, thus we first write to a temporary file, rename
// it to the target path.
@@ -253,26 +249,16 @@ public class RetriableFileCopyCommand extends RetriableCommand {
ThrottledInputStream inStream = null;
long totalBytesRead = 0;
- long chunkLength = source2.getChunkLength();
- boolean finished = false;
try {
inStream = getInputStream(source, context.getConfiguration());
int bytesRead = readBytes(inStream, buf, sourceOffset);
while (bytesRead >= 0) {
- if (chunkLength > 0 &&
- (totalBytesRead + bytesRead) >= chunkLength) {
- bytesRead = (int)(chunkLength - totalBytesRead);
- finished = true;
- }
totalBytesRead += bytesRead;
if (action == FileAction.APPEND) {
sourceOffset += bytesRead;
}
outStream.write(buf, 0, bytesRead);
updateContextStatus(totalBytesRead, context, source2);
- if (finished) {
- break;
- }
bytesRead = readBytes(inStream, buf, sourceOffset);
}
outStream.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
index d1c18ea..3e86d09 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
@@ -99,8 +99,7 @@ public class UniformSizeInputFormat
while (reader.next(srcRelPath, srcFileStatus)) {
// If adding the current file would cause the bytes per map to exceed
// limit. Add the current file to new split
- if (currentSplitSize + srcFileStatus.getChunkLength() > nBytesPerSplit
- && lastPosition != 0) {
+ if (currentSplitSize + srcFileStatus.getLen() > nBytesPerSplit && lastPosition != 0) {
FileSplit split = new FileSplit(listingFilePath, lastSplitStart,
lastPosition - lastSplitStart, null);
if (LOG.isDebugEnabled()) {
@@ -110,7 +109,7 @@ public class UniformSizeInputFormat
lastSplitStart = lastPosition;
currentSplitSize = 0;
}
- currentSplitSize += srcFileStatus.getChunkLength();
+ currentSplitSize += srcFileStatus.getLen();
lastPosition = reader.getPosition();
}
if (lastPosition > lastSplitStart) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
index e315b84..76bc4c5 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
@@ -19,11 +19,9 @@
package org.apache.hadoop.tools.util;
import com.google.common.collect.Maps;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -32,7 +30,6 @@ import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputFormat;
@@ -47,7 +44,6 @@ import org.apache.hadoop.util.StringUtils;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.EnumSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -120,7 +116,7 @@ public class DistCpUtils {
* @return Class implementing the strategy specified in options.
*/
public static Class<? extends InputFormat> getStrategy(Configuration conf,
- DistCpOptions options) {
+ DistCpOptions options) {
String confLabel = "distcp."
+ StringUtils.toLowerCase(options.getCopyStrategy())
+ ".strategy" + ".impl";
@@ -302,86 +298,6 @@ public class DistCpUtils {
}
/**
- * Converts FileStatus to a list of CopyListingFileStatus.
- * The resulted list contains either one CopyListingFileStatus per chunk of
- * file-blocks (if file-size exceeds blockSize * blocksPerChunk, and there
- * are more blocks in the file than blocksperChunk), or a single
- * CopyListingFileStatus for the entire file (if file-size is too small to
- * split).
- * If preserving ACLs, populates the CopyListingFileStatus with the ACLs.
- * If preserving XAttrs, populates the CopyListingFileStatus with the XAttrs.
- *
- * @param fileSystem FileSystem containing the file
- * @param fileStatus FileStatus of file
- * @param preserveAcls boolean true if preserving ACLs
- * @param preserveXAttrs boolean true if preserving XAttrs
- * @param preserveRawXAttrs boolean true if preserving raw.* XAttrs
- * @param blocksPerChunk size of chunks when copying chunks in parallel
- * @return list of CopyListingFileStatus
- * @throws IOException if there is an I/O error
- */
- public static LinkedList<CopyListingFileStatus> toCopyListingFileStatus(
- FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls,
- boolean preserveXAttrs, boolean preserveRawXAttrs, int blocksPerChunk)
- throws IOException {
- LinkedList<CopyListingFileStatus> copyListingFileStatus =
- new LinkedList<CopyListingFileStatus>();
-
- final CopyListingFileStatus clfs = toCopyListingFileStatusHelper(
- fileSystem, fileStatus, preserveAcls,
- preserveXAttrs, preserveRawXAttrs,
- 0, fileStatus.getLen());
- final long blockSize = fileStatus.getBlockSize();
- if (LOG.isDebugEnabled()) {
- LOG.debug("toCopyListing: " + fileStatus + " chunkSize: "
- + blocksPerChunk + " isDFS: " +
- (fileSystem instanceof DistributedFileSystem));
- }
- if ((blocksPerChunk > 0) &&
- !fileStatus.isDirectory() &&
- (fileStatus.getLen() > blockSize * blocksPerChunk)) {
- // split only when the file size is larger than the intended chunk size
- final BlockLocation[] blockLocations;
- blockLocations = fileSystem.getFileBlockLocations(fileStatus, 0,
- fileStatus.getLen());
-
- int numBlocks = blockLocations.length;
- long curPos = 0;
- if (numBlocks <= blocksPerChunk) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(" add file " + clfs);
- }
- copyListingFileStatus.add(clfs);
- } else {
- int i = 0;
- while (i < numBlocks) {
- long curLength = 0;
- for (int j = 0; j < blocksPerChunk && i < numBlocks; ++j, ++i) {
- curLength += blockLocations[i].getLength();
- }
- if (curLength > 0) {
- CopyListingFileStatus clfs1 = new CopyListingFileStatus(clfs);
- clfs1.setChunkOffset(curPos);
- clfs1.setChunkLength(curLength);
- if (LOG.isDebugEnabled()) {
- LOG.debug(" add file chunk " + clfs1);
- }
- copyListingFileStatus.add(clfs1);
- curPos += curLength;
- }
- }
- }
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug(" add file/dir " + clfs);
- }
- copyListingFileStatus.add(clfs);
- }
-
- return copyListingFileStatus;
- }
-
- /**
* Converts a FileStatus to a CopyListingFileStatus. If preserving ACLs,
* populates the CopyListingFileStatus with the ACLs. If preserving XAttrs,
* populates the CopyListingFileStatus with the XAttrs.
@@ -391,17 +307,13 @@ public class DistCpUtils {
* @param preserveAcls boolean true if preserving ACLs
* @param preserveXAttrs boolean true if preserving XAttrs
* @param preserveRawXAttrs boolean true if preserving raw.* XAttrs
- * @param chunkOffset chunk offset in bytes
- * @param chunkLength chunk length in bytes
- * @return CopyListingFileStatus
* @throws IOException if there is an I/O error
*/
- public static CopyListingFileStatus toCopyListingFileStatusHelper(
+ public static CopyListingFileStatus toCopyListingFileStatus(
FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls,
- boolean preserveXAttrs, boolean preserveRawXAttrs,
- long chunkOffset, long chunkLength) throws IOException {
+ boolean preserveXAttrs, boolean preserveRawXAttrs) throws IOException {
CopyListingFileStatus copyListingFileStatus =
- new CopyListingFileStatus(fileStatus, chunkOffset, chunkLength);
+ new CopyListingFileStatus(fileStatus);
if (preserveAcls) {
FsPermission perm = fileStatus.getPermission();
if (perm.getAclBit()) {
@@ -558,19 +470,4 @@ public class DistCpUtils {
return (sourceChecksum == null || targetChecksum == null ||
sourceChecksum.equals(targetChecksum));
}
-
- /*
- * Return the Path for a given chunk.
- * Used when splitting large file into chunks to copy in parallel.
- * @param targetFile path to target file
- * @param srcFileStatus source file status in copy listing
- * @return path to the chunk specified by the parameters to store
- * in target cluster temporarily
- */
- public static Path getSplitChunkPath(Path targetFile,
- CopyListingFileStatus srcFileStatus) {
- return new Path(targetFile.toString()
- + ".____distcpSplit____" + srcFileStatus.getChunkOffset()
- + "." + srcFileStatus.getChunkLength());
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
index a77deb2..41a6e94 100644
--- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
+++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
@@ -237,7 +237,6 @@ Flag | Description | Notes
`-rdiff <newSnapshot> <oldSnapshot>` | Use snapshot diff report between given two snapshots to identify what has been changed on the target since the snapshot `<oldSnapshot>` was created on the target, and apply the diff reversely to the target, and copy modified files from the source's `<oldSnapshot>`, to make the target the same as `<oldSnapshot>`. | This option is valid only with `-update` option and the following conditions should be satisfied. <ol><li>Both the source and the target FileSystem must be DistributedFileSystem. The source and the target can be two different clusters/paths, or they can be exactly the same cluster/path. In the latter case, modified files are copied from target's `<oldSnapshot>` to target's current state).</li> <li> Two snapshots `<newSnapshot>` and `<oldSnapshot>` have been created on the target FS, and `<oldSnapshot>` is older than `<newSnapshot>`. No change has been made on target since `<newSnapshot>` was created on the target. </li> <li> The sour
ce has the same snapshot `<oldSnapshot>`, which has the same content as the `<oldSnapshot>` on the target. All the files/directories in the target's `<oldSnapshot>` are the same with source's `<oldSnapshot>`.</li> </ol> |
`-numListstatusThreads` | Number of threads to use for building file listing | At most 40 threads.
`-skipcrccheck` | Whether to skip CRC checks between source and target paths. |
-`-blocksperchunk <blocksperchunk>` | Number of blocks per chunk. When specified, split files into chunks to copy in parallel | If set to a positive value, files with more blocks than this value will be split into chunks of `<blocksperchunk>` blocks to be transferred in parallel, and reassembled on the destination. By default, `<blocksperchunk>` is 0 and the files will be transmitted in their entirety without splitting. This switch is only applicable when the source file system implements getBlockLocations method and the target file system implements concat method. |
Architecture of DistCp
----------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
index b2266b3..e3018a0 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
@@ -23,27 +23,17 @@ import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.PrintStream;
+import java.io.OutputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
-import java.util.Random;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.util.ToolRunner;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -57,15 +47,11 @@ import org.junit.rules.Timeout;
*/
public class TestDistCpSystem {
- private static final Log LOG =
- LogFactory.getLog(TestDistCpSystem.class);
-
@Rule
public Timeout globalTimeout = new Timeout(30000);
private static final String SRCDAT = "srcdat";
private static final String DSTDAT = "dstdat";
- private static final long BLOCK_SIZE = 1024;
private static MiniDFSCluster cluster;
private static Configuration conf;
@@ -77,76 +63,27 @@ public class TestDistCpSystem {
this.path = path;
this.isDir = isDir;
}
-
- String getPath() {
- return path;
- }
-
- boolean isDirectory() {
- return isDir;
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws IOException {
- conf = new Configuration();
- conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCK_SIZE);
- conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- cluster.waitActive();
- }
-
- @AfterClass
- public static void afterClass() throws IOException {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- static String execCmd(FsShell shell, String... args) throws Exception {
- ByteArrayOutputStream baout = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(baout, true);
- PrintStream old = System.out;
- System.setOut(out);
- shell.run(args);
- out.close();
- System.setOut(old);
- return baout.toString();
+ String getPath() { return path; }
+ boolean isDirectory() { return isDir; }
}
- private void createFiles(DistributedFileSystem fs, String topdir,
- FileEntry[] entries, long chunkSize) throws IOException {
- long seed = System.currentTimeMillis();
- Random rand = new Random(seed);
- short replicationFactor = 2;
+ private void createFiles(FileSystem fs, String topdir,
+ FileEntry[] entries) throws IOException {
for (FileEntry entry : entries) {
- Path newPath = new Path(topdir + "/" + entry.getPath());
+ Path newpath = new Path(topdir + "/" + entry.getPath());
if (entry.isDirectory()) {
- fs.mkdirs(newPath);
+ fs.mkdirs(newpath);
} else {
- long fileSize = BLOCK_SIZE *100;
- int bufSize = 128;
- if (chunkSize == -1) {
- DFSTestUtil.createFile(fs, newPath, bufSize,
- fileSize, BLOCK_SIZE, replicationFactor, seed);
- } else {
- // Create a variable length block file, by creating
- // one block of half block size at the chunk boundary
- long seg1 = chunkSize * BLOCK_SIZE - BLOCK_SIZE / 2;
- long seg2 = fileSize - seg1;
- DFSTestUtil.createFile(fs, newPath, bufSize,
- seg1, BLOCK_SIZE, replicationFactor, seed);
- DFSTestUtil.appendFileNewBlock(fs, newPath, (int)seg2);
+ OutputStream out = fs.create(newpath);
+ try {
+ out.write((topdir + "/" + entry).getBytes());
+ out.write("\n".getBytes());
+ } finally {
+ out.close();
}
}
- seed = System.currentTimeMillis() + rand.nextLong();
}
}
-
- private void createFiles(DistributedFileSystem fs, String topdir,
- FileEntry[] entries) throws IOException {
- createFiles(fs, topdir, entries, -1);
- }
private static FileStatus[] getFileStatus(FileSystem fs,
String topdir, FileEntry[] files) throws IOException {
@@ -167,19 +104,18 @@ public class TestDistCpSystem {
}
private void testPreserveUserHelper(String testRoot,
- FileEntry[] srcEntries,
- FileEntry[] dstEntries,
- boolean createSrcDir,
- boolean createTgtDir,
- boolean update) throws Exception {
+ FileEntry[] srcEntries,
+ FileEntry[] dstEntries,
+ boolean createSrcDir,
+ boolean createTgtDir,
+ boolean update) throws Exception {
final String testSrcRel = SRCDAT;
final String testSrc = testRoot + "/" + testSrcRel;
final String testDstRel = DSTDAT;
final String testDst = testRoot + "/" + testDstRel;
String nnUri = FileSystem.getDefaultUri(conf).toString();
- DistributedFileSystem fs = (DistributedFileSystem)
- FileSystem.get(URI.create(nnUri), conf);
+ FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
fs.mkdirs(new Path(testRoot));
if (createSrcDir) {
fs.mkdirs(new Path(testSrc));
@@ -193,8 +129,8 @@ public class TestDistCpSystem {
for(int i = 0; i < srcEntries.length; i++) {
fs.setOwner(srcstats[i].getPath(), "u" + i, null);
}
- String[] args = update? new String[]{"-pub", "-update", nnUri+testSrc,
- nnUri+testDst} : new String[]{"-pub", nnUri+testSrc, nnUri+testDst};
+ String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc,
+ nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst};
ToolRunner.run(conf, new DistCp(), args);
@@ -209,261 +145,18 @@ public class TestDistCpSystem {
deldir(fs, testRoot);
}
- private void compareFiles(FileSystem fs, FileStatus srcStat,
- FileStatus dstStat) throws Exception {
- LOG.info("Comparing " + srcStat + " and " + dstStat);
- assertEquals(srcStat.isDirectory(), dstStat.isDirectory());
- assertEquals(srcStat.getReplication(), dstStat.getReplication());
- assertEquals("File POSIX permission should match",
- srcStat.getPermission(), dstStat.getPermission());
- assertEquals("File user ownership should match",
- srcStat.getOwner(), dstStat.getOwner());
- assertEquals("File group ownership should match",
- srcStat.getGroup(), dstStat.getGroup());
- // TODO; check ACL attributes
-
- if (srcStat.isDirectory()) {
- return;
- }
-
- assertEquals("File length should match (" + srcStat.getPath() + ")",
- srcStat.getLen(), dstStat.getLen());
-
- FSDataInputStream srcIn = fs.open(srcStat.getPath());
- FSDataInputStream dstIn = fs.open(dstStat.getPath());
- try {
- byte[] readSrc = new byte[(int)
- HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT];
- byte[] readDst = new byte[(int)
- HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT];
-
- int srcBytesRead = 0, tgtBytesRead = 0;
- int srcIdx = 0, tgtIdx = 0;
- long totalComparedBytes = 0;
- while (true) {
- if (srcBytesRead == 0) {
- srcBytesRead = srcIn.read(readSrc);
- srcIdx = 0;
- }
- if (tgtBytesRead == 0) {
- tgtBytesRead = dstIn.read(readDst);
- tgtIdx = 0;
- }
- if (srcBytesRead == 0 || tgtBytesRead == 0) {
- LOG.info("______ compared src and dst files for "
- + totalComparedBytes + " bytes, content match.");
- if (srcBytesRead != tgtBytesRead) {
- Assert.fail("Read mismatching size, compared "
- + totalComparedBytes + " bytes between src and dst file "
- + srcStat + " and " + dstStat);
- }
- if (totalComparedBytes != srcStat.getLen()) {
- Assert.fail("Only read/compared " + totalComparedBytes +
- " bytes between src and dst file " + srcStat +
- " and " + dstStat);
- } else {
- // success
- break;
- }
- }
- for (; srcIdx < srcBytesRead && tgtIdx < tgtBytesRead;
- ++srcIdx, ++tgtIdx) {
- if (readSrc[srcIdx] != readDst[tgtIdx]) {
- Assert.fail("src and dst file does not match at "
- + totalComparedBytes + " between "
- + srcStat + " and " + dstStat);
- }
- ++totalComparedBytes;
- }
- LOG.info("______ compared src and dst files for "
- + totalComparedBytes + " bytes, content match. FileLength: "
- + srcStat.getLen());
- if (totalComparedBytes == srcStat.getLen()) {
- LOG.info("______ Final:" + srcIdx + " "
- + srcBytesRead + " " + tgtIdx + " " + tgtBytesRead);
- break;
- }
- if (srcIdx == srcBytesRead) {
- srcBytesRead = 0;
- }
- if (tgtIdx == tgtBytesRead) {
- tgtBytesRead = 0;
- }
- }
- } finally {
- if (srcIn != null) {
- srcIn.close();
- }
- if (dstIn != null) {
- dstIn.close();
- }
- }
- }
-
- // WC: needed because the current distcp does not create target dirs
- private void createDestDir(FileSystem fs, String testDst,
- FileStatus[] srcStats, FileEntry[] srcFiles) throws IOException {
- fs.mkdirs(new Path(testDst));
-
- for (int i=0; i<srcStats.length; i++) {
- FileStatus srcStat = srcStats[i];
- if (srcStat.isDirectory()) {
- Path dstPath = new Path(testDst, srcFiles[i].getPath());
- fs.mkdirs(dstPath);
- fs.setOwner(dstPath, srcStat.getOwner(), srcStat.getGroup());
- }
- }
- }
-
- private void copyAndVerify(final DistributedFileSystem fs,
- final FileEntry[] srcFiles, final FileStatus[] srcStats,
- final String testDst,
- final String[] args) throws Exception {
- final String testRoot = "/testdir";
- FsShell shell = new FsShell(fs.getConf());
-
- LOG.info("ls before distcp");
- LOG.info(execCmd(shell, "-lsr", testRoot));
-
- LOG.info("_____ running distcp: " + args[0] + " " + args[1]);
- ToolRunner.run(conf, new DistCp(), args);
-
- LOG.info("ls after distcp");
- LOG.info(execCmd(shell, "-lsr", testRoot));
-
- FileStatus[] dstStat = getFileStatus(fs, testDst, srcFiles);
- for (int i=0; i< dstStat.length; i++) {
- compareFiles(fs, srcStats[i], dstStat[i]);
- }
- }
-
- private void chunkCopy(FileEntry[] srcFiles) throws Exception {
- final String testRoot = "/testdir";
- final String testSrcRel = SRCDAT;
- final String testSrc = testRoot + "/" + testSrcRel;
- final String testDstRel = DSTDAT;
- final String testDst = testRoot + "/" + testDstRel;
- long chunkSize = 8;
-
- String nnUri = FileSystem.getDefaultUri(conf).toString();
- DistributedFileSystem fs = (DistributedFileSystem)
- FileSystem.get(URI.create(nnUri), conf);
-
- createFiles(fs, testRoot, srcFiles, chunkSize);
-
- FileStatus[] srcStats = getFileStatus(fs, testRoot, srcFiles);
- for (int i = 0; i < srcFiles.length; i++) {
- fs.setOwner(srcStats[i].getPath(), "u" + i, "g" + i);
- }
- // get file status after updating owners
- srcStats = getFileStatus(fs, testRoot, srcFiles);
-
- createDestDir(fs, testDst, srcStats, srcFiles);
-
- String[] args = new String[] {"-pugp", "-blocksperchunk",
- String.valueOf(chunkSize),
- nnUri + testSrc, nnUri + testDst};
-
- copyAndVerify(fs, srcFiles, srcStats, testDst, args);
- // Do it again
- copyAndVerify(fs, srcFiles, srcStats, testDst, args);
-
- // modify last file and rerun distcp with -update option
- LOG.info("Modify a file and copy again");
- for(int i=srcFiles.length-1; i >=0; --i) {
- if (!srcFiles[i].isDirectory()) {
- LOG.info("Modifying " + srcStats[i].getPath());
- DFSTestUtil.appendFileNewBlock(fs, srcStats[i].getPath(),
- (int)BLOCK_SIZE * 3);
- break;
- }
- }
- // get file status after modifying file
- srcStats = getFileStatus(fs, testRoot, srcFiles);
-
- args = new String[] {"-pugp", "-update", "-blocksperchunk",
- String.valueOf(chunkSize),
- nnUri + testSrc, nnUri + testDst + "/" + testSrcRel};
-
- copyAndVerify(fs, srcFiles, srcStats, testDst, args);
-
- deldir(fs, testRoot);
- }
-
- @Test
- public void testRecursiveChunkCopy() throws Exception {
- FileEntry[] srcFiles = {
- new FileEntry(SRCDAT, true),
- new FileEntry(SRCDAT + "/file0", false),
- new FileEntry(SRCDAT + "/dir1", true),
- new FileEntry(SRCDAT + "/dir2", true),
- new FileEntry(SRCDAT + "/dir1/file1", false)
- };
- chunkCopy(srcFiles);
- }
-
- @Test
- public void testChunkCopyOneFile() throws Exception {
- FileEntry[] srcFiles = {
- new FileEntry(SRCDAT, true),
- new FileEntry(SRCDAT + "/file0", false)
- };
- chunkCopy(srcFiles);
+ @BeforeClass
+ public static void beforeClass() throws IOException {
+ conf = new Configuration();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+ cluster.waitActive();
}
- @Test
- public void testDistcpLargeFile() throws Exception {
- FileEntry[] srcfiles = {
- new FileEntry(SRCDAT, true),
- new FileEntry(SRCDAT + "/file", false)
- };
-
- final String testRoot = "/testdir";
- final String testSrcRel = SRCDAT;
- final String testSrc = testRoot + "/" + testSrcRel;
- final String testDstRel = DSTDAT;
- final String testDst = testRoot + "/" + testDstRel;
-
- String nnUri = FileSystem.getDefaultUri(conf).toString();
- DistributedFileSystem fs =
- (DistributedFileSystem) FileSystem.get(URI.create(nnUri), conf);
- fs.mkdirs(new Path(testRoot));
- fs.mkdirs(new Path(testSrc));
- fs.mkdirs(new Path(testDst));
- long chunkSize = 6;
- createFiles(fs, testRoot, srcfiles, chunkSize);
-
- String srcFileName = testRoot + Path.SEPARATOR + srcfiles[1].getPath();
- Path srcfile = new Path(srcFileName);
-
- if(!cluster.getFileSystem().exists(srcfile)){
- throw new Exception("src not exist");
- }
-
- final long srcLen = fs.getFileStatus(srcfile).getLen();
-
- FileStatus[] srcstats = getFileStatus(fs, testRoot, srcfiles);
- for (int i = 0; i < srcfiles.length; i++) {
- fs.setOwner(srcstats[i].getPath(), "u" + i, null);
+ @AfterClass
+ public static void afterClass() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
}
- String[] args = new String[] {
- "-blocksperchunk",
- String.valueOf(chunkSize),
- nnUri + testSrc,
- nnUri + testDst
- };
-
- LOG.info("_____ running distcp: " + args[0] + " " + args[1]);
- ToolRunner.run(conf, new DistCp(), args);
-
- String realTgtPath = testDst;
- FileStatus[] dststat = getFileStatus(fs, realTgtPath, srcfiles);
- assertEquals("File length should match", srcLen,
- dststat[dststat.length - 1].getLen());
-
- this.compareFiles(fs, srcstats[srcstats.length-1],
- dststat[dststat.length-1]);
- deldir(fs, testRoot);
}
@Test
@@ -487,6 +180,7 @@ public class TestDistCpSystem {
testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false);
}
+
@Test
public void testPreserveUserEmptyDir() throws Exception {
String testRoot = "/testdir." + getMethodName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index f94ba97..efe4627 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -394,7 +394,7 @@ public class TestOptionsParser {
+ "copyStrategy='uniformsize', preserveStatus=[], "
+ "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, "
+ "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, "
- + "targetPathExists=true, filtersFile='null', blocksPerChunk=0}";
+ + "targetPathExists=true, filtersFile='null'}";
String optionString = option.toString();
Assert.assertEquals(val, optionString);
Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f1cf7/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
index 2452d6f..2e9a350 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
@@ -81,10 +81,6 @@ public class TestCopyCommitter {
@Before
public void createMetaFolder() {
config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta");
- // Unset listing file path since the config is shared by
- // multiple tests, and some test doesn't set it, such as
- // testNoCommitAction, but the distcp code will check it.
- config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, "");
Path meta = new Path("/meta");
try {
cluster.getFileSystem().mkdirs(meta);
@@ -330,6 +326,7 @@ public class TestCopyCommitter {
committer.commitJob(jobContext);
Assert.assertFalse(fs.exists(new Path(workPath)));
Assert.assertTrue(fs.exists(new Path(finalPath)));
+
} catch (IOException e) {
LOG.error("Exception encountered while testing for preserve status", e);
Assert.fail("Atomic commit failure");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDFS-10971. Distcp should not copy
replication factor if source file is erasure coded. Contributed by Manoj
Govindassamy.
Posted by ae...@apache.org.
HDFS-10971. Distcp should not copy replication factor if source file is erasure coded. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e6f8e4b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e6f8e4b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e6f8e4b
Branch: refs/heads/HDFS-7240
Commit: 0e6f8e4bc6642f90dc7b33848bfb1129ec20ee49
Parents: 82fb9ce
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Mar 28 22:14:03 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Mar 28 22:14:03 2017 -0700
----------------------------------------------------------------------
.../hadoop/tools/CopyListingFileStatus.java | 4 +
.../tools/mapred/RetriableFileCopyCommand.java | 3 +
.../apache/hadoop/tools/util/DistCpUtils.java | 9 +-
.../hadoop/tools/TestCopyListingFileStatus.java | 1 +
.../hadoop/tools/util/TestDistCpUtils.java | 123 ++++++++++++++++++-
5 files changed, 137 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6f8e4b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
index 2b1e7e4..00d4b32 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
@@ -159,6 +159,10 @@ public final class CopyListingFileStatus implements Writable {
return permission;
}
+ public boolean isErasureCoded() {
+ return getPermission().getErasureCodedBit();
+ }
+
/**
* Returns the full logical ACL.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6f8e4b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
index d1cdfdd..06acd78 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
@@ -167,6 +167,9 @@ public class RetriableFileCopyCommand extends RetriableCommand {
FsPermission.getUMask(targetFS.getConf()));
final OutputStream outStream;
if (action == FileAction.OVERWRITE) {
+ // If there is an erasure coding policy set on the target directory,
+ // files will be written to the target directory using the same EC policy.
+ // The replication factor of the source file is ignored and not preserved.
final short repl = getReplicationFactor(fileAttributes, source,
targetFS, targetPath);
final long blockSize = getBlockSize(fileAttributes, source,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6f8e4b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
index c308e6f..76bc4c5 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
@@ -236,8 +236,13 @@ public class DistCpUtils {
}
}
- if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() &&
- (srcFileStatus.getReplication() != targetFileStatus.getReplication())) {
+ // The replication factor can only be preserved for replicated files.
+ // It is ignored when either the source or target file are erasure coded.
+ if (attributes.contains(FileAttribute.REPLICATION) &&
+ !targetFileStatus.isDirectory() &&
+ !targetFileStatus.isErasureCoded() &&
+ !srcFileStatus.isErasureCoded() &&
+ srcFileStatus.getReplication() != targetFileStatus.getReplication()) {
targetFS.setReplication(path, srcFileStatus.getReplication());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6f8e4b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java
index f512ef6..8efc5cf 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java
@@ -62,6 +62,7 @@ public class TestCopyListingFileStatus {
assertEquals(stat.getOwner(), clfs.getOwner());
assertEquals(stat.getGroup(), clfs.getGroup());
assertEquals(stat.getPath(), clfs.getPath());
+ assertEquals(stat.isErasureCoded(), clfs.isErasureCoded());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6f8e4b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java
index 8c79bec..c42e546 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.tools.util;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
import java.io.IOException;
import java.io.OutputStream;
import java.util.EnumSet;
@@ -31,11 +35,15 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.tools.ECAdmin;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.tools.CopyListingFileStatus;
import org.apache.hadoop.tools.DistCpOptionSwitch;
import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
+import org.apache.hadoop.util.ToolRunner;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -52,8 +60,10 @@ public class TestDistCpUtils {
@BeforeClass
public static void create() throws IOException {
+ config.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
+ "XOR-2-1-64k");
cluster = new MiniDFSCluster.Builder(config)
- .numDataNodes(1)
+ .numDataNodes(2)
.format(true)
.build();
}
@@ -539,6 +549,117 @@ public class TestDistCpUtils {
Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication());
}
+ @Test (timeout = 60000)
+ public void testReplFactorNotPreservedOnErasureCodedFile() throws Exception {
+ FileSystem fs = FileSystem.get(config);
+
+ // Case 1: Verify replication attribute not preserved when the source
+ // file is erasure coded and the target file is replicated.
+ Path srcECDir = new Path("/tmp/srcECDir");
+ Path srcECFile = new Path(srcECDir, "srcECFile");
+ Path dstReplDir = new Path("/tmp/dstReplDir");
+ Path dstReplFile = new Path(dstReplDir, "destReplFile");
+ fs.mkdirs(srcECDir);
+ fs.mkdirs(dstReplDir);
+ String[] args = {"-setPolicy", "-path", "/tmp/srcECDir",
+ "-policy", "XOR-2-1-64k"};
+ int res = ToolRunner.run(config, new ECAdmin(config), args);
+ assertEquals("Setting EC policy should succeed!", 0, res);
+ verifyReplFactorNotPreservedOnErasureCodedFile(srcECFile, true,
+ dstReplFile, false);
+
+ // Case 2: Verify replication attribute not preserved when the source
+ // file is replicated and the target file is erasure coded.
+ Path srcReplDir = new Path("/tmp/srcReplDir");
+ Path srcReplFile = new Path(srcReplDir, "srcReplFile");
+ Path dstECDir = new Path("/tmp/dstECDir");
+ Path dstECFile = new Path(dstECDir, "destECFile");
+ fs.mkdirs(srcReplDir);
+ fs.mkdirs(dstECDir);
+ args = new String[]{"-setPolicy", "-path", "/tmp/dstECDir",
+ "-policy", "XOR-2-1-64k"};
+ res = ToolRunner.run(config, new ECAdmin(config), args);
+ assertEquals("Setting EC policy should succeed!", 0, res);
+ verifyReplFactorNotPreservedOnErasureCodedFile(srcReplFile,
+ false, dstECFile, true);
+
+ // Case 3: Verify replication attribute not altered from the default
+ // INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS when both source and
+ // target files are erasure coded.
+ verifyReplFactorNotPreservedOnErasureCodedFile(srcECFile,
+ true, dstECFile, true);
+ }
+
+ private void verifyReplFactorNotPreservedOnErasureCodedFile(Path srcFile,
+ boolean isSrcEC, Path dstFile, boolean isDstEC) throws Exception {
+ FileSystem fs = FileSystem.get(config);
+ createFile(fs, srcFile);
+ CopyListingFileStatus srcStatus = new CopyListingFileStatus(
+ fs.getFileStatus(srcFile));
+ if (isSrcEC) {
+ assertTrue(srcFile + "should be erasure coded!",
+ srcStatus.isErasureCoded());
+ assertEquals(INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS,
+ srcStatus.getReplication());
+ } else {
+ assertEquals("Unexpected replication factor for " + srcFile,
+ fs.getDefaultReplication(srcFile), srcStatus.getReplication());
+ }
+
+ createFile(fs, dstFile);
+ CopyListingFileStatus dstStatus = new CopyListingFileStatus(
+ fs.getFileStatus(dstFile));
+ if (isDstEC) {
+ assertTrue(dstFile + "should be erasure coded!",
+ dstStatus.isErasureCoded());
+ assertEquals("Unexpected replication factor for erasure coded file!",
+ INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS,
+ dstStatus.getReplication());
+ } else {
+ assertEquals("Unexpected replication factor for " + dstFile,
+ fs.getDefaultReplication(dstFile), dstStatus.getReplication());
+ }
+
+ // Let srcFile and dstFile differ on their FileAttribute
+ fs.setPermission(srcFile, fullPerm);
+ fs.setOwner(srcFile, "ec", "ec-group");
+ fs.setTimes(srcFile, 0, 0);
+
+ fs.setPermission(dstFile, noPerm);
+ fs.setOwner(dstFile, "normal", "normal-group");
+ fs.setTimes(dstFile, 100, 100);
+
+ // Running preserve operations only for replication attribute
+ srcStatus = new CopyListingFileStatus(fs.getFileStatus(srcFile));
+ EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION);
+ DistCpUtils.preserve(fs, dstFile, srcStatus, attributes, false);
+ dstStatus = new CopyListingFileStatus(fs.getFileStatus(dstFile));
+
+ assertFalse("Permission for " + srcFile + " and " + dstFile +
+ " should not be same after preserve only for replication attr!",
+ srcStatus.getPermission().equals(dstStatus.getPermission()));
+ assertFalse("File ownership should not match!",
+ srcStatus.getOwner().equals(dstStatus.getOwner()));
+ assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+ assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
+ assertFalse(
+ srcStatus.getModificationTime() == dstStatus.getModificationTime());
+ if (isDstEC) {
+ assertEquals("Unexpected replication factor for erasure coded file!",
+ INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS,
+ dstStatus.getReplication());
+ } else {
+ assertEquals(dstFile + " replication factor should be same as dst " +
+ "filesystem!", fs.getDefaultReplication(dstFile),
+ dstStatus.getReplication());
+ }
+ if (!isSrcEC || !isDstEC) {
+ assertFalse(dstFile + " replication factor should not be " +
+ "same as " + srcFile,
+ srcStatus.getReplication() == dstStatus.getReplication());
+ }
+ }
+
@Test
public void testPreserveTimestampOnFile() throws IOException {
FileSystem fs = FileSystem.get(config);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-5368. Memory leak in timeline
server (Jonathan Eagles via Varun Saxena)
Posted by ae...@apache.org.
YARN-5368. Memory leak in timeline server (Jonathan Eagles via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01aca54a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01aca54a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01aca54a
Branch: refs/heads/HDFS-7240
Commit: 01aca54a22c8586d232a8f79fe9977aeb8d09b83
Parents: 6b09336
Author: Varun Saxena <va...@apache.org>
Authored: Wed Mar 29 01:53:20 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Wed Mar 29 01:53:20 2017 +0530
----------------------------------------------------------------------
.../timeline/RollingLevelDBTimelineStore.java | 263 +++++++++----------
1 file changed, 117 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01aca54a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java
index 4d38008..20e0379 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java
@@ -275,9 +275,7 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
Path domainDBPath = new Path(dbPath, DOMAIN);
Path starttimeDBPath = new Path(dbPath, STARTTIME);
Path ownerDBPath = new Path(dbPath, OWNER);
- FileSystem localFS = null;
- try {
- localFS = FileSystem.getLocal(conf);
+ try (FileSystem localFS = FileSystem.getLocal(conf)) {
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb "
@@ -306,8 +304,6 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
}
localFS.setPermission(ownerDBPath, LEVELDB_DIR_UMASK);
}
- } finally {
- IOUtils.cleanup(LOG, localFS);
}
options.maxOpenFiles(conf.getInt(
TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES,
@@ -408,19 +404,15 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
.add(writeReverseOrderedLong(revStartTime)).add(entityId)
.getBytesForLookup();
- DBIterator iterator = null;
- try {
- DB db = entitydb.getDBForStartTime(revStartTime);
- if (db == null) {
- return null;
- }
- iterator = db.iterator();
+ DB db = entitydb.getDBForStartTime(revStartTime);
+ if (db == null) {
+ return null;
+ }
+ try (DBIterator iterator = db.iterator()) {
iterator.seek(prefix);
return getEntity(entityId, entityType, revStartTime, fields, iterator,
prefix, prefix.length);
- } finally {
- IOUtils.cleanup(LOG, iterator);
}
}
@@ -533,62 +525,61 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
o2.length);
}
});
- DBIterator iterator = null;
- try {
+
// look up start times for the specified entities
// skip entities with no start time
- for (String entityId : entityIds) {
- byte[] startTime = getStartTime(entityId, entityType);
- if (startTime != null) {
- List<EntityIdentifier> entities = startTimeMap.get(startTime);
- if (entities == null) {
- entities = new ArrayList<EntityIdentifier>();
- startTimeMap.put(startTime, entities);
- }
- entities.add(new EntityIdentifier(entityId, entityType));
+ for (String entityId : entityIds) {
+ byte[] startTime = getStartTime(entityId, entityType);
+ if (startTime != null) {
+ List<EntityIdentifier> entities = startTimeMap.get(startTime);
+ if (entities == null) {
+ entities = new ArrayList<EntityIdentifier>();
+ startTimeMap.put(startTime, entities);
}
+ entities.add(new EntityIdentifier(entityId, entityType));
}
- for (Entry<byte[], List<EntityIdentifier>> entry : startTimeMap
+ }
+ for (Entry<byte[], List<EntityIdentifier>> entry : startTimeMap
.entrySet()) {
- // look up the events matching the given parameters (limit,
- // start time, end time, event types) for entities whose start times
- // were found and add the entities to the return list
- byte[] revStartTime = entry.getKey();
- for (EntityIdentifier entityIdentifier : entry.getValue()) {
- EventsOfOneEntity entity = new EventsOfOneEntity();
- entity.setEntityId(entityIdentifier.getId());
- entity.setEntityType(entityType);
- events.addEvent(entity);
- KeyBuilder kb = KeyBuilder.newInstance().add(entityType)
- .add(revStartTime).add(entityIdentifier.getId())
- .add(EVENTS_COLUMN);
- byte[] prefix = kb.getBytesForLookup();
- if (windowEnd == null) {
- windowEnd = Long.MAX_VALUE;
- }
- byte[] revts = writeReverseOrderedLong(windowEnd);
- kb.add(revts);
- byte[] first = kb.getBytesForLookup();
- byte[] last = null;
- if (windowStart != null) {
- last = KeyBuilder.newInstance().add(prefix)
- .add(writeReverseOrderedLong(windowStart)).getBytesForLookup();
- }
- if (limit == null) {
- limit = DEFAULT_LIMIT;
- }
- DB db = entitydb.getDBForStartTime(readReverseOrderedLong(
- revStartTime, 0));
- if (db == null) {
- continue;
- }
- iterator = db.iterator();
+ // look up the events matching the given parameters (limit,
+ // start time, end time, event types) for entities whose start times
+ // were found and add the entities to the return list
+ byte[] revStartTime = entry.getKey();
+ for (EntityIdentifier entityIdentifier : entry.getValue()) {
+ EventsOfOneEntity entity = new EventsOfOneEntity();
+ entity.setEntityId(entityIdentifier.getId());
+ entity.setEntityType(entityType);
+ events.addEvent(entity);
+ KeyBuilder kb = KeyBuilder.newInstance().add(entityType)
+ .add(revStartTime).add(entityIdentifier.getId())
+ .add(EVENTS_COLUMN);
+ byte[] prefix = kb.getBytesForLookup();
+ if (windowEnd == null) {
+ windowEnd = Long.MAX_VALUE;
+ }
+ byte[] revts = writeReverseOrderedLong(windowEnd);
+ kb.add(revts);
+ byte[] first = kb.getBytesForLookup();
+ byte[] last = null;
+ if (windowStart != null) {
+ last = KeyBuilder.newInstance().add(prefix)
+ .add(writeReverseOrderedLong(windowStart)).getBytesForLookup();
+ }
+ if (limit == null) {
+ limit = DEFAULT_LIMIT;
+ }
+ DB db = entitydb.getDBForStartTime(readReverseOrderedLong(
+ revStartTime, 0));
+ if (db == null) {
+ continue;
+ }
+ try (DBIterator iterator = db.iterator()) {
for (iterator.seek(first); entity.getEvents().size() < limit
&& iterator.hasNext(); iterator.next()) {
byte[] key = iterator.peekNext().getKey();
if (!prefixMatches(prefix, prefix.length, key)
|| (last != null && WritableComparator.compareBytes(key, 0,
- key.length, last, 0, last.length) > 0)) {
+ key.length, last, 0, last.length) > 0)) {
break;
}
TimelineEvent event = getEntityEvent(eventType, key, prefix.length,
@@ -599,8 +590,6 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
}
}
}
- } finally {
- IOUtils.cleanup(LOG, iterator);
}
return events;
}
@@ -657,66 +646,64 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
Long limit, Long starttime, Long endtime, String fromId, Long fromTs,
Collection<NameValuePair> secondaryFilters, EnumSet<Field> fields,
CheckAcl checkAcl, boolean usingPrimaryFilter) throws IOException {
- DBIterator iterator = null;
- try {
- KeyBuilder kb = KeyBuilder.newInstance().add(base).add(entityType);
- // only db keys matching the prefix (base + entity type) will be parsed
- byte[] prefix = kb.getBytesForLookup();
- if (endtime == null) {
- // if end time is null, place no restriction on end time
- endtime = Long.MAX_VALUE;
- }
+ KeyBuilder kb = KeyBuilder.newInstance().add(base).add(entityType);
+ // only db keys matching the prefix (base + entity type) will be parsed
+ byte[] prefix = kb.getBytesForLookup();
+ if (endtime == null) {
+ // if end time is null, place no restriction on end time
+ endtime = Long.MAX_VALUE;
+ }
- // Sanitize the fields parameter
- if (fields == null) {
- fields = EnumSet.allOf(Field.class);
- }
+ // Sanitize the fields parameter
+ if (fields == null) {
+ fields = EnumSet.allOf(Field.class);
+ }
- // construct a first key that will be seeked to using end time or fromId
- long firstStartTime = Long.MAX_VALUE;
- byte[] first = null;
- if (fromId != null) {
- Long fromIdStartTime = getStartTimeLong(fromId, entityType);
- if (fromIdStartTime == null) {
- // no start time for provided id, so return empty entities
- return new TimelineEntities();
- }
- if (fromIdStartTime <= endtime) {
- // if provided id's start time falls before the end of the window,
- // use it to construct the seek key
- firstStartTime = fromIdStartTime;
- first = kb.add(writeReverseOrderedLong(fromIdStartTime)).add(fromId)
- .getBytesForLookup();
- }
- }
- // if seek key wasn't constructed using fromId, construct it using end ts
- if (first == null) {
- firstStartTime = endtime;
- first = kb.add(writeReverseOrderedLong(endtime)).getBytesForLookup();
+ // construct a first key that will be seeked to using end time or fromId
+ long firstStartTime = Long.MAX_VALUE;
+ byte[] first = null;
+ if (fromId != null) {
+ Long fromIdStartTime = getStartTimeLong(fromId, entityType);
+ if (fromIdStartTime == null) {
+ // no start time for provided id, so return empty entities
+ return new TimelineEntities();
}
- byte[] last = null;
- if (starttime != null) {
- // if start time is not null, set a last key that will not be
- // iterated past
- last = KeyBuilder.newInstance().add(base).add(entityType)
- .add(writeReverseOrderedLong(starttime)).getBytesForLookup();
- }
- if (limit == null) {
- // if limit is not specified, use the default
- limit = DEFAULT_LIMIT;
+ if (fromIdStartTime <= endtime) {
+ // if provided id's start time falls before the end of the window,
+ // use it to construct the seek key
+ firstStartTime = fromIdStartTime;
+ first = kb.add(writeReverseOrderedLong(fromIdStartTime)).add(fromId)
+ .getBytesForLookup();
}
+ }
+ // if seek key wasn't constructed using fromId, construct it using end ts
+ if (first == null) {
+ firstStartTime = endtime;
+ first = kb.add(writeReverseOrderedLong(endtime)).getBytesForLookup();
+ }
+ byte[] last = null;
+ if (starttime != null) {
+ // if start time is not null, set a last key that will not be
+ // iterated past
+ last = KeyBuilder.newInstance().add(base).add(entityType)
+ .add(writeReverseOrderedLong(starttime)).getBytesForLookup();
+ }
+ if (limit == null) {
+ // if limit is not specified, use the default
+ limit = DEFAULT_LIMIT;
+ }
- TimelineEntities entities = new TimelineEntities();
- RollingLevelDB rollingdb = null;
- if (usingPrimaryFilter) {
- rollingdb = indexdb;
- } else {
- rollingdb = entitydb;
- }
+ TimelineEntities entities = new TimelineEntities();
+ RollingLevelDB rollingdb = null;
+ if (usingPrimaryFilter) {
+ rollingdb = indexdb;
+ } else {
+ rollingdb = entitydb;
+ }
- DB db = rollingdb.getDBForStartTime(firstStartTime);
- while (entities.getEntities().size() < limit && db != null) {
- iterator = db.iterator();
+ DB db = rollingdb.getDBForStartTime(firstStartTime);
+ while (entities.getEntities().size() < limit && db != null) {
+ try (DBIterator iterator = db.iterator()) {
iterator.seek(first);
// iterate until one of the following conditions is met: limit is
@@ -726,7 +713,7 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
byte[] key = iterator.peekNext().getKey();
if (!prefixMatches(prefix, prefix.length, key)
|| (last != null && WritableComparator.compareBytes(key, 0,
- key.length, last, 0, last.length) > 0)) {
+ key.length, last, 0, last.length) > 0)) {
break;
}
// read the start time and entity id from the current key
@@ -814,10 +801,8 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
}
db = rollingdb.getPreviousDB(db);
}
- return entities;
- } finally {
- IOUtils.cleanup(LOG, iterator);
}
+ return entities;
}
/**
@@ -1459,15 +1444,14 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
long startTimesCount = 0;
WriteBatch writeBatch = null;
- DBIterator iterator = null;
- try {
- writeBatch = starttimedb.createWriteBatch();
- ReadOptions readOptions = new ReadOptions();
- readOptions.fillCache(false);
- iterator = starttimedb.iterator(readOptions);
+ ReadOptions readOptions = new ReadOptions();
+ readOptions.fillCache(false);
+ try (DBIterator iterator = starttimedb.iterator(readOptions)) {
+
// seek to the first start time entry
iterator.seekToFirst();
+ writeBatch = starttimedb.createWriteBatch();
// evaluate each start time entry to see if it needs to be evicted or not
while (iterator.hasNext()) {
@@ -1513,7 +1497,6 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
+ " start time entities earlier than " + minStartTime);
} finally {
IOUtils.cleanup(LOG, writeBatch);
- IOUtils.cleanup(LOG, iterator);
}
return startTimesCount;
}
@@ -1598,11 +1581,9 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
// TODO: make data retention work with the domain data as well
@Override
public void put(TimelineDomain domain) throws IOException {
- WriteBatch domainWriteBatch = null;
- WriteBatch ownerWriteBatch = null;
- try {
- domainWriteBatch = domaindb.createWriteBatch();
- ownerWriteBatch = ownerdb.createWriteBatch();
+ try (WriteBatch domainWriteBatch = domaindb.createWriteBatch();
+ WriteBatch ownerWriteBatch = ownerdb.createWriteBatch();) {
+
if (domain.getId() == null || domain.getId().length() == 0) {
throw new IllegalArgumentException("Domain doesn't have an ID");
}
@@ -1682,9 +1663,6 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
ownerWriteBatch.put(ownerLookupEntryKey, timestamps);
domaindb.write(domainWriteBatch);
ownerdb.write(ownerWriteBatch);
- } finally {
- IOUtils.cleanup(LOG, domainWriteBatch);
- IOUtils.cleanup(LOG, ownerWriteBatch);
}
}
@@ -1709,26 +1687,21 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
@Override
public TimelineDomain getDomain(String domainId) throws IOException {
- DBIterator iterator = null;
- try {
+ try (DBIterator iterator = domaindb.iterator()) {
byte[] prefix = KeyBuilder.newInstance().add(domainId)
.getBytesForLookup();
- iterator = domaindb.iterator();
iterator.seek(prefix);
return getTimelineDomain(iterator, domainId, prefix);
- } finally {
- IOUtils.cleanup(LOG, iterator);
}
}
@Override
public TimelineDomains getDomains(String owner) throws IOException {
- DBIterator iterator = null;
- try {
+ try (DBIterator iterator = ownerdb.iterator()) {
byte[] prefix = KeyBuilder.newInstance().add(owner).getBytesForLookup();
+ iterator.seek(prefix);
List<TimelineDomain> domains = new ArrayList<TimelineDomain>();
- for (iterator = ownerdb.iterator(), iterator.seek(prefix); iterator
- .hasNext();) {
+ while (iterator.hasNext()) {
byte[] key = iterator.peekNext().getKey();
if (!prefixMatches(prefix, prefix.length, key)) {
break;
@@ -1761,8 +1734,6 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
TimelineDomains domainsToReturn = new TimelineDomains();
domainsToReturn.addDomains(domains);
return domainsToReturn;
- } finally {
- IOUtils.cleanup(LOG, iterator);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HDFS-11577. Combine the old and
the new chooseRandom for better performance. Contributed by Chen Liang.
Posted by ae...@apache.org.
HDFS-11577. Combine the old and the new chooseRandom for better performance. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b093364
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b093364
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b093364
Branch: refs/heads/HDFS-7240
Commit: 6b0933643835d7696ced011cfdb8b74f63022e8b
Parents: fdf8f8e
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Mar 28 23:02:07 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Tue Mar 28 23:02:07 2017 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/net/NetworkTopology.java | 2 +-
.../hadoop/hdfs/net/DFSNetworkTopology.java | 76 +++++++++++++++++---
.../hadoop/hdfs/net/TestDFSNetworkTopology.java | 57 +++++++++++++++
3 files changed, 126 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b093364/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 051012b..f8cecdd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -496,7 +496,7 @@ public class NetworkTopology {
}
}
- private Node chooseRandom(final String scope, String excludedScope,
+ protected Node chooseRandom(final String scope, String excludedScope,
final Collection<Node> excludedNodes) {
if (excludedScope != null) {
if (scope.startsWith(excludedScope)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b093364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
index ee83dba..259e275 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
@@ -36,7 +36,6 @@ import java.util.Random;
* remaining parts should be the same.
*
* Currently a placeholder to test storage type info.
- * TODO : add "chooseRandom with storageType info" function.
*/
public class DFSNetworkTopology extends NetworkTopology {
@@ -56,6 +55,7 @@ public class DFSNetworkTopology extends NetworkTopology {
*
* @param scope range of nodes from which a node will be chosen
* @param excludedNodes nodes to be excluded from
+ * @param type the storage type we search for
* @return the chosen node
*/
public Node chooseRandomWithStorageType(final String scope,
@@ -75,6 +75,69 @@ public class DFSNetworkTopology extends NetworkTopology {
}
/**
+ * Randomly choose one node from <i>scope</i> with the given storage type.
+ *
+ * If scope starts with ~, choose one from the all nodes except for the
+ * ones in <i>scope</i>; otherwise, choose one from <i>scope</i>.
+ * If excludedNodes is given, choose a node that's not in excludedNodes.
+ *
+ * This call would make up to two calls. It first tries to get a random node
+ * (with old method) and check if it satisfies. If yes, simply return it.
+ * Otherwise, it make a second call (with the new method) by passing in a
+ * storage type.
+ *
+ * This is for better performance reason. Put in short, the key note is that
+ * the old method is faster but may take several runs, while the new method
+ * is somewhat slower, and always succeed in one trial.
+ * See HDFS-11535 for more detail.
+ *
+ * @param scope range of nodes from which a node will be chosen
+ * @param excludedNodes nodes to be excluded from
+ * @param type the storage type we search for
+ * @return the chosen node
+ */
+ public Node chooseRandomWithStorageTypeTwoTrial(final String scope,
+ final Collection<Node> excludedNodes, StorageType type) {
+ netlock.readLock().lock();
+ try {
+ String searchScope;
+ String excludedScope;
+ if (scope.startsWith("~")) {
+ searchScope = NodeBase.ROOT;
+ excludedScope = scope.substring(1);
+ } else {
+ searchScope = scope;
+ excludedScope = null;
+ }
+ // next do a two-trial search
+ // first trial, call the old method, inherited from NetworkTopology
+ Node n = chooseRandom(searchScope, excludedScope, excludedNodes);
+ if (n == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No node to choose.");
+ }
+ // this means there is simply no node to choose from
+ return null;
+ }
+ Preconditions.checkArgument(n instanceof DatanodeDescriptor);
+ DatanodeDescriptor dnDescriptor = (DatanodeDescriptor)n;
+
+ if (dnDescriptor.hasStorageType(type)) {
+ // the first trial succeeded, just return
+ return dnDescriptor;
+ } else {
+ // otherwise, make the second trial by calling the new method
+ LOG.debug("First trial failed, node has no type {}, " +
+ "making second trial carrying this type", type);
+ return chooseRandomWithStorageType(searchScope, excludedScope,
+ excludedNodes, type);
+ }
+ } finally {
+ netlock.readLock().unlock();
+ }
+ }
+
+ /**
* Choose a random node based on given scope, excludedScope and excludedNodes
* set. Although in general the topology has at most three layers, this class
* will not impose such assumption.
@@ -99,13 +162,10 @@ public class DFSNetworkTopology extends NetworkTopology {
* all it's ancestors' storage counters accordingly, this way the excluded
* root is out of the picture.
*
- * TODO : this function has duplicate code as NetworkTopology, need to
- * refactor in the future.
- *
- * @param scope
- * @param excludedScope
- * @param excludedNodes
- * @return
+ * @param scope the scope where we look for node.
+ * @param excludedScope the scope where the node must NOT be from.
+ * @param excludedNodes the returned node must not be in this set
+ * @return a node with required storage type
*/
@VisibleForTesting
Node chooseRandomWithStorageType(final String scope,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b093364/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
index 30ef2ac..26d96b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
@@ -508,4 +508,61 @@ public class TestDFSNetworkTopology {
assertEquals(1,
innerl2d3r3.getSubtreeStorageCount(StorageType.DISK));
}
+
+ @Test
+ public void testChooseRandomWithStorageTypeTwoTrial() throws Exception {
+ Node n;
+ DatanodeDescriptor dd;
+ n = CLUSTER.chooseRandomWithStorageType("/l2/d3/r4", null, null,
+ StorageType.ARCHIVE);
+ HashSet<Node> excluded = new HashSet<>();
+ // exclude the host on r4 (since there is only one host, no randomness here)
+ excluded.add(n);
+
+ // search with given scope being desired scope
+ for (int i = 0; i<10; i++) {
+ n = CLUSTER.chooseRandomWithStorageTypeTwoTrial(
+ "/l2/d3", null, StorageType.ARCHIVE);
+ assertTrue(n instanceof DatanodeDescriptor);
+ dd = (DatanodeDescriptor) n;
+ assertTrue(dd.getHostName().equals("host12") ||
+ dd.getHostName().equals("host13"));
+ }
+
+ for (int i = 0; i<10; i++) {
+ n = CLUSTER.chooseRandomWithStorageTypeTwoTrial(
+ "/l2/d3", excluded, StorageType.ARCHIVE);
+ assertTrue(n instanceof DatanodeDescriptor);
+ dd = (DatanodeDescriptor) n;
+ assertTrue(dd.getHostName().equals("host13"));
+ }
+
+ // search with given scope being exclude scope
+
+ // a total of 4 ramdisk nodes:
+ // /l1/d2/r3/host7, /l2/d3/r2/host10, /l2/d4/r1/host7 and /l2/d4/r1/host10
+ // so if we exclude /l2/d4/r1, if should be always either host7 or host10
+ for (int i = 0; i<10; i++) {
+ n = CLUSTER.chooseRandomWithStorageTypeTwoTrial(
+ "~/l2/d4", null, StorageType.RAM_DISK);
+ assertTrue(n instanceof DatanodeDescriptor);
+ dd = (DatanodeDescriptor) n;
+ assertTrue(dd.getHostName().equals("host7") ||
+ dd.getHostName().equals("host10"));
+ }
+
+ // similar to above, except that we also exclude host10 here. so it should
+ // always be host7
+ n = CLUSTER.chooseRandomWithStorageType("/l2/d3/r2", null, null,
+ StorageType.RAM_DISK);
+ // add host10 to exclude
+ excluded.add(n);
+ for (int i = 0; i<10; i++) {
+ n = CLUSTER.chooseRandomWithStorageTypeTwoTrial(
+ "~/l2/d4", excluded, StorageType.RAM_DISK);
+ assertTrue(n instanceof DatanodeDescriptor);
+ dd = (DatanodeDescriptor) n;
+ assertTrue(dd.getHostName().equals("host7"));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: MAPREDUCE-6873. MR Job Submission
Fails if MR framework application path not on defaultFS. Contributed by Erik
Krogen.
Posted by ae...@apache.org.
MAPREDUCE-6873. MR Job Submission Fails if MR framework application path not on defaultFS. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de7efd26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de7efd26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de7efd26
Branch: refs/heads/HDFS-7240
Commit: de7efd2687ccb74608249abf4c83c84a9345ec0a
Parents: ceab00a
Author: Zhe Zhang <zh...@apache.org>
Authored: Wed Mar 29 22:10:55 2017 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Wed Mar 29 22:10:55 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de7efd26/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
index 09edd94..6ade376 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
@@ -458,7 +458,7 @@ class JobSubmitter {
// resolve any symlinks in the URI path so using a "current" symlink
// to point to a specific version shows the specific version
// in the distributed cache configuration
- FileSystem fs = FileSystem.get(conf);
+ FileSystem fs = FileSystem.get(uri, conf);
Path frameworkPath = fs.makeQualified(
new Path(uri.getScheme(), uri.getAuthority(), uri.getPath()));
FileContext fc = FileContext.getFileContext(frameworkPath.toUri(), conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HADOOP-14250. Correct spelling of
'separate' and variants. Contributed by Doris Gu
Posted by ae...@apache.org.
HADOOP-14250. Correct spelling of 'separate' and variants. Contributed by Doris Gu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7297e82c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7297e82c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7297e82c
Branch: refs/heads/HDFS-7240
Commit: 7297e82cb4ab1dff06da48acdb76e3d568f998f2
Parents: c8bd5fc
Author: Chris Douglas <cd...@apache.org>
Authored: Thu Mar 30 10:14:22 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Thu Mar 30 10:17:11 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/net/NodeBase.java | 2 +-
.../main/java/org/apache/hadoop/util/StringUtils.java | 8 ++++----
.../org/apache/hadoop/fs/http/server/FSOperations.java | 2 +-
.../hadoop/mapred/lib/FieldSelectionMapReduce.java | 2 +-
.../hadoop/mapred/lib/KeyFieldBasedComparator.java | 2 +-
.../java/org/apache/hadoop/mapreduce/MRJobConfig.java | 8 +++++++-
.../mapreduce/lib/fieldsel/FieldSelectionHelper.java | 7 ++++++-
.../mapreduce/lib/fieldsel/FieldSelectionMapper.java | 2 +-
.../mapreduce/lib/fieldsel/FieldSelectionReducer.java | 2 +-
.../mapreduce/lib/input/KeyValueLineRecordReader.java | 13 +++++++++----
.../hadoop/mapreduce/lib/output/TextOutputFormat.java | 9 +++++++--
.../lib/partition/KeyFieldBasedComparator.java | 4 ++--
.../lib/partition/KeyFieldBasedPartitioner.java | 2 +-
.../org/apache/hadoop/mapreduce/util/ConfigUtil.java | 8 ++++----
.../org/apache/hadoop/fs/slive/OperationOutput.java | 4 ++--
.../org/apache/hadoop/mapred/TestFieldSelection.java | 2 +-
.../hadoop/mapred/lib/TestKeyFieldBasedComparator.java | 4 ++--
.../mapreduce/lib/fieldsel/TestMRFieldSelection.java | 2 +-
.../lib/partition/TestMRKeyFieldBasedComparator.java | 4 ++--
.../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +-
20 files changed, 55 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
index 7494adf..9da9ca2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
@@ -45,7 +45,7 @@ public class NodeBase implements Node {
/** Construct a node from its path
* @param path
- * a concatenation of this node's location, the path seperator, and its name
+ * a concatenation of this node's location, the path separator, and its name
*/
public NodeBase(String path) {
path = normalize(path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index d2b572b..e773806 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -372,8 +372,8 @@ public class StringUtils {
/**
* Returns an arraylist of strings.
- * @param str the comma seperated string values
- * @return the arraylist of the comma seperated string values
+ * @param str the comma separated string values
+ * @return the arraylist of the comma separated string values
*/
public static String[] getStrings(String str){
String delim = ",";
@@ -384,7 +384,7 @@ public class StringUtils {
* Returns an arraylist of strings.
* @param str the string values
* @param delim delimiter to separate the values
- * @return the arraylist of the seperated string values
+ * @return the arraylist of the separated string values
*/
public static String[] getStrings(String str, String delim){
Collection<String> values = getStringCollection(str, delim);
@@ -396,7 +396,7 @@ public class StringUtils {
/**
* Returns a collection of strings.
- * @param str comma seperated string values
+ * @param str comma separated string values
* @return an <code>ArrayList</code> of string values
*/
public static Collection<String> getStringCollection(String str){
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index eec6981..5ff42a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -358,7 +358,7 @@ public class FSOperations {
* Creates a Concat executor.
*
* @param path target path to concat to.
- * @param sources comma seperated absolute paths to use as sources.
+ * @param sources comma separated absolute paths to use as sources.
*/
public FSConcat(String path, String[] sources) {
this.sources = new Path[sources.length];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.java
index 03335f9..7a48785 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.java
@@ -159,7 +159,7 @@ public class FieldSelectionMapReduce<K, V>
}
public void configure(JobConf job) {
- this.fieldSeparator = job.get(FieldSelectionHelper.DATA_FIELD_SEPERATOR,
+ this.fieldSeparator = job.get(FieldSelectionHelper.DATA_FIELD_SEPARATOR,
"\t");
this.mapOutputKeyValueSpec = job.get(
FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "0-:");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/KeyFieldBasedComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/KeyFieldBasedComparator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/KeyFieldBasedComparator.java
index c989d77..7eda92a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/KeyFieldBasedComparator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/KeyFieldBasedComparator.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.mapreduce.JobContext;
* of the field); if omitted from pos2, it defaults to 0 (the end of the
* field). opts are ordering options (any of 'nr' as described above).
* We assume that the fields in the key are separated by
- * {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPERATOR}
+ * {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPARATOR}
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 0a4d222..18bf139 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -318,7 +318,13 @@ public interface MRJobConfig {
public static final String MAP_OUTPUT_VALUE_CLASS = "mapreduce.map.output.value.class";
- public static final String MAP_OUTPUT_KEY_FIELD_SEPERATOR = "mapreduce.map.output.key.field.separator";
+ public static final String MAP_OUTPUT_KEY_FIELD_SEPARATOR = "mapreduce.map.output.key.field.separator";
+
+ /**
+ * @deprecated Use {@link #MAP_OUTPUT_KEY_FIELD_SEPARATOR}
+ */
+ @Deprecated
+ public static final String MAP_OUTPUT_KEY_FIELD_SEPERATOR = MAP_OUTPUT_KEY_FIELD_SEPARATOR;
public static final String MAP_LOG_LEVEL = "mapreduce.map.log.level";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java
index 6e22fe9..5ee7e0f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java
@@ -60,8 +60,13 @@ import org.apache.hadoop.io.Text;
public class FieldSelectionHelper {
public static Text emptyText = new Text("");
- public static final String DATA_FIELD_SEPERATOR =
+ public static final String DATA_FIELD_SEPARATOR =
"mapreduce.fieldsel.data.field.separator";
+ /**
+ * @deprecated Use {@link #DATA_FIELD_SEPARATOR}
+ */
+ @Deprecated
+ public static final String DATA_FIELD_SEPERATOR = DATA_FIELD_SEPARATOR;
public static final String MAP_OUTPUT_KEY_VALUE_SPEC =
"mapreduce.fieldsel.map.output.key.value.fields.spec";
public static final String REDUCE_OUTPUT_KEY_VALUE_SPEC =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionMapper.java
index 627e662..fbc1f60 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionMapper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionMapper.java
@@ -79,7 +79,7 @@ public class FieldSelectionMapper<K, V>
throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
this.fieldSeparator =
- conf.get(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "\t");
+ conf.get(FieldSelectionHelper.DATA_FIELD_SEPARATOR, "\t");
this.mapOutputKeyValueSpec =
conf.get(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "0-:");
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionReducer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionReducer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionReducer.java
index 8a6df33..5feeaf4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionReducer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionReducer.java
@@ -77,7 +77,7 @@ public class FieldSelectionReducer<K, V>
Configuration conf = context.getConfiguration();
this.fieldSeparator =
- conf.get(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "\t");
+ conf.get(FieldSelectionHelper.DATA_FIELD_SEPARATOR, "\t");
this.reduceOutputKeyValueSpec =
conf.get(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, "0-:");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java
index 6d66121..06ab6b3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java
@@ -37,9 +37,14 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
@InterfaceAudience.Public
@InterfaceStability.Stable
public class KeyValueLineRecordReader extends RecordReader<Text, Text> {
- public static final String KEY_VALUE_SEPERATOR =
- "mapreduce.input.keyvaluelinerecordreader.key.value.separator";
-
+ public static final String KEY_VALUE_SEPARATOR =
+ "mapreduce.input.keyvaluelinerecordreader.key.value.separator";
+ /**
+ * @deprecated Use {@link #KEY_VALUE_SEPARATOR}
+ */
+ @Deprecated
+ public static final String KEY_VALUE_SEPERATOR = KEY_VALUE_SEPARATOR;
+
private final LineRecordReader lineRecordReader;
private byte separator = (byte) '\t';
@@ -56,7 +61,7 @@ public class KeyValueLineRecordReader extends RecordReader<Text, Text> {
throws IOException {
lineRecordReader = new LineRecordReader();
- String sepStr = conf.get(KEY_VALUE_SEPERATOR, "\t");
+ String sepStr = conf.get(KEY_VALUE_SEPARATOR, "\t");
this.separator = (byte) sepStr.charAt(0);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
index 2e49f68..8b27223 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
@@ -42,7 +42,12 @@ import org.apache.hadoop.util.*;
@InterfaceAudience.Public
@InterfaceStability.Stable
public class TextOutputFormat<K, V> extends FileOutputFormat<K, V> {
- public static String SEPERATOR = "mapreduce.output.textoutputformat.separator";
+ public static String SEPARATOR = "mapreduce.output.textoutputformat.separator";
+ /**
+ * @deprecated Use {@link #SEPARATOR}
+ */
+ @Deprecated
+ public static String SEPERATOR = SEPARATOR;
protected static class LineRecordWriter<K, V>
extends RecordWriter<K, V> {
private static final byte[] NEWLINE =
@@ -107,7 +112,7 @@ public class TextOutputFormat<K, V> extends FileOutputFormat<K, V> {
) throws IOException, InterruptedException {
Configuration conf = job.getConfiguration();
boolean isCompressed = getCompressOutput(job);
- String keyValueSeparator= conf.get(SEPERATOR, "\t");
+ String keyValueSeparator= conf.get(SEPARATOR, "\t");
CompressionCodec codec = null;
String extension = "";
if (isCompressed) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.java
index 4b4d0e9..416bc8c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.java
@@ -46,7 +46,7 @@ import org.apache.hadoop.mapreduce.lib.partition.KeyFieldHelper.KeyDescription;
* of the field); if omitted from pos2, it defaults to 0 (the end of the
* field). opts are ordering options (any of 'nr' as described above).
* We assume that the fields in the key are separated by
- * {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPERATOR}.
+ * {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPARATOR}.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
@@ -62,7 +62,7 @@ public class KeyFieldBasedComparator<K, V> extends WritableComparator
public void setConf(Configuration conf) {
this.conf = conf;
String option = conf.get(COMPARATOR_OPTIONS);
- String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR,"\t");
+ String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPARATOR,"\t");
keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator);
keyFieldHelper.parseOption(option);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java
index 44cb624..73f6a34 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java
@@ -65,7 +65,7 @@ public class KeyFieldBasedPartitioner<K2, V2> extends Partitioner<K2, V2>
this.conf = conf;
keyFieldHelper = new KeyFieldHelper();
String keyFieldSeparator =
- conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, "\t");
+ conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPARATOR, "\t");
keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator);
if (conf.get("num.key.fields.for.partition") != null) {
LOG.warn("Using deprecated num.key.fields.for.partition. " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
index d3f62e4..4b47379 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
@@ -290,7 +290,7 @@ public class ConfigUtil {
new DeprecationDelta("mapred.mapoutput.value.class",
MRJobConfig.MAP_OUTPUT_VALUE_CLASS),
new DeprecationDelta("map.output.key.field.separator",
- MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR),
+ MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPARATOR),
new DeprecationDelta("mapred.map.child.log.level",
MRJobConfig.MAP_LOG_LEVEL),
new DeprecationDelta("mapred.inmem.merge.threshold",
@@ -412,7 +412,7 @@ public class ConfigUtil {
ControlledJob.CREATE_DIR),
new DeprecationDelta("mapred.data.field.separator",
org.apache.hadoop.mapreduce.lib.fieldsel.
- FieldSelectionHelper.DATA_FIELD_SEPERATOR),
+ FieldSelectionHelper.DATA_FIELD_SEPARATOR),
new DeprecationDelta("map.output.key.value.fields.spec",
org.apache.hadoop.mapreduce.lib.fieldsel.
FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC),
@@ -427,7 +427,7 @@ public class ConfigUtil {
CombineFileInputFormat.SPLIT_MINSIZE_PERRACK),
new DeprecationDelta("key.value.separator.in.input.line",
org.apache.hadoop.mapreduce.lib.input.
- KeyValueLineRecordReader.KEY_VALUE_SEPERATOR),
+ KeyValueLineRecordReader.KEY_VALUE_SEPARATOR),
new DeprecationDelta("mapred.linerecordreader.maxlength",
org.apache.hadoop.mapreduce.lib.input.
LineRecordReader.MAX_LINE_LENGTH),
@@ -436,7 +436,7 @@ public class ConfigUtil {
LazyOutputFormat.OUTPUT_FORMAT),
new DeprecationDelta("mapred.textoutputformat.separator",
org.apache.hadoop.mapreduce.lib.output.
- TextOutputFormat.SEPERATOR),
+ TextOutputFormat.SEPARATOR),
new DeprecationDelta("mapred.join.expr",
org.apache.hadoop.mapreduce.lib.join.
CompositeInputFormat.JOIN_EXPR),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/OperationOutput.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/OperationOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/OperationOutput.java
index bca5a1c..02584f1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/OperationOutput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/OperationOutput.java
@@ -65,7 +65,7 @@ class OperationOutput {
int place = key.indexOf(TYPE_SEP);
if (place == -1) {
throw new IllegalArgumentException(
- "Invalid key format - no type seperator - " + TYPE_SEP);
+ "Invalid key format - no type separator - " + TYPE_SEP);
}
try {
dataType = OutputType.valueOf(
@@ -78,7 +78,7 @@ class OperationOutput {
place = key.indexOf(MEASUREMENT_SEP);
if (place == -1) {
throw new IllegalArgumentException(
- "Invalid key format - no measurement seperator - " + MEASUREMENT_SEP);
+ "Invalid key format - no measurement separator - " + MEASUREMENT_SEP);
}
opType = key.substring(0, place);
measurementType = key.substring(place + 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
index 8688968..4e14797 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
@@ -77,7 +77,7 @@ private static NumberFormat idFormat = NumberFormat.getInstance();
job.setOutputFormat(TextOutputFormat.class);
job.setNumReduceTasks(1);
- job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
+ job.set(FieldSelectionHelper.DATA_FIELD_SEPARATOR, "-");
job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
index 35b3f24..3f31546 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
@@ -63,7 +63,7 @@ public class TestKeyFieldBasedComparator extends HadoopTestCase {
super(HadoopTestCase.LOCAL_MR, HadoopTestCase.LOCAL_FS, 1, 1);
conf = createJobConf();
localConf = createJobConf();
- localConf.set(JobContext.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " ");
+ localConf.set(JobContext.MAP_OUTPUT_KEY_FIELD_SEPARATOR, " ");
}
public void configure(String keySpec, int expect) throws Exception {
@@ -85,7 +85,7 @@ public class TestKeyFieldBasedComparator extends HadoopTestCase {
conf.setOutputKeyComparatorClass(KeyFieldBasedComparator.class);
conf.setKeyFieldComparatorOptions(keySpec);
conf.setKeyFieldPartitionerOptions("-k1.1,1.1");
- conf.set(JobContext.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " ");
+ conf.set(JobContext.MAP_OUTPUT_KEY_FIELD_SEPARATOR, " ");
conf.setMapperClass(InverseMapper.class);
conf.setReducerClass(IdentityReducer.class);
if (!fs.mkdirs(testdir)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java
index 6f9183a..26cc328 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java
@@ -57,7 +57,7 @@ private static NumberFormat idFormat = NumberFormat.getInstance();
StringBuffer expectedOutput = new StringBuffer();
constructInputOutputData(inputData, expectedOutput, numOfInputLines);
- conf.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
+ conf.set(FieldSelectionHelper.DATA_FIELD_SEPARATOR, "-");
conf.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
conf.set(
FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java
index 0d75d2f..889137c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java
@@ -50,7 +50,7 @@ public class TestMRKeyFieldBasedComparator extends HadoopTestCase {
public TestMRKeyFieldBasedComparator() throws IOException {
super(HadoopTestCase.LOCAL_MR, HadoopTestCase.LOCAL_FS, 1, 1);
conf = createJobConf();
- conf.set(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " ");
+ conf.set(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPARATOR, " ");
}
private void testComparator(String keySpec, int expect)
@@ -61,7 +61,7 @@ public class TestMRKeyFieldBasedComparator extends HadoopTestCase {
conf.set("mapreduce.partition.keycomparator.options", keySpec);
conf.set("mapreduce.partition.keypartitioner.options", "-k1.1,1.1");
- conf.set(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " ");
+ conf.set(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPARATOR, " ");
Job job = MapReduceTestUtil.createJob(conf, inDir, outDir, 1, 1,
line1 +"\n" + line2 + "\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7297e82c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f52e487..c239249 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1897,7 +1897,7 @@ public class YarnConfiguration extends Configuration {
public static final float DEFAULT_TIMELINE_SERVICE_VERSION = 1.0f;
/**
- * Comma seperated list of names for UIs hosted in the timeline server
+ * Comma separated list of names for UIs hosted in the timeline server
* (For pluggable UIs).
*/
public static final String TIMELINE_SERVICE_UI_NAMES =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HADOOP-14267. Make DistCpOptions
immutable. Contributed by Mingliang Liu
Posted by ae...@apache.org.
HADOOP-14267. Make DistCpOptions immutable. Contributed by Mingliang Liu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26172a94
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26172a94
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26172a94
Branch: refs/heads/HDFS-7240
Commit: 26172a94d6431e70d7fe15d66be9a7e195f79f60
Parents: 73835c7
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Jun 23 00:21:49 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Fri Mar 31 20:04:26 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/tools/CopyListing.java | 38 +-
.../java/org/apache/hadoop/tools/DistCp.java | 106 +--
.../org/apache/hadoop/tools/DistCpContext.java | 198 ++++
.../apache/hadoop/tools/DistCpOptionSwitch.java | 2 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 925 +++++++++----------
.../org/apache/hadoop/tools/DistCpSync.java | 42 +-
.../hadoop/tools/FileBasedCopyListing.java | 12 +-
.../apache/hadoop/tools/GlobbedCopyListing.java | 17 +-
.../org/apache/hadoop/tools/OptionsParser.java | 299 ++----
.../apache/hadoop/tools/SimpleCopyListing.java | 115 +--
.../hadoop/tools/mapred/CopyCommitter.java | 15 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 8 +-
.../apache/hadoop/tools/TestCopyListing.java | 51 +-
.../apache/hadoop/tools/TestDistCpOptions.java | 500 ++++++++++
.../org/apache/hadoop/tools/TestDistCpSync.java | 68 +-
.../hadoop/tools/TestDistCpSyncReverseBase.java | 44 +-
.../apache/hadoop/tools/TestDistCpViewFs.java | 10 +-
.../hadoop/tools/TestFileBasedCopyListing.java | 9 +-
.../hadoop/tools/TestGlobbedCopyListing.java | 11 +-
.../apache/hadoop/tools/TestIntegration.java | 20 +-
.../apache/hadoop/tools/TestOptionsParser.java | 81 +-
.../contract/AbstractContractDistCpTest.java | 6 +-
.../hadoop/tools/mapred/TestCopyCommitter.java | 34 +-
.../mapred/TestUniformSizeInputFormat.java | 15 +-
.../mapred/lib/TestDynamicInputFormat.java | 17 +-
25 files changed, 1574 insertions(+), 1069 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
index 9ebf9d2..908b558 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
@@ -77,41 +77,41 @@ public abstract class CopyListing extends Configured {
* TARGET IS DIR : Key-"/file1", Value-FileStatus(/tmp/file1)
*
* @param pathToListFile - Output file where the listing would be stored
- * @param options - Input options to distcp
+ * @param distCpContext - distcp context associated with input options
* @throws IOException - Exception if any
*/
public final void buildListing(Path pathToListFile,
- DistCpOptions options) throws IOException {
- validatePaths(options);
- doBuildListing(pathToListFile, options);
+ DistCpContext distCpContext) throws IOException {
+ validatePaths(distCpContext);
+ doBuildListing(pathToListFile, distCpContext);
Configuration config = getConf();
config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, pathToListFile.toString());
config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, getBytesToCopy());
config.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, getNumberOfPaths());
- validateFinalListing(pathToListFile, options);
+ validateFinalListing(pathToListFile, distCpContext);
LOG.info("Number of paths in the copy list: " + this.getNumberOfPaths());
}
/**
* Validate input and output paths
*
- * @param options - Input options
+ * @param distCpContext - Distcp context
* @throws InvalidInputException If inputs are invalid
* @throws IOException any Exception with FS
*/
- protected abstract void validatePaths(DistCpOptions options)
+ protected abstract void validatePaths(DistCpContext distCpContext)
throws IOException, InvalidInputException;
/**
* The interface to be implemented by sub-classes, to create the source/target file listing.
* @param pathToListFile Path on HDFS where the listing file is written.
- * @param options Input Options for DistCp (indicating source/target paths.)
+ * @param distCpContext - Distcp context
* @throws IOException Thrown on failure to create the listing file.
*/
protected abstract void doBuildListing(Path pathToListFile,
- DistCpOptions options) throws IOException;
+ DistCpContext distCpContext) throws IOException;
/**
* Return the total bytes that distCp should copy for the source paths
@@ -135,17 +135,17 @@ public abstract class CopyListing extends Configured {
* If preserving XAttrs, checks that file system can support XAttrs.
*
* @param pathToListFile - path listing build by doBuildListing
- * @param options - Input options to distcp
+ * @param context - Distcp context with associated input options
* @throws IOException - Any issues while checking for duplicates and throws
* @throws DuplicateFileException - if there are duplicates
*/
- private void validateFinalListing(Path pathToListFile, DistCpOptions options)
+ private void validateFinalListing(Path pathToListFile, DistCpContext context)
throws DuplicateFileException, IOException {
Configuration config = getConf();
FileSystem fs = pathToListFile.getFileSystem(config);
- final boolean splitLargeFile = options.splitLargeFile();
+ final boolean splitLargeFile = context.splitLargeFile();
// When splitLargeFile is enabled, we don't randomize the copylist
// earlier, so we don't do the sorting here. For a file that has
@@ -188,7 +188,7 @@ public abstract class CopyListing extends Configured {
}
}
reader.getCurrentValue(lastFileStatus);
- if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
+ if (context.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
URI lastFsUri = lastFs.getUri();
if (!aclSupportCheckFsSet.contains(lastFsUri)) {
@@ -196,7 +196,7 @@ public abstract class CopyListing extends Configured {
aclSupportCheckFsSet.add(lastFsUri);
}
}
- if (options.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) {
+ if (context.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) {
FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
URI lastFsUri = lastFs.getUri();
if (!xAttrSupportCheckFsSet.contains(lastFsUri)) {
@@ -210,7 +210,7 @@ public abstract class CopyListing extends Configured {
lastChunkOffset = lastFileStatus.getChunkOffset();
lastChunkLength = lastFileStatus.getChunkLength();
}
- if (options.shouldUseDiff() && LOG.isDebugEnabled()) {
+ if (context.shouldUseDiff() && LOG.isDebugEnabled()) {
LOG.debug("Copy list entry " + idx + ": " +
lastFileStatus.getPath().toUri().getPath());
idx++;
@@ -253,14 +253,12 @@ public abstract class CopyListing extends Configured {
* Public Factory method with which the appropriate CopyListing implementation may be retrieved.
* @param configuration The input configuration.
* @param credentials Credentials object on which the FS delegation tokens are cached
- * @param options The input Options, to help choose the appropriate CopyListing Implementation.
+ * @param context Distcp context with associated input options
* @return An instance of the appropriate CopyListing implementation.
* @throws java.io.IOException - Exception if any
*/
public static CopyListing getCopyListing(Configuration configuration,
- Credentials credentials,
- DistCpOptions options)
- throws IOException {
+ Credentials credentials, DistCpContext context) throws IOException {
String copyListingClassName = configuration.get(DistCpConstants.
CONF_LABEL_COPY_LISTING_CLASS, "");
Class<? extends CopyListing> copyListingClass;
@@ -270,7 +268,7 @@ public abstract class CopyListing extends Configured {
CONF_LABEL_COPY_LISTING_CLASS, GlobbedCopyListing.class,
CopyListing.class);
} else {
- if (options.getSourceFileListing() == null) {
+ if (context.getSourceFileListing() == null) {
copyListingClass = GlobbedCopyListing.class;
} else {
copyListingClass = FileBasedCopyListing.class;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 8c2fa24..df9c328 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.tools;
import java.io.IOException;
import java.util.Random;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -35,7 +36,6 @@ import org.apache.hadoop.mapreduce.Cluster;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
-import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.CopyListing.*;
import org.apache.hadoop.tools.mapred.CopyMapper;
import org.apache.hadoop.tools.mapred.CopyOutputFormat;
@@ -66,7 +66,9 @@ public class DistCp extends Configured implements Tool {
static final Log LOG = LogFactory.getLog(DistCp.class);
- private DistCpOptions inputOptions;
+ @VisibleForTesting
+ DistCpContext context;
+
private Path metaFolder;
private static final String PREFIX = "_distcp";
@@ -79,15 +81,14 @@ public class DistCp extends Configured implements Tool {
private FileSystem jobFS;
private void prepareFileListing(Job job) throws Exception {
- if (inputOptions.shouldUseSnapshotDiff()) {
+ if (context.shouldUseSnapshotDiff()) {
// When "-diff" or "-rdiff" is passed, do sync() first, then
// create copyListing based on snapshot diff.
- DistCpSync distCpSync = new DistCpSync(inputOptions, getConf());
+ DistCpSync distCpSync = new DistCpSync(context, getConf());
if (distCpSync.sync()) {
createInputFileListingWithDiff(job, distCpSync);
} else {
- throw new Exception("DistCp sync failed, input options: "
- + inputOptions);
+ throw new Exception("DistCp sync failed, input options: " + context);
}
} else {
// When no "-diff" or "-rdiff" is passed, create copyListing
@@ -99,16 +100,19 @@ public class DistCp extends Configured implements Tool {
/**
* Public Constructor. Creates DistCp object with specified input-parameters.
* (E.g. source-paths, target-location, etc.)
- * @param inputOptions Options (indicating source-paths, target-location.)
- * @param configuration The Hadoop configuration against which the Copy-mapper must run.
+ * @param configuration configuration against which the Copy-mapper must run
+ * @param inputOptions Immutable options
* @throws Exception
*/
- public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception {
+ public DistCp(Configuration configuration, DistCpOptions inputOptions)
+ throws Exception {
Configuration config = new Configuration(configuration);
config.addResource(DISTCP_DEFAULT_XML);
config.addResource(DISTCP_SITE_XML);
setConf(config);
- this.inputOptions = inputOptions;
+ if (inputOptions != null) {
+ this.context = new DistCpContext(inputOptions);
+ }
this.metaFolder = createMetaFolderPath();
}
@@ -134,10 +138,10 @@ public class DistCp extends Configured implements Tool {
}
try {
- inputOptions = (OptionsParser.parse(argv));
- setOptionsForSplitLargeFile();
+ context = new DistCpContext(OptionsParser.parse(argv));
+ checkSplitLargeFile();
setTargetPathExists();
- LOG.info("Input Options: " + inputOptions);
+ LOG.info("Input Options: " + context);
} catch (Throwable e) {
LOG.error("Invalid arguments: ", e);
System.err.println("Invalid arguments: " + e.getMessage());
@@ -173,9 +177,11 @@ public class DistCp extends Configured implements Tool {
* @throws Exception
*/
public Job execute() throws Exception {
+ Preconditions.checkState(context != null,
+ "The DistCpContext should have been created before running DistCp!");
Job job = createAndSubmitJob();
- if (inputOptions.shouldBlock()) {
+ if (context.shouldBlock()) {
waitForJobCompletion(job);
}
return job;
@@ -186,7 +192,7 @@ public class DistCp extends Configured implements Tool {
* @return The mapreduce job object that has been submitted
*/
public Job createAndSubmitJob() throws Exception {
- assert inputOptions != null;
+ assert context != null;
assert getConf() != null;
Job job = null;
try {
@@ -230,53 +236,36 @@ public class DistCp extends Configured implements Tool {
* for the benefit of CopyCommitter
*/
private void setTargetPathExists() throws IOException {
- Path target = inputOptions.getTargetPath();
+ Path target = context.getTargetPath();
FileSystem targetFS = target.getFileSystem(getConf());
boolean targetExists = targetFS.exists(target);
- inputOptions.setTargetPathExists(targetExists);
+ context.setTargetPathExists(targetExists);
getConf().setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS,
targetExists);
}
/**
- * Check if concat is supported by fs.
- * Throws UnsupportedOperationException if not.
+ * Check splitting large files is supported and populate configs.
*/
- private void checkConcatSupport(FileSystem fs) {
+ private void checkSplitLargeFile() throws IOException {
+ if (!context.splitLargeFile()) {
+ return;
+ }
+
+ final Path target = context.getTargetPath();
+ final FileSystem targetFS = target.getFileSystem(getConf());
try {
Path[] src = null;
Path tgt = null;
- fs.concat(tgt, src);
+ targetFS.concat(tgt, src);
} catch (UnsupportedOperationException use) {
throw new UnsupportedOperationException(
DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() +
- " is not supported since the target file system doesn't" +
- " support concat.", use);
+ " is not supported since the target file system doesn't" +
+ " support concat.", use);
} catch (Exception e) {
// Ignore other exception
}
- }
-
- /**
- * Set up needed options for splitting large files.
- */
- private void setOptionsForSplitLargeFile() throws IOException {
- if (!inputOptions.splitLargeFile()) {
- return;
- }
- Path target = inputOptions.getTargetPath();
- FileSystem targetFS = target.getFileSystem(getConf());
- checkConcatSupport(targetFS);
-
- LOG.info("Enabling preserving blocksize since "
- + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " is passed.");
- inputOptions.preserve(FileAttribute.BLOCKSIZE);
-
- LOG.info("Set " +
- DistCpOptionSwitch.APPEND.getSwitch()
- + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()
- + " is passed.");
- inputOptions.setAppend(false);
LOG.info("Set " +
DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES
@@ -286,7 +275,6 @@ public class DistCp extends Configured implements Tool {
DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false);
}
-
/**
* Create Job object for submitting it, with all the configuration
*
@@ -300,7 +288,7 @@ public class DistCp extends Configured implements Tool {
jobName += ": " + userChosenName;
Job job = Job.getInstance(getConf());
job.setJobName(jobName);
- job.setInputFormatClass(DistCpUtils.getStrategy(getConf(), inputOptions));
+ job.setInputFormatClass(DistCpUtils.getStrategy(getConf(), context));
job.setJarByClass(CopyMapper.class);
configureOutputFormat(job);
@@ -311,9 +299,9 @@ public class DistCp extends Configured implements Tool {
job.setOutputFormatClass(CopyOutputFormat.class);
job.getConfiguration().set(JobContext.MAP_SPECULATIVE, "false");
job.getConfiguration().set(JobContext.NUM_MAPS,
- String.valueOf(inputOptions.getMaxMaps()));
+ String.valueOf(context.getMaxMaps()));
- inputOptions.appendToConf(job.getConfiguration());
+ context.appendToConf(job.getConfiguration());
return job;
}
@@ -325,18 +313,20 @@ public class DistCp extends Configured implements Tool {
*/
private void configureOutputFormat(Job job) throws IOException {
final Configuration configuration = job.getConfiguration();
- Path targetPath = inputOptions.getTargetPath();
+ Path targetPath = context.getTargetPath();
FileSystem targetFS = targetPath.getFileSystem(configuration);
targetPath = targetPath.makeQualified(targetFS.getUri(),
targetFS.getWorkingDirectory());
- if (inputOptions.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
+ if (context.shouldPreserve(
+ DistCpOptions.FileAttribute.ACL)) {
DistCpUtils.checkFileSystemAclSupport(targetFS);
}
- if (inputOptions.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) {
+ if (context.shouldPreserve(
+ DistCpOptions.FileAttribute.XATTR)) {
DistCpUtils.checkFileSystemXAttrSupport(targetFS);
}
- if (inputOptions.shouldAtomicCommit()) {
- Path workDir = inputOptions.getAtomicWorkPath();
+ if (context.shouldAtomicCommit()) {
+ Path workDir = context.getAtomicWorkPath();
if (workDir == null) {
workDir = targetPath.getParent();
}
@@ -353,7 +343,7 @@ public class DistCp extends Configured implements Tool {
}
CopyOutputFormat.setCommitDirectory(job, targetPath);
- Path logPath = inputOptions.getLogPath();
+ Path logPath = context.getLogPath();
if (logPath == null) {
logPath = new Path(metaFolder, "_logs");
} else {
@@ -374,8 +364,8 @@ public class DistCp extends Configured implements Tool {
protected Path createInputFileListing(Job job) throws IOException {
Path fileListingPath = getFileListingPath();
CopyListing copyListing = CopyListing.getCopyListing(job.getConfiguration(),
- job.getCredentials(), inputOptions);
- copyListing.buildListing(fileListingPath, inputOptions);
+ job.getCredentials(), context);
+ copyListing.buildListing(fileListingPath, context);
return fileListingPath;
}
@@ -391,7 +381,7 @@ public class DistCp extends Configured implements Tool {
Path fileListingPath = getFileListingPath();
CopyListing copyListing = new SimpleCopyListing(job.getConfiguration(),
job.getCredentials(), distCpSync);
- copyListing.buildListing(fileListingPath, inputOptions);
+ copyListing.buildListing(fileListingPath, context);
return fileListingPath;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java
new file mode 100644
index 0000000..c34005e
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.tools;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This is the context of the distcp at runtime.
+ *
+ * It has the immutable {@link DistCpOptions} and mutable runtime status.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DistCpContext {
+ private final DistCpOptions options;
+
+ /** The source paths can be set at runtime via snapshots. */
+ private List<Path> sourcePaths;
+
+ /** This is a derived field, it's initialized in the beginning of distcp. */
+ private boolean targetPathExists = true;
+
+ /** Indicate that raw.* xattrs should be preserved if true. */
+ private boolean preserveRawXattrs = false;
+
+ public DistCpContext(DistCpOptions options) {
+ this.options = options;
+ this.sourcePaths = options.getSourcePaths();
+ }
+
+ public void setSourcePaths(List<Path> sourcePaths) {
+ this.sourcePaths = sourcePaths;
+ }
+
+ /**
+ * @return the sourcePaths. Please note this method does not directly delegate
+ * to the {@link #options}.
+ */
+ public List<Path> getSourcePaths() {
+ return sourcePaths;
+ }
+
+ public Path getSourceFileListing() {
+ return options.getSourceFileListing();
+ }
+
+ public Path getTargetPath() {
+ return options.getTargetPath();
+ }
+
+ public boolean shouldAtomicCommit() {
+ return options.shouldAtomicCommit();
+ }
+
+ public boolean shouldSyncFolder() {
+ return options.shouldSyncFolder();
+ }
+
+ public boolean shouldDeleteMissing() {
+ return options.shouldDeleteMissing();
+ }
+
+ public boolean shouldIgnoreFailures() {
+ return options.shouldIgnoreFailures();
+ }
+
+ public boolean shouldOverwrite() {
+ return options.shouldOverwrite();
+ }
+
+ public boolean shouldAppend() {
+ return options.shouldAppend();
+ }
+
+ public boolean shouldSkipCRC() {
+ return options.shouldSkipCRC();
+ }
+
+ public boolean shouldBlock() {
+ return options.shouldBlock();
+ }
+
+ public boolean shouldUseDiff() {
+ return options.shouldUseDiff();
+ }
+
+ public boolean shouldUseRdiff() {
+ return options.shouldUseRdiff();
+ }
+
+ public boolean shouldUseSnapshotDiff() {
+ return options.shouldUseSnapshotDiff();
+ }
+
+ public String getFromSnapshot() {
+ return options.getFromSnapshot();
+ }
+
+ public String getToSnapshot() {
+ return options.getToSnapshot();
+ }
+
+ public final String getFiltersFile() {
+ return options.getFiltersFile();
+ }
+
+ public int getNumListstatusThreads() {
+ return options.getNumListstatusThreads();
+ }
+
+ public int getMaxMaps() {
+ return options.getMaxMaps();
+ }
+
+ public float getMapBandwidth() {
+ return options.getMapBandwidth();
+ }
+
+ public Set<FileAttribute> getPreserveAttributes() {
+ return options.getPreserveAttributes();
+ }
+
+ public boolean shouldPreserve(FileAttribute attribute) {
+ return options.shouldPreserve(attribute);
+ }
+
+ public boolean shouldPreserveRawXattrs() {
+ return preserveRawXattrs;
+ }
+
+ public void setPreserveRawXattrs(boolean preserveRawXattrs) {
+ this.preserveRawXattrs = preserveRawXattrs;
+ }
+
+ public Path getAtomicWorkPath() {
+ return options.getAtomicWorkPath();
+ }
+
+ public Path getLogPath() {
+ return options.getLogPath();
+ }
+
+ public String getCopyStrategy() {
+ return options.getCopyStrategy();
+ }
+
+ public int getBlocksPerChunk() {
+ return options.getBlocksPerChunk();
+ }
+
+ public final boolean splitLargeFile() {
+ return options.getBlocksPerChunk() > 0;
+ }
+
+ public void setTargetPathExists(boolean targetPathExists) {
+ this.targetPathExists = targetPathExists;
+ }
+
+ public boolean isTargetPathExists() {
+ return targetPathExists;
+ }
+
+ public void appendToConf(Configuration conf) {
+ options.appendToConf(conf);
+ }
+
+ @Override
+ public String toString() {
+ return options.toString() +
+ ", sourcePaths=" + sourcePaths +
+ ", targetPathExists=" + targetPathExists +
+ ", preserveRawXattrs" + preserveRawXattrs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index ced9b54..81abb7d 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -81,7 +81,7 @@ public enum DistCpOptionSwitch {
NUM_LISTSTATUS_THREADS(DistCpConstants.CONF_LABEL_LISTSTATUS_THREADS,
new Option("numListstatusThreads", true, "Number of threads to " +
"use for building file listing (max " +
- DistCpOptions.maxNumListstatusThreads + ").")),
+ DistCpOptions.MAX_NUM_LISTSTATUS_THREADS + ").")),
/**
* Max number of maps to use during copy. DistCp will split work
* as equally as possible among these maps
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index 9822d83..97ae0c4 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -18,43 +18,88 @@
package org.apache.hadoop.tools;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.tools.util.DistCpUtils;
+import java.util.Collections;
import java.util.EnumSet;
-import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
+import java.util.Set;
/**
* The Options class encapsulates all DistCp options.
- * These may be set from command-line (via the OptionsParser)
- * or may be set manually.
+ *
+ * When you add a new option, please:
+ * - Add the field along with javadoc in DistCpOptions and its Builder
+ * - Add setter method in the {@link Builder} class
+ *
+ * This class is immutable.
*/
-public class DistCpOptions {
-
- private boolean atomicCommit = false;
- private boolean syncFolder = false;
- private boolean deleteMissing = false;
- private boolean ignoreFailures = false;
- private boolean overwrite = false;
- private boolean append = false;
- private boolean skipCRC = false;
- private boolean blocking = true;
+public final class DistCpOptions {
+ private static final Logger LOG = LoggerFactory.getLogger(Builder.class);
+ public static final int MAX_NUM_LISTSTATUS_THREADS = 40;
+
+ /** File path (hdfs:// or file://) that contains the list of actual files to
+ * copy.
+ */
+ private final Path sourceFileListing;
+
+ /** List of source-paths (including wildcards) to be copied to target. */
+ private final List<Path> sourcePaths;
+
+ /** Destination path for the dist-copy. */
+ private final Path targetPath;
+
+ /** Whether data need to be committed automatically. */
+ private final boolean atomicCommit;
+
+ /** the work path for atomic commit. If null, the work
+ * path would be parentOf(targetPath) + "/._WIP_" + nameOf(targetPath). */
+ private final Path atomicWorkPath;
+
+ /** Whether source and target folder contents be sync'ed up. */
+ private final boolean syncFolder;
+
+ /** Whether files only present in target should be deleted. */
+ private boolean deleteMissing;
+
+ /** Whether failures during copy be ignored. */
+ private final boolean ignoreFailures;
+
+ /** Whether files should always be overwritten on target. */
+ private final boolean overwrite;
+
+ /** Whether we want to append new data to target files. This is valid only
+ * with update option and CRC is not skipped. */
+ private final boolean append;
+
+ /** Whether checksum comparison should be skipped while determining if source
+ * and destination files are identical. */
+ private final boolean skipCRC;
+
+ /** Whether to run blocking or non-blocking. */
+ private final boolean blocking;
+
// When "-diff s1 s2 src tgt" is passed, apply forward snapshot diff (from s1
// to s2) of source cluster to the target cluster to sync target cluster with
// the source cluster. Referred to as "Fdiff" in the code.
// It's required that s2 is newer than s1.
- private boolean useDiff = false;
+ private final boolean useDiff;
// When "-rdiff s2 s1 src tgt" is passed, apply reversed snapshot diff (from
// s2 to s1) of target cluster to the target cluster, so to make target
// cluster go back to s1. Referred to as "Rdiff" in the code.
// It's required that s2 is newer than s1, and src and tgt have exact same
// content at their s1, if src is not the same as tgt.
- private boolean useRdiff = false;
+ private final boolean useRdiff;
// For both -diff and -rdiff, given the example command line switches, two
// steps are taken:
@@ -66,44 +111,53 @@ public class DistCpOptions {
// could be the tgt itself (HDFS-9820).
//
- public static final int maxNumListstatusThreads = 40;
- private int numListstatusThreads = 0; // Indicates that flag is not set.
- private int maxMaps = DistCpConstants.DEFAULT_MAPS;
- private float mapBandwidth = 0; // Indicates that we should use the default.
-
- private String copyStrategy = DistCpConstants.UNIFORMSIZE;
+ private final String fromSnapshot;
+ private final String toSnapshot;
- private EnumSet<FileAttribute> preserveStatus = EnumSet.noneOf(FileAttribute.class);
+ /** The path to a file containing a list of paths to filter out of copy. */
+ private final String filtersFile;
- private boolean preserveRawXattrs;
+ /** Path where output logs are stored. If not specified, it will use the
+ * default value JobStagingDir/_logs and delete upon job completion. */
+ private final Path logPath;
- private Path atomicWorkPath;
+ /** Set the copy strategy to use. Should map to a strategy implementation
+ * in distp-default.xml. */
+ private final String copyStrategy;
- private Path logPath;
+ /** per map bandwidth in MB. */
+ private final float mapBandwidth;
- private Path sourceFileListing;
- private List<Path> sourcePaths;
+ /** The number of threads to use for listStatus. We allow max
+ * {@link #MAX_NUM_LISTSTATUS_THREADS} threads. Setting numThreads to zero
+ * signify we should use the value from conf properties. */
+ private final int numListstatusThreads;
- private String fromSnapshot;
- private String toSnapshot;
+ /** The max number of maps to use for copy. */
+ private final int maxMaps;
- private Path targetPath;
-
- /**
- * The path to a file containing a list of paths to filter out of the copy.
- */
- private String filtersFile;
-
- // targetPathExist is a derived field, it's initialized in the
- // beginning of distcp.
- private boolean targetPathExists = true;
+ /** File attributes that need to be preserved. */
+ private final EnumSet<FileAttribute> preserveStatus;
// Size of chunk in number of blocks when splitting large file into chunks
// to copy in parallel. Default is 0 and file are not splitted.
- private int blocksPerChunk = 0;
+ private final int blocksPerChunk;
- public static enum FileAttribute{
- REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES;
+ /**
+ * File attributes for preserve.
+ *
+ * Each enum entry uses the first char as its symbol.
+ */
+ public enum FileAttribute {
+ REPLICATION, // R
+ BLOCKSIZE, // B
+ USER, // U
+ GROUP, // G
+ PERMISSION, // P
+ CHECKSUMTYPE, // C
+ ACL, // A
+ XATTR, // X
+ TIMES; // T
public static FileAttribute getAttribute(char symbol) {
for (FileAttribute attribute : values()) {
@@ -115,187 +169,86 @@ public class DistCpOptions {
}
}
- /**
- * Constructor, to initialize source/target paths.
- * @param sourcePaths List of source-paths (including wildcards)
- * to be copied to target.
- * @param targetPath Destination path for the dist-copy.
- */
- public DistCpOptions(List<Path> sourcePaths, Path targetPath) {
- assert sourcePaths != null && !sourcePaths.isEmpty() : "Invalid source paths";
- assert targetPath != null : "Invalid Target path";
+ private DistCpOptions(Builder builder) {
+ this.sourceFileListing = builder.sourceFileListing;
+ this.sourcePaths = builder.sourcePaths;
+ this.targetPath = builder.targetPath;
+
+ this.atomicCommit = builder.atomicCommit;
+ this.atomicWorkPath = builder.atomicWorkPath;
+ this.syncFolder = builder.syncFolder;
+ this.deleteMissing = builder.deleteMissing;
+ this.ignoreFailures = builder.ignoreFailures;
+ this.overwrite = builder.overwrite;
+ this.append = builder.append;
+ this.skipCRC = builder.skipCRC;
+ this.blocking = builder.blocking;
+
+ this.useDiff = builder.useDiff;
+ this.useRdiff = builder.useRdiff;
+ this.fromSnapshot = builder.fromSnapshot;
+ this.toSnapshot = builder.toSnapshot;
- this.sourcePaths = sourcePaths;
- this.targetPath = targetPath;
+ this.filtersFile = builder.filtersFile;
+ this.logPath = builder.logPath;
+ this.copyStrategy = builder.copyStrategy;
+
+ this.mapBandwidth = builder.mapBandwidth;
+ this.numListstatusThreads = builder.numListstatusThreads;
+ this.maxMaps = builder.maxMaps;
+
+ this.preserveStatus = builder.preserveStatus;
+
+ this.blocksPerChunk = builder.blocksPerChunk;
}
- /**
- * Constructor, to initialize source/target paths.
- * @param sourceFileListing File containing list of source paths
- * @param targetPath Destination path for the dist-copy.
- */
- public DistCpOptions(Path sourceFileListing, Path targetPath) {
- assert sourceFileListing != null : "Invalid source paths";
- assert targetPath != null : "Invalid Target path";
+ public Path getSourceFileListing() {
+ return sourceFileListing;
+ }
- this.sourceFileListing = sourceFileListing;
- this.targetPath = targetPath;
+ public List<Path> getSourcePaths() {
+ return sourcePaths == null ?
+ null : Collections.unmodifiableList(sourcePaths);
}
- /**
- * Copy constructor.
- * @param that DistCpOptions being copied from.
- */
- public DistCpOptions(DistCpOptions that) {
- if (this != that && that != null) {
- this.atomicCommit = that.atomicCommit;
- this.syncFolder = that.syncFolder;
- this.deleteMissing = that.deleteMissing;
- this.ignoreFailures = that.ignoreFailures;
- this.overwrite = that.overwrite;
- this.skipCRC = that.skipCRC;
- this.blocking = that.blocking;
- this.useDiff = that.useDiff;
- this.useRdiff = that.useRdiff;
- this.numListstatusThreads = that.numListstatusThreads;
- this.maxMaps = that.maxMaps;
- this.mapBandwidth = that.mapBandwidth;
- this.copyStrategy = that.copyStrategy;
- this.preserveStatus = that.preserveStatus;
- this.preserveRawXattrs = that.preserveRawXattrs;
- this.atomicWorkPath = that.getAtomicWorkPath();
- this.logPath = that.getLogPath();
- this.sourceFileListing = that.getSourceFileListing();
- this.sourcePaths = that.getSourcePaths();
- this.targetPath = that.getTargetPath();
- this.targetPathExists = that.getTargetPathExists();
- this.filtersFile = that.getFiltersFile();
- this.blocksPerChunk = that.blocksPerChunk;
- }
+ public Path getTargetPath() {
+ return targetPath;
}
- /**
- * Should the data be committed atomically?
- *
- * @return true if data should be committed automically. false otherwise
- */
public boolean shouldAtomicCommit() {
return atomicCommit;
}
- /**
- * Set if data need to be committed automatically
- *
- * @param atomicCommit - boolean switch
- */
- public void setAtomicCommit(boolean atomicCommit) {
- this.atomicCommit = atomicCommit;
+ public Path getAtomicWorkPath() {
+ return atomicWorkPath;
}
- /**
- * Should the data be sync'ed between source and target paths?
- *
- * @return true if data should be sync'ed up. false otherwise
- */
public boolean shouldSyncFolder() {
return syncFolder;
}
- /**
- * Set if source and target folder contents be sync'ed up
- *
- * @param syncFolder - boolean switch
- */
- public void setSyncFolder(boolean syncFolder) {
- this.syncFolder = syncFolder;
- }
-
- /**
- * Should target files missing in source should be deleted?
- *
- * @return true if zoombie target files to be removed. false otherwise
- */
public boolean shouldDeleteMissing() {
return deleteMissing;
}
- /**
- * Set if files only present in target should be deleted
- *
- * @param deleteMissing - boolean switch
- */
- public void setDeleteMissing(boolean deleteMissing) {
- this.deleteMissing = deleteMissing;
- }
-
- /**
- * Should failures be logged and ignored during copy?
- *
- * @return true if failures are to be logged and ignored. false otherwise
- */
public boolean shouldIgnoreFailures() {
return ignoreFailures;
}
- /**
- * Set if failures during copy be ignored
- *
- * @param ignoreFailures - boolean switch
- */
- public void setIgnoreFailures(boolean ignoreFailures) {
- this.ignoreFailures = ignoreFailures;
- }
-
- /**
- * Should DistCp be running in blocking mode
- *
- * @return true if should run in blocking, false otherwise
- */
- public boolean shouldBlock() {
- return blocking;
- }
-
- /**
- * Set if Disctp should run blocking or non-blocking
- *
- * @param blocking - boolean switch
- */
- public void setBlocking(boolean blocking) {
- this.blocking = blocking;
- }
-
- /**
- * Should files be overwritten always?
- *
- * @return true if files in target that may exist before distcp, should always
- * be overwritten. false otherwise
- */
public boolean shouldOverwrite() {
return overwrite;
}
- /**
- * Set if files should always be overwritten on target
- *
- * @param overwrite - boolean switch
- */
- public void setOverwrite(boolean overwrite) {
- this.overwrite = overwrite;
- }
-
- /**
- * @return whether we can append new data to target files
- */
public boolean shouldAppend() {
return append;
}
- /**
- * Set if we want to append new data to target files. This is valid only with
- * update option and CRC is not skipped.
- */
- public void setAppend(boolean append) {
- this.append = append;
+ public boolean shouldSkipCRC() {
+ return skipCRC;
+ }
+
+ public boolean shouldBlock() {
+ return blocking;
}
public boolean shouldUseDiff() {
@@ -318,104 +271,34 @@ public class DistCpOptions {
return this.toSnapshot;
}
- public void setUseDiff(String fromSS, String toSS) {
- this.useDiff = true;
- this.fromSnapshot = fromSS;
- this.toSnapshot = toSS;
- }
-
- public void setUseRdiff(String fromSS, String toSS) {
- this.useRdiff = true;
- this.fromSnapshot = fromSS;
- this.toSnapshot = toSS;
+ public String getFiltersFile() {
+ return filtersFile;
}
- /**
- * Should CRC/checksum check be skipped while checking files are identical
- *
- * @return true if checksum check should be skipped while checking files are
- * identical. false otherwise
- */
- public boolean shouldSkipCRC() {
- return skipCRC;
+ public Path getLogPath() {
+ return logPath;
}
- /**
- * Set if checksum comparison should be skipped while determining if
- * source and destination files are identical
- *
- * @param skipCRC - boolean switch
- */
- public void setSkipCRC(boolean skipCRC) {
- this.skipCRC = skipCRC;
+ public String getCopyStrategy() {
+ return copyStrategy;
}
- /** Get the number of threads to use for listStatus
- *
- * @return Number of threads to do listStatus
- */
public int getNumListstatusThreads() {
return numListstatusThreads;
}
- /** Set the number of threads to use for listStatus. We allow max 40
- * threads. Setting numThreads to zero signify we should use the value
- * from conf properties.
- *
- * @param numThreads - Number of threads
- */
- public void setNumListstatusThreads(int numThreads) {
- if (numThreads > maxNumListstatusThreads) {
- this.numListstatusThreads = maxNumListstatusThreads;
- } else if (numThreads > 0) {
- this.numListstatusThreads = numThreads;
- } else {
- this.numListstatusThreads = 0;
- }
- }
-
- /** Get the max number of maps to use for this copy
- *
- * @return Max number of maps
- */
public int getMaxMaps() {
return maxMaps;
}
- /**
- * Set the max number of maps to use for copy
- *
- * @param maxMaps - Number of maps
- */
- public void setMaxMaps(int maxMaps) {
- this.maxMaps = Math.max(maxMaps, 1);
- }
-
- /** Get the map bandwidth in MB
- *
- * @return Bandwidth in MB
- */
public float getMapBandwidth() {
return mapBandwidth;
}
- /**
- * Set per map bandwidth
- *
- * @param mapBandwidth - per map bandwidth
- */
- public void setMapBandwidth(float mapBandwidth) {
- assert mapBandwidth > 0 : "Bandwidth " + mapBandwidth + " is invalid (should be > 0)";
- this.mapBandwidth = mapBandwidth;
- }
-
- /**
- * Returns an iterator with the list of file attributes to preserve
- *
- * @return iterator of file attributes to preserve
- */
- public Iterator<FileAttribute> preserveAttributes() {
- return preserveStatus.iterator();
+ public Set<FileAttribute> getPreserveAttributes() {
+ return (preserveStatus == null)
+ ? null
+ : Collections.unmodifiableSet(preserveStatus);
}
/**
@@ -428,230 +311,10 @@ public class DistCpOptions {
return preserveStatus.contains(attribute);
}
- /**
- * Add file attributes that need to be preserved. This method may be
- * called multiple times to add attributes.
- *
- * @param fileAttribute - Attribute to add, one at a time
- */
- public void preserve(FileAttribute fileAttribute) {
- for (FileAttribute attribute : preserveStatus) {
- if (attribute.equals(fileAttribute)) {
- return;
- }
- }
- preserveStatus.add(fileAttribute);
- }
-
- /**
- * Return true if raw.* xattrs should be preserved.
- * @return true if raw.* xattrs should be preserved.
- */
- public boolean shouldPreserveRawXattrs() {
- return preserveRawXattrs;
- }
-
- /**
- * Indicate that raw.* xattrs should be preserved
- */
- public void preserveRawXattrs() {
- preserveRawXattrs = true;
- }
-
- /** Get work path for atomic commit. If null, the work
- * path would be parentOf(targetPath) + "/._WIP_" + nameOf(targetPath)
- *
- * @return Atomic work path on the target cluster. Null if not set
- */
- public Path getAtomicWorkPath() {
- return atomicWorkPath;
- }
-
- /**
- * Set the work path for atomic commit
- *
- * @param atomicWorkPath - Path on the target cluster
- */
- public void setAtomicWorkPath(Path atomicWorkPath) {
- this.atomicWorkPath = atomicWorkPath;
- }
-
- /** Get output directory for writing distcp logs. Otherwise logs
- * are temporarily written to JobStagingDir/_logs and deleted
- * upon job completion
- *
- * @return Log output path on the cluster where distcp job is run
- */
- public Path getLogPath() {
- return logPath;
- }
-
- /**
- * Set the log path where distcp output logs are stored
- * Uses JobStagingDir/_logs by default
- *
- * @param logPath - Path where logs will be saved
- */
- public void setLogPath(Path logPath) {
- this.logPath = logPath;
- }
-
- /**
- * Get the copy strategy to use. Uses appropriate input format
- *
- * @return copy strategy to use
- */
- public String getCopyStrategy() {
- return copyStrategy;
- }
-
- /**
- * Set the copy strategy to use. Should map to a strategy implementation
- * in distp-default.xml
- *
- * @param copyStrategy - copy Strategy to use
- */
- public void setCopyStrategy(String copyStrategy) {
- this.copyStrategy = copyStrategy;
- }
-
- /**
- * File path (hdfs:// or file://) that contains the list of actual
- * files to copy
- *
- * @return - Source listing file path
- */
- public Path getSourceFileListing() {
- return sourceFileListing;
- }
-
- /**
- * Getter for sourcePaths.
- * @return List of source-paths.
- */
- public List<Path> getSourcePaths() {
- return sourcePaths;
- }
-
- /**
- * Setter for sourcePaths.
- * @param sourcePaths The new list of source-paths.
- */
- public void setSourcePaths(List<Path> sourcePaths) {
- assert sourcePaths != null && sourcePaths.size() != 0;
- this.sourcePaths = sourcePaths;
- }
-
- /**
- * Getter for the targetPath.
- * @return The target-path.
- */
- public Path getTargetPath() {
- return targetPath;
- }
-
- /**
- * Getter for the targetPathExists.
- * @return The target-path.
- */
- public boolean getTargetPathExists() {
- return targetPathExists;
- }
-
- /**
- * Set targetPathExists.
- * @param targetPathExists Whether the target path of distcp exists.
- */
- public boolean setTargetPathExists(boolean targetPathExists) {
- return this.targetPathExists = targetPathExists;
- }
-
- /**
- * File path that contains the list of patterns
- * for paths to be filtered from the file copy.
- * @return - Filter file path.
- */
- public final String getFiltersFile() {
- return filtersFile;
- }
-
- /**
- * Set filtersFile.
- * @param filtersFilename The path to a list of patterns to exclude from copy.
- */
- public final void setFiltersFile(String filtersFilename) {
- this.filtersFile = filtersFilename;
- }
-
- public final void setBlocksPerChunk(int csize) {
- this.blocksPerChunk = csize;
- }
-
- public final int getBlocksPerChunk() {
+ public int getBlocksPerChunk() {
return blocksPerChunk;
}
- public final boolean splitLargeFile() {
- return blocksPerChunk > 0;
- }
-
- void validate() {
- if ((useDiff || useRdiff) && deleteMissing) {
- // -delete and -diff/-rdiff are mutually exclusive. For backward
- // compatibility, we ignore the -delete option here, instead of throwing
- // an IllegalArgumentException. See HDFS-10397 for more discussion.
- OptionsParser.LOG.warn(
- "-delete and -diff/-rdiff are mutually exclusive. " +
- "The -delete option will be ignored.");
- setDeleteMissing(false);
- }
-
- if (syncFolder && atomicCommit) {
- throw new IllegalArgumentException("Atomic commit can't be used with " +
- "sync folder or overwrite options");
- }
-
- if (deleteMissing && !(overwrite || syncFolder)) {
- throw new IllegalArgumentException("Delete missing is applicable " +
- "only with update or overwrite options");
- }
-
- if (overwrite && syncFolder) {
- throw new IllegalArgumentException("Overwrite and update options are " +
- "mutually exclusive");
- }
-
- if (!syncFolder && skipCRC) {
- throw new IllegalArgumentException("Skip CRC is valid only with update options");
- }
-
- if (!syncFolder && append) {
- throw new IllegalArgumentException(
- "Append is valid only with update options");
- }
- if (skipCRC && append) {
- throw new IllegalArgumentException(
- "Append is disallowed when skipping CRC");
- }
- if (!syncFolder && (useDiff || useRdiff)) {
- throw new IllegalArgumentException(
- "-diff/-rdiff is valid only with -update option");
- }
-
- if (useDiff || useRdiff) {
- if (StringUtils.isBlank(fromSnapshot) ||
- StringUtils.isBlank(toSnapshot)) {
- throw new IllegalArgumentException(
- "Must provide both the starting and ending " +
- "snapshot names for -diff/-rdiff");
- }
- }
- if (useDiff && useRdiff) {
- throw new IllegalArgumentException(
- "-diff and -rdiff are mutually exclusive");
- }
- }
-
/**
* Add options to configuration. These will be used in the Mapper/committer
*
@@ -715,20 +378,292 @@ public class DistCpOptions {
", mapBandwidth=" + mapBandwidth +
", copyStrategy='" + copyStrategy + '\'' +
", preserveStatus=" + preserveStatus +
- ", preserveRawXattrs=" + preserveRawXattrs +
", atomicWorkPath=" + atomicWorkPath +
", logPath=" + logPath +
", sourceFileListing=" + sourceFileListing +
", sourcePaths=" + sourcePaths +
", targetPath=" + targetPath +
- ", targetPathExists=" + targetPathExists +
", filtersFile='" + filtersFile + '\'' +
", blocksPerChunk=" + blocksPerChunk +
'}';
}
- @Override
- protected DistCpOptions clone() throws CloneNotSupportedException {
- return (DistCpOptions) super.clone();
+ /**
+ * The builder of the {@link DistCpOptions}.
+ *
+ * This is designed to be the only public interface to create a
+ * {@link DistCpOptions} object for users. It follows a simple Builder design
+ * pattern.
+ */
+ public static class Builder {
+ private Path sourceFileListing;
+ private List<Path> sourcePaths;
+ private Path targetPath;
+
+ private boolean atomicCommit = false;
+ private Path atomicWorkPath;
+ private boolean syncFolder = false;
+ private boolean deleteMissing = false;
+ private boolean ignoreFailures = false;
+ private boolean overwrite = false;
+ private boolean append = false;
+ private boolean skipCRC = false;
+ private boolean blocking = true;
+
+ private boolean useDiff = false;
+ private boolean useRdiff = false;
+ private String fromSnapshot;
+ private String toSnapshot;
+
+ private String filtersFile;
+
+ private Path logPath;
+ private String copyStrategy = DistCpConstants.UNIFORMSIZE;
+
+ private int numListstatusThreads = 0; // 0 indicates that flag is not set.
+ private int maxMaps = DistCpConstants.DEFAULT_MAPS;
+ private float mapBandwidth = 0; // 0 indicates we should use the default
+
+ private EnumSet<FileAttribute> preserveStatus =
+ EnumSet.noneOf(FileAttribute.class);
+
+ private int blocksPerChunk = 0;
+
+ public Builder(List<Path> sourcePaths, Path targetPath) {
+ Preconditions.checkArgument(sourcePaths != null && !sourcePaths.isEmpty(),
+ "Source paths should not be null or empty!");
+ Preconditions.checkArgument(targetPath != null,
+ "Target path should not be null!");
+ this.sourcePaths = sourcePaths;
+ this.targetPath = targetPath;
+ }
+
+ public Builder(Path sourceFileListing, Path targetPath) {
+ Preconditions.checkArgument(sourceFileListing != null,
+ "Source file listing should not be null!");
+ Preconditions.checkArgument(targetPath != null,
+ "Target path should not be null!");
+
+ this.sourceFileListing = sourceFileListing;
+ this.targetPath = targetPath;
+ }
+
+ /**
+ * This is the single entry point for constructing DistCpOptions objects.
+ *
+ * Before a new DistCpOptions object is returned, it will set the dependent
+ * options, validate the option combinations. After constructing, the
+ * DistCpOptions instance is immutable.
+ */
+ public DistCpOptions build() {
+ setOptionsForSplitLargeFile();
+
+ validate();
+
+ return new DistCpOptions(this);
+ }
+
+ /**
+ * Override options for split large files.
+ */
+ private void setOptionsForSplitLargeFile() {
+ if (blocksPerChunk <= 0) {
+ return;
+ }
+
+ LOG.info("Enabling preserving blocksize since "
+ + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " is passed.");
+ preserve(FileAttribute.BLOCKSIZE);
+
+ LOG.info("Set " + DistCpOptionSwitch.APPEND.getSwitch()
+ + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()
+ + " is passed.");
+ this.append = false;
+ }
+
+ private void validate() {
+ if ((useDiff || useRdiff) && deleteMissing) {
+ // -delete and -diff/-rdiff are mutually exclusive.
+ throw new IllegalArgumentException("-delete and -diff/-rdiff are "
+ + "mutually exclusive. The -delete option will be ignored.");
+ }
+
+ if (!atomicCommit && atomicWorkPath != null) {
+ throw new IllegalArgumentException(
+ "-tmp work-path can only be specified along with -atomic");
+ }
+
+ if (syncFolder && atomicCommit) {
+ throw new IllegalArgumentException("Atomic commit can't be used with "
+ + "sync folder or overwrite options");
+ }
+
+ if (deleteMissing && !(overwrite || syncFolder)) {
+ throw new IllegalArgumentException("Delete missing is applicable "
+ + "only with update or overwrite options");
+ }
+
+ if (overwrite && syncFolder) {
+ throw new IllegalArgumentException("Overwrite and update options are "
+ + "mutually exclusive");
+ }
+
+ if (!syncFolder && skipCRC) {
+ throw new IllegalArgumentException(
+ "Skip CRC is valid only with update options");
+ }
+
+ if (!syncFolder && append) {
+ throw new IllegalArgumentException(
+ "Append is valid only with update options");
+ }
+ if (skipCRC && append) {
+ throw new IllegalArgumentException(
+ "Append is disallowed when skipping CRC");
+ }
+ if (!syncFolder && (useDiff || useRdiff)) {
+ throw new IllegalArgumentException(
+ "-diff/-rdiff is valid only with -update option");
+ }
+
+ if (useDiff || useRdiff) {
+ if (StringUtils.isBlank(fromSnapshot) ||
+ StringUtils.isBlank(toSnapshot)) {
+ throw new IllegalArgumentException(
+ "Must provide both the starting and ending " +
+ "snapshot names for -diff/-rdiff");
+ }
+ }
+ if (useDiff && useRdiff) {
+ throw new IllegalArgumentException(
+ "-diff and -rdiff are mutually exclusive");
+ }
+ }
+
+ @VisibleForTesting
+ Builder withSourcePaths(List<Path> newSourcePaths) {
+ this.sourcePaths = newSourcePaths;
+ return this;
+ }
+
+ public Builder withAtomicCommit(boolean newAtomicCommit) {
+ this.atomicCommit = newAtomicCommit;
+ return this;
+ }
+
+ public Builder withAtomicWorkPath(Path newAtomicWorkPath) {
+ this.atomicWorkPath = newAtomicWorkPath;
+ return this;
+ }
+
+ public Builder withSyncFolder(boolean newSyncFolder) {
+ this.syncFolder = newSyncFolder;
+ return this;
+ }
+
+ public Builder withDeleteMissing(boolean newDeleteMissing) {
+ this.deleteMissing = newDeleteMissing;
+ return this;
+ }
+
+ public Builder withIgnoreFailures(boolean newIgnoreFailures) {
+ this.ignoreFailures = newIgnoreFailures;
+ return this;
+ }
+
+ public Builder withOverwrite(boolean newOverwrite) {
+ this.overwrite = newOverwrite;
+ return this;
+ }
+
+ public Builder withAppend(boolean newAppend) {
+ this.append = newAppend;
+ return this;
+ }
+
+ public Builder withCRC(boolean newSkipCRC) {
+ this.skipCRC = newSkipCRC;
+ return this;
+ }
+
+ public Builder withBlocking(boolean newBlocking) {
+ this.blocking = newBlocking;
+ return this;
+ }
+
+ public Builder withUseDiff(String newFromSnapshot, String newToSnapshot) {
+ this.useDiff = true;
+ this.fromSnapshot = newFromSnapshot;
+ this.toSnapshot = newToSnapshot;
+ return this;
+ }
+
+ public Builder withUseRdiff(String newFromSnapshot, String newToSnapshot) {
+ this.useRdiff = true;
+ this.fromSnapshot = newFromSnapshot;
+ this.toSnapshot = newToSnapshot;
+ return this;
+ }
+
+ public Builder withFiltersFile(String newFiletersFile) {
+ this.filtersFile = newFiletersFile;
+ return this;
+ }
+
+ public Builder withLogPath(Path newLogPath) {
+ this.logPath = newLogPath;
+ return this;
+ }
+
+ public Builder withCopyStrategy(String newCopyStrategy) {
+ this.copyStrategy = newCopyStrategy;
+ return this;
+ }
+
+ public Builder withMapBandwidth(float newMapBandwidth) {
+ Preconditions.checkArgument(newMapBandwidth > 0,
+ "Bandwidth " + newMapBandwidth + " is invalid (should be > 0)");
+ this.mapBandwidth = newMapBandwidth;
+ return this;
+ }
+
+ public Builder withNumListstatusThreads(int newNumListstatusThreads) {
+ if (newNumListstatusThreads > MAX_NUM_LISTSTATUS_THREADS) {
+ this.numListstatusThreads = MAX_NUM_LISTSTATUS_THREADS;
+ } else if (newNumListstatusThreads > 0) {
+ this.numListstatusThreads = newNumListstatusThreads;
+ } else {
+ this.numListstatusThreads = 0;
+ }
+ return this;
+ }
+
+ public Builder maxMaps(int newMaxMaps) {
+ this.maxMaps = Math.max(newMaxMaps, 1);
+ return this;
+ }
+
+ public Builder preserve(String attributes) {
+ if (attributes == null || attributes.isEmpty()) {
+ preserveStatus = EnumSet.allOf(FileAttribute.class);
+ } else {
+ for (int index = 0; index < attributes.length(); index++) {
+ preserveStatus.add(FileAttribute.
+ getAttribute(attributes.charAt(index)));
+ }
+ }
+ return this;
+ }
+
+ public Builder preserve(FileAttribute attribute) {
+ preserveStatus.add(attribute);
+ return this;
+ }
+
+ public Builder withBlocksPerChunk(int newBlocksPerChunk) {
+ this.blocksPerChunk = newBlocksPerChunk;
+ return this;
+ }
}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
index bcae96a..a78320b 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
@@ -48,7 +48,7 @@ import java.util.HashSet;
* source.s1
*/
class DistCpSync {
- private DistCpOptions inputOptions;
+ private DistCpContext context;
private Configuration conf;
// diffMap maps snapshot diff op type to a list of diff ops.
// It's initially created based on the snapshot diff. Then the individual
@@ -58,13 +58,13 @@ class DistCpSync {
private EnumMap<SnapshotDiffReport.DiffType, List<DiffInfo>> diffMap;
private DiffInfo[] renameDiffs;
- DistCpSync(DistCpOptions options, Configuration conf) {
- this.inputOptions = options;
+ DistCpSync(DistCpContext context, Configuration conf) {
+ this.context = context;
this.conf = conf;
}
private boolean isRdiff() {
- return inputOptions.shouldUseRdiff();
+ return context.shouldUseRdiff();
}
/**
@@ -77,14 +77,14 @@ class DistCpSync {
* default distcp if the third condition isn't met.
*/
private boolean preSyncCheck() throws IOException {
- List<Path> sourcePaths = inputOptions.getSourcePaths();
+ List<Path> sourcePaths = context.getSourcePaths();
if (sourcePaths.size() != 1) {
// we only support one source dir which must be a snapshottable directory
throw new IllegalArgumentException(sourcePaths.size()
+ " source paths are provided");
}
final Path sourceDir = sourcePaths.get(0);
- final Path targetDir = inputOptions.getTargetPath();
+ final Path targetDir = context.getTargetPath();
final FileSystem srcFs = sourceDir.getFileSystem(conf);
final FileSystem tgtFs = targetDir.getFileSystem(conf);
@@ -104,13 +104,15 @@ class DistCpSync {
// make sure targetFS has no change between from and the current states
if (!checkNoChange(targetFs, targetDir)) {
// set the source path using the snapshot path
- inputOptions.setSourcePaths(Arrays.asList(getSnapshotPath(sourceDir,
- inputOptions.getToSnapshot())));
+ context.setSourcePaths(Arrays.asList(getSnapshotPath(sourceDir,
+ context.getToSnapshot())));
return false;
}
- final String from = getSnapshotName(inputOptions.getFromSnapshot());
- final String to = getSnapshotName(inputOptions.getToSnapshot());
+ final String from = getSnapshotName(
+ context.getFromSnapshot());
+ final String to = getSnapshotName(
+ context.getToSnapshot());
try {
final FileStatus fromSnapshotStat =
@@ -152,9 +154,9 @@ class DistCpSync {
return false;
}
- List<Path> sourcePaths = inputOptions.getSourcePaths();
+ List<Path> sourcePaths = context.getSourcePaths();
final Path sourceDir = sourcePaths.get(0);
- final Path targetDir = inputOptions.getTargetPath();
+ final Path targetDir = context.getTargetPath();
final FileSystem tfs = targetDir.getFileSystem(conf);
final DistributedFileSystem targetFs = (DistributedFileSystem) tfs;
@@ -175,8 +177,8 @@ class DistCpSync {
deleteTargetTmpDir(targetFs, tmpDir);
// TODO: since we have tmp directory, we can support "undo" with failures
// set the source path using the snapshot path
- inputOptions.setSourcePaths(Arrays.asList(getSnapshotPath(sourceDir,
- inputOptions.getToSnapshot())));
+ context.setSourcePaths(Arrays.asList(getSnapshotPath(sourceDir,
+ context.getToSnapshot())));
}
}
@@ -187,13 +189,13 @@ class DistCpSync {
*/
private boolean getAllDiffs() throws IOException {
Path ssDir = isRdiff()?
- inputOptions.getTargetPath() : inputOptions.getSourcePaths().get(0);
+ context.getTargetPath() : context.getSourcePaths().get(0);
try {
DistributedFileSystem fs =
(DistributedFileSystem) ssDir.getFileSystem(conf);
- final String from = getSnapshotName(inputOptions.getFromSnapshot());
- final String to = getSnapshotName(inputOptions.getToSnapshot());
+ final String from = getSnapshotName(context.getFromSnapshot());
+ final String to = getSnapshotName(context.getToSnapshot());
SnapshotDiffReport report = fs.getSnapshotDiffReport(ssDir,
from, to);
this.diffMap = new EnumMap<>(SnapshotDiffReport.DiffType.class);
@@ -273,19 +275,19 @@ class DistCpSync {
*/
private boolean checkNoChange(DistributedFileSystem fs, Path path) {
try {
- final String from = getSnapshotName(inputOptions.getFromSnapshot());
+ final String from = getSnapshotName(context.getFromSnapshot());
SnapshotDiffReport targetDiff =
fs.getSnapshotDiffReport(path, from, "");
if (!targetDiff.getDiffList().isEmpty()) {
DistCp.LOG.warn("The target has been modified since snapshot "
- + inputOptions.getFromSnapshot());
+ + context.getFromSnapshot());
return false;
} else {
return true;
}
} catch (IOException e) {
DistCp.LOG.warn("Failed to compute snapshot diff on " + path
- + " at snapshot " + inputOptions.getFromSnapshot(), e);
+ + " at snapshot " + context.getFromSnapshot(), e);
}
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java
index 2bc343e..c356edd 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java
@@ -52,7 +52,7 @@ public class FileBasedCopyListing extends CopyListing {
/** {@inheritDoc} */
@Override
- protected void validatePaths(DistCpOptions options)
+ protected void validatePaths(DistCpContext context)
throws IOException, InvalidInputException {
}
@@ -60,14 +60,14 @@ public class FileBasedCopyListing extends CopyListing {
* Implementation of CopyListing::buildListing().
* Iterates over all source paths mentioned in the input-file.
* @param pathToListFile Path on HDFS where the listing file is written.
- * @param options Input Options for DistCp (indicating source/target paths.)
+ * @param context Distcp context with associated input options.
* @throws IOException
*/
@Override
- public void doBuildListing(Path pathToListFile, DistCpOptions options) throws IOException {
- DistCpOptions newOption = new DistCpOptions(options);
- newOption.setSourcePaths(fetchFileList(options.getSourceFileListing()));
- globbedListing.buildListing(pathToListFile, newOption);
+ public void doBuildListing(Path pathToListFile, DistCpContext context)
+ throws IOException {
+ context.setSourcePaths(fetchFileList(context.getSourceFileListing()));
+ globbedListing.buildListing(pathToListFile, context);
}
private List<Path> fetchFileList(Path sourceListing) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/GlobbedCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/GlobbedCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/GlobbedCopyListing.java
index 27330b7..63c6f43 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/GlobbedCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/GlobbedCopyListing.java
@@ -51,7 +51,7 @@ public class GlobbedCopyListing extends CopyListing {
/** {@inheritDoc} */
@Override
- protected void validatePaths(DistCpOptions options)
+ protected void validatePaths(DistCpContext context)
throws IOException, InvalidInputException {
}
@@ -60,19 +60,19 @@ public class GlobbedCopyListing extends CopyListing {
* Creates the copy listing by "globbing" all source-paths.
* @param pathToListingFile The location at which the copy-listing file
* is to be created.
- * @param options Input Options for DistCp (indicating source/target paths.)
+ * @param context The distcp context with associated input options.
* @throws IOException
*/
@Override
- public void doBuildListing(Path pathToListingFile,
- DistCpOptions options) throws IOException {
+ public void doBuildListing(Path pathToListingFile, DistCpContext context)
+ throws IOException {
List<Path> globbedPaths = new ArrayList<Path>();
- if (options.getSourcePaths().isEmpty()) {
+ if (context.getSourcePaths().isEmpty()) {
throw new InvalidInputException("Nothing to process. Source paths::EMPTY");
}
- for (Path p : options.getSourcePaths()) {
+ for (Path p : context.getSourcePaths()) {
FileSystem fs = p.getFileSystem(getConf());
FileStatus[] inputs = fs.globStatus(p);
@@ -85,9 +85,8 @@ public class GlobbedCopyListing extends CopyListing {
}
}
- DistCpOptions optionsGlobbed = new DistCpOptions(options);
- optionsGlobbed.setSourcePaths(globbedPaths);
- simpleListing.buildListing(pathToListingFile, optionsGlobbed);
+ context.setSourcePaths(globbedPaths);
+ simpleListing.buildListing(pathToListingFile, context);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
index 8881264..21ff0f8 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
@@ -32,7 +32,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import com.google.common.base.Preconditions;
@@ -95,253 +94,126 @@ public class OptionsParser {
Arrays.toString(args), e);
}
- DistCpOptions option = parseSourceAndTargetPaths(command);
-
- option.setIgnoreFailures(
- command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch()));
-
- option.setAtomicCommit(
- command.hasOption(DistCpOptionSwitch.ATOMIC_COMMIT.getSwitch()));
-
- option.setSyncFolder(
- command.hasOption(DistCpOptionSwitch.SYNC_FOLDERS.getSwitch()));
-
- option.setOverwrite(
- command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch()));
-
- option.setAppend(
- command.hasOption(DistCpOptionSwitch.APPEND.getSwitch()));
-
- option.setDeleteMissing(
- command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch()));
-
- option.setSkipCRC(
- command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch()));
-
- if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch()) &&
- option.shouldAtomicCommit()) {
- String workPath = getVal(command, DistCpOptionSwitch.WORK_PATH.getSwitch());
- if (workPath != null && !workPath.isEmpty()) {
- option.setAtomicWorkPath(new Path(workPath));
- }
- } else if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch())) {
- throw new IllegalArgumentException("-tmp work-path can only be specified along with -atomic");
- }
-
- if (command.hasOption(DistCpOptionSwitch.LOG_PATH.getSwitch())) {
- option.setLogPath(new Path(getVal(command, DistCpOptionSwitch.LOG_PATH.getSwitch())));
- }
-
-
- if (command.hasOption(DistCpOptionSwitch.BLOCKING.getSwitch())) {
- option.setBlocking(false);
- }
-
- parseBandwidth(command, option);
-
- parseNumListStatusThreads(command, option);
-
- parseMaxMaps(command, option);
-
- if (command.hasOption(DistCpOptionSwitch.COPY_STRATEGY.getSwitch())) {
- option.setCopyStrategy(
- getVal(command, DistCpOptionSwitch.COPY_STRATEGY.getSwitch()));
- }
-
- parsePreserveStatus(command, option);
+ DistCpOptions.Builder builder = parseSourceAndTargetPaths(command);
+ builder
+ .withAtomicCommit(
+ command.hasOption(DistCpOptionSwitch.ATOMIC_COMMIT.getSwitch()))
+ .withSyncFolder(
+ command.hasOption(DistCpOptionSwitch.SYNC_FOLDERS.getSwitch()))
+ .withDeleteMissing(
+ command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch()))
+ .withIgnoreFailures(
+ command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch()))
+ .withOverwrite(
+ command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch()))
+ .withAppend(
+ command.hasOption(DistCpOptionSwitch.APPEND.getSwitch()))
+ .withCRC(
+ command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch()))
+ .withBlocking(
+ !command.hasOption(DistCpOptionSwitch.BLOCKING.getSwitch()));
if (command.hasOption(DistCpOptionSwitch.DIFF.getSwitch())) {
String[] snapshots = getVals(command,
DistCpOptionSwitch.DIFF.getSwitch());
checkSnapshotsArgs(snapshots);
- option.setUseDiff(snapshots[0], snapshots[1]);
+ builder.withUseDiff(snapshots[0], snapshots[1]);
}
if (command.hasOption(DistCpOptionSwitch.RDIFF.getSwitch())) {
String[] snapshots = getVals(command,
DistCpOptionSwitch.RDIFF.getSwitch());
checkSnapshotsArgs(snapshots);
- option.setUseRdiff(snapshots[0], snapshots[1]);
+ builder.withUseRdiff(snapshots[0], snapshots[1]);
}
- parseFileLimit(command);
-
- parseSizeLimit(command);
-
if (command.hasOption(DistCpOptionSwitch.FILTERS.getSwitch())) {
- option.setFiltersFile(getVal(command,
- DistCpOptionSwitch.FILTERS.getSwitch()));
+ builder.withFiltersFile(
+ getVal(command, DistCpOptionSwitch.FILTERS.getSwitch()));
}
- parseBlocksPerChunk(command, option);
-
- option.validate();
-
- return option;
- }
-
-
- /**
- * A helper method to parse chunk size in number of blocks.
- * Used when breaking large file into chunks to copy in parallel.
- *
- * @param command command line arguments
- */
- private static void parseBlocksPerChunk(CommandLine command,
- DistCpOptions option) {
- boolean hasOption =
- command.hasOption(DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch());
- LOG.info("parseChunkSize: " +
- DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " " + hasOption);
- if (hasOption) {
- String chunkSizeString = getVal(command,
- DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch().trim());
- try {
- int csize = Integer.parseInt(chunkSizeString);
- if (csize < 0) {
- csize = 0;
- }
- LOG.info("Set distcp blocksPerChunk to " + csize);
- option.setBlocksPerChunk(csize);
- }
- catch (NumberFormatException e) {
- throw new IllegalArgumentException("blocksPerChunk is invalid: "
- + chunkSizeString, e);
- }
+ if (command.hasOption(DistCpOptionSwitch.LOG_PATH.getSwitch())) {
+ builder.withLogPath(
+ new Path(getVal(command, DistCpOptionSwitch.LOG_PATH.getSwitch())));
}
- }
- /**
- * parseSizeLimit is a helper method for parsing the deprecated
- * argument SIZE_LIMIT.
- *
- * @param command command line arguments
- */
- private static void parseSizeLimit(CommandLine command) {
- if (command.hasOption(DistCpOptionSwitch.SIZE_LIMIT.getSwitch())) {
- String sizeLimitString = getVal(command,
- DistCpOptionSwitch.SIZE_LIMIT.getSwitch().trim());
- try {
- Long.parseLong(sizeLimitString);
- }
- catch (NumberFormatException e) {
- throw new IllegalArgumentException("Size-limit is invalid: "
- + sizeLimitString, e);
+ if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch())) {
+ final String workPath = getVal(command,
+ DistCpOptionSwitch.WORK_PATH.getSwitch());
+ if (workPath != null && !workPath.isEmpty()) {
+ builder.withAtomicWorkPath(new Path(workPath));
}
- LOG.warn(DistCpOptionSwitch.SIZE_LIMIT.getSwitch() + " is a deprecated" +
- " option. Ignoring.");
}
- }
- /**
- * parseFileLimit is a helper method for parsing the deprecated
- * argument FILE_LIMIT.
- *
- * @param command command line arguments
- */
- private static void parseFileLimit(CommandLine command) {
- if (command.hasOption(DistCpOptionSwitch.FILE_LIMIT.getSwitch())) {
- String fileLimitString = getVal(command,
- DistCpOptionSwitch.FILE_LIMIT.getSwitch().trim());
+ if (command.hasOption(DistCpOptionSwitch.BANDWIDTH.getSwitch())) {
try {
- Integer.parseInt(fileLimitString);
+ final Float mapBandwidth = Float.parseFloat(
+ getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()));
+ builder.withMapBandwidth(mapBandwidth);
} catch (NumberFormatException e) {
- throw new IllegalArgumentException("File-limit is invalid: "
- + fileLimitString, e);
+ throw new IllegalArgumentException("Bandwidth specified is invalid: " +
+ getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()), e);
}
- LOG.warn(DistCpOptionSwitch.FILE_LIMIT.getSwitch() + " is a deprecated" +
- " option. Ignoring.");
}
- }
- /**
- * parsePreserveStatus is a helper method for parsing PRESERVE_STATUS.
- *
- * @param command command line arguments
- * @param option parsed distcp options
- */
- private static void parsePreserveStatus(CommandLine command,
- DistCpOptions option) {
- if (command.hasOption(DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())) {
- String attributes =
- getVal(command, DistCpOptionSwitch.PRESERVE_STATUS.getSwitch());
- if (attributes == null || attributes.isEmpty()) {
- for (FileAttribute attribute : FileAttribute.values()) {
- option.preserve(attribute);
- }
- } else {
- for (int index = 0; index < attributes.length(); index++) {
- option.preserve(FileAttribute.
- getAttribute(attributes.charAt(index)));
- }
+ if (command.hasOption(
+ DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch())) {
+ try {
+ final Integer numThreads = Integer.parseInt(getVal(command,
+ DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch()));
+ builder.withNumListstatusThreads(numThreads);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(
+ "Number of liststatus threads is invalid: " + getVal(command,
+ DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch()), e);
}
}
- }
- /**
- * parseMaxMaps is a helper method for parsing MAX_MAPS.
- *
- * @param command command line arguments
- * @param option parsed distcp options
- */
- private static void parseMaxMaps(CommandLine command,
- DistCpOptions option) {
if (command.hasOption(DistCpOptionSwitch.MAX_MAPS.getSwitch())) {
try {
- Integer maps = Integer.parseInt(
- getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()).trim());
- option.setMaxMaps(maps);
+ final Integer maps = Integer.parseInt(
+ getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()));
+ builder.maxMaps(maps);
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Number of maps is invalid: " +
getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()), e);
}
}
- }
- /**
- * parseNumListStatusThreads is a helper method for parsing
- * NUM_LISTSTATUS_THREADS.
- *
- * @param command command line arguments
- * @param option parsed distcp options
- */
- private static void parseNumListStatusThreads(CommandLine command,
- DistCpOptions option) {
- if (command.hasOption(
- DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch())) {
- try {
- Integer numThreads = Integer.parseInt(getVal(command,
- DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch()).trim());
- option.setNumListstatusThreads(numThreads);
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException(
- "Number of liststatus threads is invalid: " + getVal(command,
- DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch()), e);
- }
+ if (command.hasOption(DistCpOptionSwitch.COPY_STRATEGY.getSwitch())) {
+ builder.withCopyStrategy(
+ getVal(command, DistCpOptionSwitch.COPY_STRATEGY.getSwitch()));
}
- }
- /**
- * parseBandwidth is a helper method for parsing BANDWIDTH.
- *
- * @param command command line arguments
- * @param option parsed distcp options
- */
- private static void parseBandwidth(CommandLine command,
- DistCpOptions option) {
- if (command.hasOption(DistCpOptionSwitch.BANDWIDTH.getSwitch())) {
+ if (command.hasOption(DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())) {
+ builder.preserve(
+ getVal(command, DistCpOptionSwitch.PRESERVE_STATUS.getSwitch()));
+ }
+
+ if (command.hasOption(DistCpOptionSwitch.FILE_LIMIT.getSwitch())) {
+ LOG.warn(DistCpOptionSwitch.FILE_LIMIT.getSwitch() + " is a deprecated" +
+ " option. Ignoring.");
+ }
+
+ if (command.hasOption(DistCpOptionSwitch.SIZE_LIMIT.getSwitch())) {
+ LOG.warn(DistCpOptionSwitch.SIZE_LIMIT.getSwitch() + " is a deprecated" +
+ " option. Ignoring.");
+ }
+
+ if (command.hasOption(DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch())) {
+ final String chunkSizeStr = getVal(command,
+ DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch().trim());
try {
- Float mapBandwidth = Float.parseFloat(
- getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()).trim());
- if (mapBandwidth <= 0) {
- throw new IllegalArgumentException("Bandwidth specified is not " +
- "positive: " + mapBandwidth);
- }
- option.setMapBandwidth(mapBandwidth);
+ int csize = Integer.parseInt(chunkSizeStr);
+ csize = csize > 0 ? csize : 0;
+ LOG.info("Set distcp blocksPerChunk to " + csize);
+ builder.withBlocksPerChunk(csize);
} catch (NumberFormatException e) {
- throw new IllegalArgumentException("Bandwidth specified is invalid: " +
- getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()), e);
+ throw new IllegalArgumentException("blocksPerChunk is invalid: "
+ + chunkSizeStr, e);
}
}
+
+ return builder.build();
}
/**
@@ -351,9 +223,8 @@ public class OptionsParser {
* @param command command line arguments
* @return DistCpOptions
*/
- private static DistCpOptions parseSourceAndTargetPaths(
+ private static DistCpOptions.Builder parseSourceAndTargetPaths(
CommandLine command) {
- DistCpOptions option;
Path targetPath;
List<Path> sourcePaths = new ArrayList<Path>();
@@ -378,20 +249,22 @@ public class OptionsParser {
throw new IllegalArgumentException("Both source file listing and " +
"source paths present");
}
- option = new DistCpOptions(new Path(getVal(command, DistCpOptionSwitch.
- SOURCE_FILE_LISTING.getSwitch())), targetPath);
+ return new DistCpOptions.Builder(new Path(getVal(command,
+ DistCpOptionSwitch.SOURCE_FILE_LISTING.getSwitch())), targetPath);
} else {
if (sourcePaths.isEmpty()) {
throw new IllegalArgumentException("Neither source file listing nor " +
"source paths present");
}
- option = new DistCpOptions(sourcePaths, targetPath);
+ return new DistCpOptions.Builder(sourcePaths, targetPath);
}
- return option;
}
private static String getVal(CommandLine command, String swtch) {
- String optionValue = command.getOptionValue(swtch);
+ if (swtch == null) {
+ return null;
+ }
+ String optionValue = command.getOptionValue(swtch.trim());
if (optionValue == null) {
return null;
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDFS-11571. Typo in DataStorage
exception message (Contributed by Anna Budai via Daniel Templeton)
Posted by ae...@apache.org.
HDFS-11571. Typo in DataStorage exception message (Contributed by Anna Budai via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15e3873d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15e3873d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15e3873d
Branch: refs/heads/HDFS-7240
Commit: 15e3873dc3d46016344887570e5d4aa8d20ca264
Parents: 523f467
Author: Daniel Templeton <te...@apache.org>
Authored: Wed Mar 29 10:56:36 2017 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Wed Mar 29 10:56:36 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java | 2 +-
.../org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15e3873d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 57ef1bc..835643b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -543,7 +543,7 @@ public class DataStorage extends Storage {
void recoverTransitionRead(DataNode datanode, NamespaceInfo nsInfo,
Collection<StorageLocation> dataDirs, StartupOption startOpt) throws IOException {
if (addStorageLocations(datanode, nsInfo, dataDirs, startOpt).isEmpty()) {
- throw new IOException("All specified directories are failed to load.");
+ throw new IOException("All specified directories have failed to load.");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15e3873d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
index c56a01c..6c49451 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
@@ -200,7 +200,7 @@ public class TestDataStorage {
fail("An IOException should throw: all StorageLocations are NON_EXISTENT");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains(
- "All specified directories are failed to load.", e);
+ "All specified directories have failed to load.", e);
}
assertEquals(0, storage.getNumStorageDirs());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: MAPREDUCE-6862. Fragments are not
handled correctly by resource limit checking. (Chris Trezzo via mingma)
Posted by ae...@apache.org.
MAPREDUCE-6862. Fragments are not handled correctly by resource limit checking. (Chris Trezzo via mingma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ceab00ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ceab00ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ceab00ac
Branch: refs/heads/HDFS-7240
Commit: ceab00ac62f8057a07b4b936799e6f04271e6e41
Parents: 6a5516c
Author: Ming Ma <mi...@apache.org>
Authored: Wed Mar 29 17:41:58 2017 -0700
Committer: Ming Ma <mi...@apache.org>
Committed: Wed Mar 29 17:41:58 2017 -0700
----------------------------------------------------------------------
.../hadoop/mapreduce/JobResourceUploader.java | 36 ++++++++++++------
.../mapreduce/TestJobResourceUploader.java | 40 +++++++++++++++++---
2 files changed, 59 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceab00ac/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index 4c48ff4..085c966 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -238,28 +238,42 @@ class JobResourceUploader {
Collection<String> dcArchives =
conf.getStringCollection(MRJobConfig.CACHE_ARCHIVES);
- for (String path : dcFiles) {
- explorePath(conf, new Path(path), limitChecker, statCache);
+ for (String uri : dcFiles) {
+ explorePath(conf, stringToPath(uri), limitChecker, statCache);
}
- for (String path : dcArchives) {
- explorePath(conf, new Path(path), limitChecker, statCache);
+ for (String uri : dcArchives) {
+ explorePath(conf, stringToPath(uri), limitChecker, statCache);
}
- for (String path : files) {
- explorePath(conf, new Path(path), limitChecker, statCache);
+ for (String uri : files) {
+ explorePath(conf, stringToPath(uri), limitChecker, statCache);
}
- for (String path : libjars) {
- explorePath(conf, new Path(path), limitChecker, statCache);
+ for (String uri : libjars) {
+ explorePath(conf, stringToPath(uri), limitChecker, statCache);
}
- for (String path : archives) {
- explorePath(conf, new Path(path), limitChecker, statCache);
+ for (String uri : archives) {
+ explorePath(conf, stringToPath(uri), limitChecker, statCache);
}
if (jobJar != null) {
- explorePath(conf, new Path(jobJar), limitChecker, statCache);
+ explorePath(conf, stringToPath(jobJar), limitChecker, statCache);
+ }
+ }
+
+ /**
+ * Convert a String to a Path and gracefully remove fragments/queries if they
+ * exist in the String.
+ */
+ @VisibleForTesting
+ Path stringToPath(String s) {
+ try {
+ URI uri = new URI(s);
+ return new Path(uri.getScheme(), uri.getAuthority(), uri.getPath());
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceab00ac/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java
index 36ea57a..8ba50a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java
@@ -40,6 +40,34 @@ import org.junit.Test;
public class TestJobResourceUploader {
@Test
+ public void testStringToPath() throws IOException {
+ Configuration conf = new Configuration();
+ JobResourceUploader uploader =
+ new JobResourceUploader(FileSystem.getLocal(conf), false);
+
+ Assert.assertEquals("Failed: absolute, no scheme, with fragment",
+ "/testWithFragment.txt",
+ uploader.stringToPath("/testWithFragment.txt#fragment.txt").toString());
+
+ Assert.assertEquals("Failed: absolute, with scheme, with fragment",
+ "file:/testWithFragment.txt",
+ uploader.stringToPath("file:///testWithFragment.txt#fragment.txt")
+ .toString());
+
+ Assert.assertEquals("Failed: relative, no scheme, with fragment",
+ "testWithFragment.txt",
+ uploader.stringToPath("testWithFragment.txt#fragment.txt").toString());
+
+ Assert.assertEquals("Failed: relative, no scheme, no fragment",
+ "testWithFragment.txt",
+ uploader.stringToPath("testWithFragment.txt").toString());
+
+ Assert.assertEquals("Failed: absolute, with scheme, no fragment",
+ "file:/testWithFragment.txt",
+ uploader.stringToPath("file:///testWithFragment.txt").toString());
+ }
+
+ @Test
public void testAllDefaults() throws IOException {
ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder();
runLimitsTest(b.build(), true, null);
@@ -210,17 +238,17 @@ public class TestJobResourceUploader {
rlConf.maxSingleResourceMB);
conf.set("tmpfiles",
- buildPathString("file://tmpFiles", rlConf.numOfTmpFiles));
+ buildPathString("file:///tmpFiles", rlConf.numOfTmpFiles));
conf.set("tmpjars",
- buildPathString("file://tmpjars", rlConf.numOfTmpLibJars));
+ buildPathString("file:///tmpjars", rlConf.numOfTmpLibJars));
conf.set("tmparchives",
- buildPathString("file://tmpArchives", rlConf.numOfTmpArchives));
+ buildPathString("file:///tmpArchives", rlConf.numOfTmpArchives));
conf.set(MRJobConfig.CACHE_ARCHIVES,
- buildPathString("file://cacheArchives", rlConf.numOfDCArchives));
+ buildPathString("file:///cacheArchives", rlConf.numOfDCArchives));
conf.set(MRJobConfig.CACHE_FILES,
- buildPathString("file://cacheFiles", rlConf.numOfDCFiles));
+ buildPathString("file:///cacheFiles", rlConf.numOfDCFiles));
if (rlConf.jobJar) {
- conf.setJar("file://jobjar.jar");
+ conf.setJar("file:///jobjar.jar");
}
return conf;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: YARN-6414. ATSv2 HBase related
tests fail due to guava version upgrade (Haibo Chen via Varun Saxena)
Posted by ae...@apache.org.
YARN-6414. ATSv2 HBase related tests fail due to guava version upgrade (Haibo Chen via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8838578
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8838578
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8838578
Branch: refs/heads/HDFS-7240
Commit: b8838578ce1f1ae1f93cc6b40cc98b58321ad9ee
Parents: 26172a9
Author: Varun Saxena <va...@apache.org>
Authored: Sun Apr 2 04:37:34 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Sun Apr 2 04:37:34 2017 +0530
----------------------------------------------------------------------
hadoop-project/pom.xml | 1 +
.../pom.xml | 19 +++++++++++++++++++
2 files changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8838578/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index e55308f..f327933 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -51,6 +51,7 @@
<kafka.version>0.8.2.1</kafka.version>
<hbase.version>1.2.4</hbase.version>
<hbase-compatible-hadoop.version>2.5.1</hbase-compatible-hadoop.version>
+ <hbase-compatible-guava.version>11.0.2</hbase-compatible-guava.version>
<hadoop.assemblies.version>${project.version}</hadoop.assemblies.version>
<commons-daemon.version>1.0.13</commons-daemon.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8838578/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
index d44aa22..afe440f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -141,6 +141,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
+ <version>${hbase-compatible-guava.version}</version>
<scope>test</scope>
</dependency>
@@ -369,6 +370,24 @@
</additionnalDependencies>
</configuration>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>depcheck</id>
+ <configuration>
+ <!-- disable dependency convergence check -->
+ <skip>true</skip>
+ </configuration>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <phase/>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: MAPREDUCE-6824.
TaskAttemptImpl#createCommonContainerLaunchContext is longer than 150 lines.
Contributed by Chris Trezzo.
Posted by ae...@apache.org.
MAPREDUCE-6824. TaskAttemptImpl#createCommonContainerLaunchContext is longer than 150 lines. Contributed by Chris Trezzo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/845529b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/845529b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/845529b3
Branch: refs/heads/HDFS-7240
Commit: 845529b3ab338e759665a687eb525fb2cccde7bf
Parents: a4b5aa8
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Apr 3 13:06:24 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Apr 3 13:06:54 2017 +0900
----------------------------------------------------------------------
.../v2/app/job/impl/TaskAttemptImpl.java | 285 ++++++++++---------
1 file changed, 153 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/845529b3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 4305824..9ea1b9a 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -755,7 +755,7 @@ public abstract class TaskAttemptImpl implements
new HashMap<String, LocalResource>();
// Application environment
- Map<String, String> environment = new HashMap<String, String>();
+ Map<String, String> environment;
// Service data
Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
@@ -763,157 +763,178 @@ public abstract class TaskAttemptImpl implements
// Tokens
ByteBuffer taskCredentialsBuffer = ByteBuffer.wrap(new byte[]{});
try {
- FileSystem remoteFS = FileSystem.get(conf);
-
- // //////////// Set up JobJar to be localized properly on the remote NM.
- String jobJar = conf.get(MRJobConfig.JAR);
- if (jobJar != null) {
- final Path jobJarPath = new Path(jobJar);
- final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf);
- Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(),
- jobJarFs.getWorkingDirectory());
- LocalResource rc = createLocalResource(jobJarFs, remoteJobJar,
- LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
- String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
- JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
- rc.setPattern(pattern);
- localResources.put(MRJobConfig.JOB_JAR, rc);
- LOG.info("The job-jar file on the remote FS is "
- + remoteJobJar.toUri().toASCIIString());
- } else {
- // Job jar may be null. For e.g, for pipes, the job jar is the hadoop
- // mapreduce jar itself which is already on the classpath.
- LOG.info("Job jar is not present. "
- + "Not adding any jar to the list of resources.");
- }
- // //////////// End of JobJar setup
-
- // //////////// Set up JobConf to be localized properly on the remote NM.
- Path path =
- MRApps.getStagingAreaDir(conf, UserGroupInformation
- .getCurrentUser().getShortUserName());
- Path remoteJobSubmitDir =
- new Path(path, oldJobId.toString());
- Path remoteJobConfPath =
- new Path(remoteJobSubmitDir, MRJobConfig.JOB_CONF_FILE);
- localResources.put(
- MRJobConfig.JOB_CONF_FILE,
- createLocalResource(remoteFS, remoteJobConfPath,
- LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
- LOG.info("The job-conf file on the remote FS is "
- + remoteJobConfPath.toUri().toASCIIString());
- // //////////// End of JobConf setup
- // Setup DistributedCache
- MRApps.setupDistributedCache(conf, localResources);
+ configureJobJar(conf, localResources);
- // Setup up task credentials buffer
- LOG.info("Adding #" + credentials.numberOfTokens()
- + " tokens and #" + credentials.numberOfSecretKeys()
- + " secret keys for NM use for launching container");
- Credentials taskCredentials = new Credentials(credentials);
+ configureJobConf(conf, localResources, oldJobId);
- // LocalStorageToken is needed irrespective of whether security is enabled
- // or not.
- TokenCache.setJobToken(jobToken, taskCredentials);
+ // Setup DistributedCache
+ MRApps.setupDistributedCache(conf, localResources);
- DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
- LOG.info("Size of containertokens_dob is "
- + taskCredentials.numberOfTokens());
- taskCredentials.writeTokenStorageToStream(containerTokens_dob);
taskCredentialsBuffer =
- ByteBuffer.wrap(containerTokens_dob.getData(), 0,
- containerTokens_dob.getLength());
-
- // Add shuffle secret key
- // The secret key is converted to a JobToken to preserve backwards
- // compatibility with an older ShuffleHandler running on an NM.
- LOG.info("Putting shuffle token in serviceData");
- byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials);
- if (shuffleSecret == null) {
- LOG.warn("Cannot locate shuffle secret in credentials."
- + " Using job token as shuffle secret.");
- shuffleSecret = jobToken.getPassword();
- }
- Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>(
- jobToken.getIdentifier(), shuffleSecret, jobToken.getKind(),
- jobToken.getService());
- serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
- ShuffleHandler.serializeServiceData(shuffleToken));
-
- // add external shuffle-providers - if any
- Collection<String> shuffleProviders = conf.getStringCollection(
- MRJobConfig.MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES);
- if (! shuffleProviders.isEmpty()) {
- Collection<String> auxNames = conf.getStringCollection(
- YarnConfiguration.NM_AUX_SERVICES);
-
- for (final String shuffleProvider : shuffleProviders) {
- if (shuffleProvider.equals(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID)) {
- continue; // skip built-in shuffle-provider that was already inserted with shuffle secret key
- }
- if (auxNames.contains(shuffleProvider)) {
- LOG.info("Adding ShuffleProvider Service: " + shuffleProvider + " to serviceData");
- // This only serves for INIT_APP notifications
- // The shuffle service needs to be able to work with the host:port information provided by the AM
- // (i.e. shuffle services which require custom location / other configuration are not supported)
- serviceData.put(shuffleProvider, ByteBuffer.allocate(0));
- }
- else {
- throw new YarnRuntimeException("ShuffleProvider Service: " + shuffleProvider +
- " was NOT found in the list of aux-services that are available in this NM." +
- " You may need to specify this ShuffleProvider as an aux-service in your yarn-site.xml");
- }
- }
- }
+ configureTokens(jobToken, credentials, serviceData);
- MRApps.addToEnvironment(
- environment,
- Environment.CLASSPATH.name(),
- getInitialClasspath(conf), conf);
+ addExternalShuffleProviders(conf, serviceData);
+
+ environment = configureEnv(conf);
- if (initialAppClasspath != null) {
- MRApps.addToEnvironment(
- environment,
- Environment.APP_CLASSPATH.name(),
- initialAppClasspath, conf);
- }
} catch (IOException e) {
throw new YarnRuntimeException(e);
}
- // Shell
- environment.put(
- Environment.SHELL.name(),
- conf.get(
- MRJobConfig.MAPRED_ADMIN_USER_SHELL,
- MRJobConfig.DEFAULT_SHELL)
- );
-
- // Add pwd to LD_LIBRARY_PATH, add this before adding anything else
- MRApps.addToEnvironment(
- environment,
- Environment.LD_LIBRARY_PATH.name(),
- MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
-
- // Add the env variables passed by the admin
- MRApps.setEnvFromInputString(
- environment,
- conf.get(
- MRJobConfig.MAPRED_ADMIN_USER_ENV,
- MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf
- );
-
// Construct the actual Container
// The null fields are per-container and will be constructed for each
// container separately.
ContainerLaunchContext container =
ContainerLaunchContext.newInstance(localResources, environment, null,
- serviceData, taskCredentialsBuffer, applicationACLs);
+ serviceData, taskCredentialsBuffer, applicationACLs);
return container;
}
+ private static Map<String, String> configureEnv(Configuration conf)
+ throws IOException {
+ Map<String, String> environment = new HashMap<String, String>();
+ MRApps.addToEnvironment(environment, Environment.CLASSPATH.name(),
+ getInitialClasspath(conf), conf);
+
+ if (initialAppClasspath != null) {
+ MRApps.addToEnvironment(environment, Environment.APP_CLASSPATH.name(),
+ initialAppClasspath, conf);
+ }
+
+ // Shell
+ environment.put(Environment.SHELL.name(), conf
+ .get(MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));
+
+ // Add pwd to LD_LIBRARY_PATH, add this before adding anything else
+ MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
+ MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
+
+ // Add the env variables passed by the admin
+ MRApps.setEnvFromInputString(environment,
+ conf.get(MRJobConfig.MAPRED_ADMIN_USER_ENV,
+ MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV),
+ conf);
+ return environment;
+ }
+
+ private static void configureJobJar(Configuration conf,
+ Map<String, LocalResource> localResources) throws IOException {
+ // Set up JobJar to be localized properly on the remote NM.
+ String jobJar = conf.get(MRJobConfig.JAR);
+ if (jobJar != null) {
+ final Path jobJarPath = new Path(jobJar);
+ final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf);
+ Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(),
+ jobJarFs.getWorkingDirectory());
+ LocalResource rc = createLocalResource(jobJarFs, remoteJobJar,
+ LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
+ String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
+ JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
+ rc.setPattern(pattern);
+ localResources.put(MRJobConfig.JOB_JAR, rc);
+ LOG.info("The job-jar file on the remote FS is "
+ + remoteJobJar.toUri().toASCIIString());
+ } else {
+ // Job jar may be null. For e.g, for pipes, the job jar is the hadoop
+ // mapreduce jar itself which is already on the classpath.
+ LOG.info("Job jar is not present. "
+ + "Not adding any jar to the list of resources.");
+ }
+ }
+
+ private static void configureJobConf(Configuration conf,
+ Map<String, LocalResource> localResources,
+ final org.apache.hadoop.mapred.JobID oldJobId) throws IOException {
+ // Set up JobConf to be localized properly on the remote NM.
+ Path path = MRApps.getStagingAreaDir(conf,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ Path remoteJobSubmitDir = new Path(path, oldJobId.toString());
+ Path remoteJobConfPath =
+ new Path(remoteJobSubmitDir, MRJobConfig.JOB_CONF_FILE);
+ FileSystem remoteFS = FileSystem.get(conf);
+ localResources.put(MRJobConfig.JOB_CONF_FILE,
+ createLocalResource(remoteFS, remoteJobConfPath, LocalResourceType.FILE,
+ LocalResourceVisibility.APPLICATION));
+ LOG.info("The job-conf file on the remote FS is "
+ + remoteJobConfPath.toUri().toASCIIString());
+ }
+
+ private static ByteBuffer configureTokens(Token<JobTokenIdentifier> jobToken,
+ Credentials credentials,
+ Map<String, ByteBuffer> serviceData) throws IOException {
+ // Setup up task credentials buffer
+ LOG.info("Adding #" + credentials.numberOfTokens() + " tokens and #"
+ + credentials.numberOfSecretKeys()
+ + " secret keys for NM use for launching container");
+ Credentials taskCredentials = new Credentials(credentials);
+
+ // LocalStorageToken is needed irrespective of whether security is enabled
+ // or not.
+ TokenCache.setJobToken(jobToken, taskCredentials);
+
+ DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
+ LOG.info(
+ "Size of containertokens_dob is " + taskCredentials.numberOfTokens());
+ taskCredentials.writeTokenStorageToStream(containerTokens_dob);
+ ByteBuffer taskCredentialsBuffer =
+ ByteBuffer.wrap(containerTokens_dob.getData(), 0,
+ containerTokens_dob.getLength());
+
+ // Add shuffle secret key
+ // The secret key is converted to a JobToken to preserve backwards
+ // compatibility with an older ShuffleHandler running on an NM.
+ LOG.info("Putting shuffle token in serviceData");
+ byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials);
+ if (shuffleSecret == null) {
+ LOG.warn("Cannot locate shuffle secret in credentials."
+ + " Using job token as shuffle secret.");
+ shuffleSecret = jobToken.getPassword();
+ }
+ Token<JobTokenIdentifier> shuffleToken =
+ new Token<JobTokenIdentifier>(jobToken.getIdentifier(), shuffleSecret,
+ jobToken.getKind(), jobToken.getService());
+ serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
+ ShuffleHandler.serializeServiceData(shuffleToken));
+ return taskCredentialsBuffer;
+ }
+
+ private static void addExternalShuffleProviders(Configuration conf,
+ Map<String, ByteBuffer> serviceData) {
+ // add external shuffle-providers - if any
+ Collection<String> shuffleProviders = conf.getStringCollection(
+ MRJobConfig.MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES);
+ if (!shuffleProviders.isEmpty()) {
+ Collection<String> auxNames =
+ conf.getStringCollection(YarnConfiguration.NM_AUX_SERVICES);
+
+ for (final String shuffleProvider : shuffleProviders) {
+ if (shuffleProvider
+ .equals(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID)) {
+ continue; // skip built-in shuffle-provider that was already inserted
+ // with shuffle secret key
+ }
+ if (auxNames.contains(shuffleProvider)) {
+ LOG.info("Adding ShuffleProvider Service: " + shuffleProvider
+ + " to serviceData");
+ // This only serves for INIT_APP notifications
+ // The shuffle service needs to be able to work with the host:port
+ // information provided by the AM
+ // (i.e. shuffle services which require custom location / other
+ // configuration are not supported)
+ serviceData.put(shuffleProvider, ByteBuffer.allocate(0));
+ } else {
+ throw new YarnRuntimeException("ShuffleProvider Service: "
+ + shuffleProvider
+ + " was NOT found in the list of aux-services that are "
+ + "available in this NM. You may need to specify this "
+ + "ShuffleProvider as an aux-service in your yarn-site.xml");
+ }
+ }
+ }
+ }
+
static ContainerLaunchContext createContainerLaunchContext(
Map<ApplicationAccessType, String> applicationACLs,
Configuration conf, Token<JobTokenIdentifier> jobToken, Task remoteTask,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-6411. Clean up the overwrite
of createDispatcher() in subclass of MockRM. Contributed by Yufei Gu
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
index b4adf48..75ef5c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
@@ -33,8 +33,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.TestAMRestart;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -65,8 +63,7 @@ public class TestNodeBlacklistingOnAMFailures {
conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED,
true);
- DrainDispatcher dispatcher = new DrainDispatcher();
- MockRM rm = startRM(conf, dispatcher);
+ MockRM rm = startRM(conf);
CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
// Register 5 nodes, so that we can blacklist atleast one if AM container
@@ -122,7 +119,7 @@ public class TestNodeBlacklistingOnAMFailures {
// Try the current node a few times
for (int i = 0; i <= 2; i++) {
currentNode.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(
"AppAttemptState should still be SCHEDULED if currentNode is "
@@ -132,7 +129,7 @@ public class TestNodeBlacklistingOnAMFailures {
// Now try the other node
otherNode.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
// Now the AM container should be allocated
MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
@@ -169,8 +166,7 @@ public class TestNodeBlacklistingOnAMFailures {
conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED,
true);
- DrainDispatcher dispatcher = new DrainDispatcher();
- MockRM rm = startRM(conf, dispatcher);
+ MockRM rm = startRM(conf);
CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
// Register 5 nodes, so that we can blacklist atleast one if AM container
@@ -227,7 +223,7 @@ public class TestNodeBlacklistingOnAMFailures {
System.out.println("New AppAttempt launched " + attempt.getAppAttemptId());
nm2.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
// Now the AM container should be allocated
MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
@@ -257,8 +253,7 @@ public class TestNodeBlacklistingOnAMFailures {
conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED,
true);
- DrainDispatcher dispatcher = new DrainDispatcher();
- MockRM rm = startRM(conf, dispatcher);
+ MockRM rm = startRM(conf);
CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
// Register 5 nodes, so that we can blacklist atleast one if AM container
@@ -319,7 +314,7 @@ public class TestNodeBlacklistingOnAMFailures {
nm3.nodeHeartbeat(true);
nm4.nodeHeartbeat(true);
nm5.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
// Now the AM container should be allocated
MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
@@ -352,8 +347,7 @@ public class TestNodeBlacklistingOnAMFailures {
1.5f);
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100);
- DrainDispatcher dispatcher = new DrainDispatcher();
- MockRM rm = startRM(conf, dispatcher);
+ MockRM rm = startRM(conf);
MockNM node =
new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService());
@@ -367,7 +361,7 @@ public class TestNodeBlacklistingOnAMFailures {
// Now the AM container should be allocated
RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm);
node.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
rm.sendAMLaunched(attempt.getAppAttemptId());
rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED);
@@ -394,7 +388,7 @@ public class TestNodeBlacklistingOnAMFailures {
.println("New AppAttempt launched " + attempt.getAppAttemptId());
node.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
rm.sendAMLaunched(attempt.getAppAttemptId());
@@ -418,20 +412,13 @@ public class TestNodeBlacklistingOnAMFailures {
rm.waitForState(amAttemptID.getApplicationId(), RMAppState.ACCEPTED);
}
- private MockRM startRM(YarnConfiguration conf,
- final DrainDispatcher dispatcher) {
-
+ private MockRM startRM(YarnConfiguration conf) {
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
- MockRM rm1 = new MockRM(conf, memStore) {
- @Override
- protected Dispatcher createDispatcher() {
- return dispatcher;
- }
- };
+ MockRM rm = new MockRM(conf, memStore);
- rm1.start();
- return rm1;
+ rm.start();
+ return rm;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java
index 5a6fe67..f746dc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.proto.YarnProtos.ReservationAllocationStateProto;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -186,9 +185,7 @@ public class TestReservationSystemWithRMHA extends RMHATestBase {
rm.registerNode("127.0.0.1:1", memory, vCores);
int attempts = 10;
do {
- DrainDispatcher dispatcher =
- (DrainDispatcher) rm1.getRMContext().getDispatcher();
- dispatcher.await();
+ rm1.drainEvents();
rm.getRMContext().getReservationSystem()
.synchronizePlan(ReservationSystemTestUtil.reservationQ, false);
if (rm.getRMContext().getReservationSystem()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
index c8baa60..f9f0b74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
@@ -31,8 +31,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
@@ -47,12 +45,10 @@ import org.junit.Test;
public class TestAMRMRPCNodeUpdates {
private MockRM rm;
- ApplicationMasterService amService = null;
- DrainDispatcher dispatcher = null;
+ private ApplicationMasterService amService;
@Before
public void setUp() {
- dispatcher = new DrainDispatcher();
this.rm = new MockRM() {
@Override
public void init(Configuration conf) {
@@ -61,12 +57,8 @@ public class TestAMRMRPCNodeUpdates {
"1.0");
super.init(conf);
}
-
- @Override
- protected Dispatcher createDispatcher() {
- return dispatcher;
- }
};
+
rm.start();
amService = rm.getApplicationMasterService();
}
@@ -80,14 +72,14 @@ public class TestAMRMRPCNodeUpdates {
private void syncNodeHeartbeat(MockNM nm, boolean health) throws Exception {
nm.nodeHeartbeat(health);
- dispatcher.await();
+ rm.drainEvents();
}
private void syncNodeLost(MockNM nm) throws Exception {
rm.sendNodeStarted(nm);
rm.waitForState(nm.getNodeId(), NodeState.RUNNING);
rm.sendNodeLost(nm);
- dispatcher.await();
+ rm.drainEvents();
}
private AllocateResponse allocate(final ApplicationAttemptId attemptId,
@@ -113,7 +105,7 @@ public class TestAMRMRPCNodeUpdates {
MockNM nm2 = rm.registerNode("127.0.0.2:1234", 10000);
MockNM nm3 = rm.registerNode("127.0.0.3:1234", 10000);
MockNM nm4 = rm.registerNode("127.0.0.4:1234", 10000);
- dispatcher.await();
+ rm.drainEvents();
RMApp app1 = rm.submitApp(2000);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
index e7c7e51..6a7325c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.event.InlineDispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -228,21 +227,16 @@ public class TestNMReconnect extends ParameterizedSchedulerTestBase {
// The node(127.0.0.1:1234) reconnected with RM. When it registered with
// RM, RM set its lastNodeHeartbeatResponse's id to 0 asynchronously. But
// the node's heartbeat come before RM succeeded setting the id to 0.
- final DrainDispatcher dispatcher = new DrainDispatcher();
- MockRM rm = new MockRM(){
- @Override
- protected Dispatcher createDispatcher() {
- return dispatcher;
- }
- };
+ MockRM rm = new MockRM();
rm.start();
+
MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
nm1.registerNode();
int i = 0;
while(i < 3) {
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
i++;
}
@@ -251,7 +245,7 @@ public class TestNMReconnect extends ParameterizedSchedulerTestBase {
nm2.registerNode();
RMNode rmNode = rm.getRMContext().getRMNodes().get(nm2.getNodeId());
nm2.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals("Node is Not in Running state.", NodeState.RUNNING,
rmNode.getState());
rm.stop();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 893f802..db31448 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index ff52efd..fd17bd9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -36,8 +36,6 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -612,24 +610,17 @@ public class TestApplicationPriority {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
- final DrainDispatcher dispatcher = new DrainDispatcher();
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
- MockRM rm1 = new MockRM(conf, memStore) {
- @Override
- protected Dispatcher createDispatcher() {
- return dispatcher;
- }
- };
+ MockRM rm1 = new MockRM(conf, memStore);
rm1.start();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 16384, rm1.getResourceTrackerService());
nm1.registerNode();
-
- dispatcher.await();
+ rm1.drainEvents();
ResourceScheduler scheduler = rm1.getRMContext().getScheduler();
LeafQueue defaultQueue =
@@ -648,7 +639,7 @@ public class TestApplicationPriority {
MockAM am2 = MockRM.launchAM(app2, rm1, nm1);
am2.registerAppAttempt();
- dispatcher.await();
+ rm1.drainEvents();
Assert.assertEquals(2, defaultQueue.getNumActiveApplications());
Assert.assertEquals(0, defaultQueue.getNumPendingApplications());
@@ -657,7 +648,7 @@ public class TestApplicationPriority {
Priority appPriority3 = Priority.newInstance(7);
RMApp app3 = rm1.submitApp(memory, appPriority3);
- dispatcher.await();
+ rm1.drainEvents();
Assert.assertEquals(2, defaultQueue.getNumActiveApplications());
Assert.assertEquals(1, defaultQueue.getNumPendingApplications());
@@ -676,14 +667,8 @@ public class TestApplicationPriority {
Assert.assertEquals(app3.getCurrentAppAttempt().getAppAttemptId(),
fcApp3.getApplicationAttemptId());
- final DrainDispatcher dispatcher1 = new DrainDispatcher();
// create new RM to represent restart and recover state
- MockRM rm2 = new MockRM(conf, memStore) {
- @Override
- protected Dispatcher createDispatcher() {
- return dispatcher1;
- }
- };
+ MockRM rm2 = new MockRM(conf, memStore);
// start new RM
rm2.start();
@@ -693,7 +678,7 @@ public class TestApplicationPriority {
// Verify RM Apps after this restart
Assert.assertEquals(3, rm2.getRMContext().getRMApps().size());
- dispatcher1.await();
+ rm2.drainEvents();
scheduler = rm2.getRMContext().getScheduler();
defaultQueue =
(LeafQueue) ((CapacityScheduler) scheduler).getQueue("default");
@@ -714,7 +699,7 @@ public class TestApplicationPriority {
// NM resync to new RM
nm1.registerNode();
- dispatcher1.await();
+ rm2.drainEvents();
// wait for activating applications
count = 50;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
index a451356..d4e7727 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
@@ -50,8 +50,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
@@ -199,7 +197,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
StartContainersResponse mockResponse = mock(StartContainersResponse.class);
when(containerManager.startContainers((StartContainersRequest) any()))
.thenReturn(mockResponse);
- final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
protected ClientRMService createClientRMService() {
@@ -209,11 +206,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
};
@Override
- protected Dispatcher createDispatcher() {
- return dispatcher;
- }
-
- @Override
protected void doSecureLogin() throws IOException {
}
};
@@ -225,11 +217,10 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
// Set up a node.
MockNM nm1 = rm.registerNode("localhost:1234", 3072);
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
-
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
final MockAM mockAM =
@@ -436,7 +427,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
StartContainersResponse mockResponse = mock(StartContainersResponse.class);
when(containerManager.startContainers((StartContainersRequest) any()))
.thenReturn(mockResponse);
- final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
protected ClientRMService createClientRMService() {
@@ -446,11 +436,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
};
@Override
- protected Dispatcher createDispatcher() {
- return dispatcher;
- }
-
- @Override
protected void doSecureLogin() throws IOException {
}
};
@@ -462,10 +447,10 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
// Set up a node.
MockNM nm1 = rm.registerNode("localhost:1234", 3072);
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
final MockAM mockAM =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-6411. Clean up the overwrite
of createDispatcher() in subclass of MockRM. Contributed by Yufei Gu
Posted by ae...@apache.org.
YARN-6411. Clean up the overwrite of createDispatcher() in subclass of MockRM. Contributed by Yufei Gu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d1fac5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d1fac5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d1fac5d
Branch: refs/heads/HDFS-7240
Commit: 4d1fac5df2508011adfc4c5d683beb00fd5ced45
Parents: 28cdc5a
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Fri Mar 31 10:05:34 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Fri Mar 31 10:05:34 2017 -0500
----------------------------------------------------------------------
.../v2/app/rm/TestRMContainerAllocator.java | 453 +++++++++----------
.../api/impl/TestAMRMClientOnRMRestart.java | 59 +--
.../server/resourcemanager/ACLsTestBase.java | 10 -
.../server/resourcemanager/RMHATestBase.java | 20 +-
.../ReservationACLsTestBase.java | 5 +-
.../resourcemanager/TestApplicationCleanup.java | 44 +-
.../TestApplicationMasterLauncher.java | 11 +-
.../TestApplicationMasterService.java | 19 +-
.../TestNodeBlacklistingOnAMFailures.java | 41 +-
.../TestReservationSystemWithRMHA.java | 5 +-
.../TestAMRMRPCNodeUpdates.java | 18 +-
.../resourcetracker/TestNMReconnect.java | 14 +-
.../rmcontainer/TestRMContainerImpl.java | 1 -
.../capacity/TestApplicationPriority.java | 29 +-
.../security/TestClientToAMTokens.java | 23 +-
15 files changed, 277 insertions(+), 475 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index e6aee6e..933bd01 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -179,21 +179,19 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -207,7 +205,7 @@ public class TestRMContainerAllocator {
MockNM nodeManager1 = rm.registerNode("h1:1234", 10240);
MockNM nodeManager2 = rm.registerNode("h2:1234", 10240);
MockNM nodeManager3 = rm.registerNode("h3:1234", 10240);
- dispatcher.await();
+ rm.drainEvents();
// create the container request
ContainerRequestEvent event1 = createReq(jobId, 1, 1024,
@@ -222,7 +220,7 @@ public class TestRMContainerAllocator {
// this tells the scheduler about the requests
// as nodes are not added, no allocations
List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
Assert.assertEquals(4, rm.getMyFifoScheduler().lastAsk.size());
@@ -234,7 +232,7 @@ public class TestRMContainerAllocator {
// this tells the scheduler about the requests
// as nodes are not added, no allocations
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
Assert.assertEquals(3, rm.getMyFifoScheduler().lastAsk.size());
@@ -242,18 +240,18 @@ public class TestRMContainerAllocator {
nodeManager1.nodeHeartbeat(true); // Node heartbeat
nodeManager2.nodeHeartbeat(true); // Node heartbeat
nodeManager3.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm.drainEvents();
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0, rm.getMyFifoScheduler().lastAsk.size());
checkAssignments(new ContainerRequestEvent[] { event1, event2, event3 },
assigned, false);
// check that the assigned container requests are cancelled
- assigned = allocator.schedule();
- dispatcher.await();
- Assert.assertEquals(5, rm.getMyFifoScheduler().lastAsk.size());
+ allocator.schedule();
+ rm.drainEvents();
+ Assert.assertEquals(5, rm.getMyFifoScheduler().lastAsk.size());
}
@Test
@@ -269,21 +267,19 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -297,7 +293,7 @@ public class TestRMContainerAllocator {
MockNM nodeManager1 = rm.registerNode("h1:1234", 3072); // can assign 2 maps
rm.registerNode("h2:1234", 10240); // wont heartbeat on node local node
MockNM nodeManager3 = rm.registerNode("h3:1234", 1536); // assign 1 map
- dispatcher.await();
+ rm.drainEvents();
// create the container requests for maps
ContainerRequestEvent event1 = createReq(jobId, 1, 1024,
@@ -313,7 +309,7 @@ public class TestRMContainerAllocator {
// this tells the scheduler about the requests
// as nodes are not added, no allocations
List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
// update resources in scheduler
@@ -323,10 +319,10 @@ public class TestRMContainerAllocator {
// Node heartbeat from node-local next. This allocates 2 node local
// containers for task1 and task2. These should be matched with those tasks.
nodeManager1.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
checkAssignments(new ContainerRequestEvent[] { event1, event2, event3 },
assigned, false);
// remove the rack-local assignment that should have happened for task3
@@ -350,21 +346,19 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -378,7 +372,7 @@ public class TestRMContainerAllocator {
MockNM nodeManager1 = rm.registerNode("h1:1234", 10240);
MockNM nodeManager2 = rm.registerNode("h2:1234", 10240);
MockNM nodeManager3 = rm.registerNode("h3:1234", 10240);
- dispatcher.await();
+ rm.drainEvents();
// create the container request
ContainerRequestEvent event1 = createReq(jobId, 1, 1024,
@@ -393,17 +387,17 @@ public class TestRMContainerAllocator {
// this tells the scheduler about the requests
// as nodes are not added, no allocations
List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
// update resources in scheduler
nodeManager1.nodeHeartbeat(true); // Node heartbeat
nodeManager2.nodeHeartbeat(true); // Node heartbeat
nodeManager3.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm.drainEvents();
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
checkAssignments(new ContainerRequestEvent[] { event1, event2 },
assigned, false);
}
@@ -416,19 +410,17 @@ public class TestRMContainerAllocator {
conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.0f);
final MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- final DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
final RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
final String host = "host1";
final MockNM nm = rm.registerNode(String.format("%s:1234", host), 2048);
nm.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
final JobId jobId = MRBuilderUtils
.newJobId(appAttemptId.getApplicationId(), 0);
final Job mockJob = mock(Job.class);
@@ -438,20 +430,20 @@ public class TestRMContainerAllocator {
final MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
appAttemptId, mockJob, SystemClock.getInstance());
// add resources to scheduler
- dispatcher.await();
+ rm.drainEvents();
// create the container request
final String[] locations = new String[] { host };
allocator.sendRequest(createReq(jobId, 0, 1024, locations, false, true));
for (int i = 0; i < 1;) {
- dispatcher.await();
+ rm.drainEvents();
i += allocator.schedule().size();
nm.nodeHeartbeat(true);
}
allocator.sendRequest(createReq(jobId, 0, 1024, locations, true, false));
while (allocator.getTaskAttemptKillEvents().size() == 0) {
- dispatcher.await();
+ rm.drainEvents();
allocator.schedule().size();
nm.nodeHeartbeat(true);
}
@@ -468,21 +460,19 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -521,21 +511,19 @@ public class TestRMContainerAllocator {
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -584,21 +572,19 @@ public class TestRMContainerAllocator {
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(8192, 8));
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -639,18 +625,16 @@ public class TestRMContainerAllocator {
conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.0f);
final MyResourceManager2 rm = new MyResourceManager2(conf);
rm.start();
- final DrainDispatcher dispatcher = (DrainDispatcher)rm.getRMContext()
- .getDispatcher();
final RMApp app = rm.submitApp(2048);
- dispatcher.await();
+ rm.drainEvents();
final String host = "host1";
final MockNM nm = rm.registerNode(String.format("%s:1234", host), 4096);
nm.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
final JobId jobId = MRBuilderUtils
.newJobId(appAttemptId.getApplicationId(), 0);
final Job mockJob = mock(Job.class);
@@ -666,14 +650,14 @@ public class TestRMContainerAllocator {
allocator.scheduleAllReduces();
allocator.makeRemoteRequest();
nm.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
allocator.sendRequest(createReq(jobId, 1, 1024, locations, false, false));
int assignedContainer;
for (assignedContainer = 0; assignedContainer < 1;) {
assignedContainer += allocator.schedule().size();
nm.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
}
// only 1 allocated container should be assigned
Assert.assertEquals(assignedContainer, 1);
@@ -773,21 +757,19 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -801,7 +783,7 @@ public class TestRMContainerAllocator {
MockNM nodeManager1 = rm.registerNode("h1:1234", 1024);
MockNM nodeManager2 = rm.registerNode("h2:1234", 10240);
MockNM nodeManager3 = rm.registerNode("h3:1234", 10240);
- dispatcher.await();
+ rm.drainEvents();
// create the container request
// send MAP request
@@ -822,17 +804,17 @@ public class TestRMContainerAllocator {
// this tells the scheduler about the requests
// as nodes are not added, no allocations
List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
// update resources in scheduler
nodeManager1.nodeHeartbeat(true); // Node heartbeat
nodeManager2.nodeHeartbeat(true); // Node heartbeat
nodeManager3.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm.drainEvents();
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
checkAssignments(new ContainerRequestEvent[] { event1, event3 },
assigned, false);
@@ -864,11 +846,6 @@ public class TestRMContainerAllocator {
}
@Override
- protected Dispatcher createDispatcher() {
- return new DrainDispatcher();
- }
-
- @Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
// Dispatch inline for test sanity
return new EventHandler<SchedulerEvent>() {
@@ -912,16 +889,16 @@ public class TestRMContainerAllocator {
// Submit the application
RMApp rmApp = rm.submitApp(1024);
- rmDispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 21504);
amNodeManager.nodeHeartbeat(true);
- rmDispatcher.await();
+ rm.drainEvents();
final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- rmDispatcher.await();
+ rm.drainEvents();
MRApp mrApp = new MRApp(appAttemptId, ContainerId.newContainerId(
appAttemptId, 0), 10, 10, false, this.getClass().getName(), true, 1) {
@@ -959,11 +936,11 @@ public class TestRMContainerAllocator {
amDispatcher.await();
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
amNodeManager.nodeHeartbeat(true);
- rmDispatcher.await();
+ rm.drainEvents();
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
// Wait for all map-tasks to be running
for (Task t : job.getTasks().values()) {
@@ -973,7 +950,7 @@ public class TestRMContainerAllocator {
}
allocator.schedule(); // Send heartbeat
- rmDispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0.05f, job.getProgress(), 0.001f);
Assert.assertEquals(0.05f, rmApp.getProgress(), 0.001f);
@@ -981,14 +958,14 @@ public class TestRMContainerAllocator {
Iterator<Task> it = job.getTasks().values().iterator();
finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1);
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0.095f, job.getProgress(), 0.001f);
Assert.assertEquals(0.095f, rmApp.getProgress(), 0.001f);
// Finish off 7 more so that map-progress is 80%
finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 7);
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0.41f, job.getProgress(), 0.001f);
Assert.assertEquals(0.41f, rmApp.getProgress(), 0.001f);
@@ -996,11 +973,11 @@ public class TestRMContainerAllocator {
finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 2);
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
amNodeManager.nodeHeartbeat(true);
- rmDispatcher.await();
+ rm.drainEvents();
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
// Wait for all reduce-tasks to be running
for (Task t : job.getTasks().values()) {
@@ -1013,14 +990,14 @@ public class TestRMContainerAllocator {
finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 2);
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0.59f, job.getProgress(), 0.001f);
Assert.assertEquals(0.59f, rmApp.getProgress(), 0.001f);
// Finish off the remaining 8 reduces.
finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 8);
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
// Remaining is JobCleanup
Assert.assertEquals(0.95f, job.getProgress(), 0.001f);
Assert.assertEquals(0.95f, rmApp.getProgress(), 0.001f);
@@ -1064,16 +1041,16 @@ public class TestRMContainerAllocator {
// Submit the application
RMApp rmApp = rm.submitApp(1024);
- rmDispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 11264);
amNodeManager.nodeHeartbeat(true);
- rmDispatcher.await();
+ rm.drainEvents();
final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- rmDispatcher.await();
+ rm.drainEvents();
MRApp mrApp = new MRApp(appAttemptId, ContainerId.newContainerId(
appAttemptId, 0), 10, 0, false, this.getClass().getName(), true, 1) {
@@ -1109,11 +1086,11 @@ public class TestRMContainerAllocator {
amDispatcher.await();
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
amNodeManager.nodeHeartbeat(true);
- rmDispatcher.await();
+ rm.drainEvents();
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
// Wait for all map-tasks to be running
for (Task t : job.getTasks().values()) {
@@ -1121,7 +1098,7 @@ public class TestRMContainerAllocator {
}
allocator.schedule(); // Send heartbeat
- rmDispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0.05f, job.getProgress(), 0.001f);
Assert.assertEquals(0.05f, rmApp.getProgress(), 0.001f);
@@ -1130,21 +1107,21 @@ public class TestRMContainerAllocator {
// Finish off 1 map so that map-progress is 10%
finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1);
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0.14f, job.getProgress(), 0.001f);
Assert.assertEquals(0.14f, rmApp.getProgress(), 0.001f);
// Finish off 5 more map so that map-progress is 60%
finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 5);
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0.59f, job.getProgress(), 0.001f);
Assert.assertEquals(0.59f, rmApp.getProgress(), 0.001f);
// Finish off remaining map so that map-progress is 100%
finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 4);
allocator.schedule();
- rmDispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0.95f, job.getProgress(), 0.001f);
Assert.assertEquals(0.95f, rmApp.getProgress(), 0.001f);
}
@@ -1154,21 +1131,19 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
-
+ rm.drainEvents();
+
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
@@ -1177,7 +1152,7 @@ public class TestRMContainerAllocator {
// add resources to scheduler
MockNM nm1 = rm.registerNode("h1:1234", 10240);
MockNM nm2 = rm.registerNode("h2:1234", 10240);
- dispatcher.await();
+ rm.drainEvents();
// create the map container request
ContainerRequestEvent event = createReq(jobId, 1, 1024,
@@ -1193,16 +1168,16 @@ public class TestRMContainerAllocator {
// this tells the scheduler about the requests
List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(1, allocator.getJobUpdatedNodeEvents().size());
Assert.assertEquals(3, allocator.getJobUpdatedNodeEvents().get(0).getUpdatedNodes().size());
allocator.getJobUpdatedNodeEvents().clear();
// get the assignment
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(1, assigned.size());
Assert.assertEquals(nm1.getNodeId(), assigned.get(0).getContainer().getNodeId());
// no updated nodes reported
@@ -1212,11 +1187,11 @@ public class TestRMContainerAllocator {
// mark nodes bad
nm1.nodeHeartbeat(false);
nm2.nodeHeartbeat(false);
- dispatcher.await();
-
+ rm.drainEvents();
+
// schedule response returns updated nodes
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0, assigned.size());
// updated nodes are reported
Assert.assertEquals(1, allocator.getJobUpdatedNodeEvents().size());
@@ -1227,7 +1202,7 @@ public class TestRMContainerAllocator {
allocator.getTaskAttemptKillEvents().clear();
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals(0, assigned.size());
// no updated nodes reported
Assert.assertTrue(allocator.getJobUpdatedNodeEvents().isEmpty());
@@ -1247,21 +1222,19 @@ public class TestRMContainerAllocator {
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -1275,7 +1248,7 @@ public class TestRMContainerAllocator {
MockNM nodeManager1 = rm.registerNode("h1:1234", 10240);
MockNM nodeManager2 = rm.registerNode("h2:1234", 10240);
MockNM nodeManager3 = rm.registerNode("h3:1234", 10240);
- dispatcher.await();
+ rm.drainEvents();
// create the container request
ContainerRequestEvent event1 = createReq(jobId, 1, 1024,
@@ -1295,7 +1268,7 @@ public class TestRMContainerAllocator {
// this tells the scheduler about the requests
// as nodes are not added, no allocations
List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
// Send events to blacklist nodes h1 and h2
@@ -1307,28 +1280,28 @@ public class TestRMContainerAllocator {
// update resources in scheduler
nodeManager1.nodeHeartbeat(true); // Node heartbeat
nodeManager2.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm.drainEvents();
assigned = allocator.schedule();
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
assertBlacklistAdditionsAndRemovals(2, 0, rm);
// mark h1/h2 as bad nodes
nodeManager1.nodeHeartbeat(false);
nodeManager2.nodeHeartbeat(false);
- dispatcher.await();
+ rm.drainEvents();
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
assertBlacklistAdditionsAndRemovals(0, 0, rm);
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
nodeManager3.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm.drainEvents();
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
assertBlacklistAdditionsAndRemovals(0, 0, rm);
Assert.assertTrue("No of assignments must be 3", assigned.size() == 3);
@@ -1352,24 +1325,22 @@ public class TestRMContainerAllocator {
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher =
- (DrainDispatcher) rm.getRMContext().getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM[] nodeManagers = new MockNM[10];
int nmNum = 0;
List<TaskAttemptContainerAssignedEvent> assigned = null;
- nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher);
+ nodeManagers[nmNum] = registerNodeManager(nmNum++, rm);
nodeManagers[0].nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId =
app.getCurrentAppAttempt().getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -1382,7 +1353,7 @@ public class TestRMContainerAllocator {
// Known=1, blacklisted=0, ignore should be false - assign first container
assigned =
getContainerOnHost(jobId, 1, 1024, new String[] { "h1" },
- nodeManagers[0], dispatcher, allocator, 0, 0, 0, 0, rm);
+ nodeManagers[0], allocator, 0, 0, 0, 0, rm);
Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
LOG.info("Failing container _1 on H1 (Node should be blacklisted and"
@@ -1397,47 +1368,47 @@ public class TestRMContainerAllocator {
// The current call will send blacklisted node "h1" to RM
assigned =
getContainerOnHost(jobId, 2, 1024, new String[] { "h1" },
- nodeManagers[0], dispatcher, allocator, 1, 0, 0, 1, rm);
+ nodeManagers[0], allocator, 1, 0, 0, 1, rm);
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
// Known=1, blacklisted=1, ignore should be true - assign 1
assigned =
getContainerOnHost(jobId, 2, 1024, new String[] { "h1" },
- nodeManagers[0], dispatcher, allocator, 0, 0, 0, 0, rm);
+ nodeManagers[0], allocator, 0, 0, 0, 0, rm);
Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
- nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher);
+ nodeManagers[nmNum] = registerNodeManager(nmNum++, rm);
// Known=2, blacklisted=1, ignore should be true - assign 1 anyway.
assigned =
getContainerOnHost(jobId, 3, 1024, new String[] { "h2" },
- nodeManagers[1], dispatcher, allocator, 0, 0, 0, 0, rm);
+ nodeManagers[1], allocator, 0, 0, 0, 0, rm);
Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
- nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher);
+ nodeManagers[nmNum] = registerNodeManager(nmNum++, rm);
// Known=3, blacklisted=1, ignore should be true - assign 1 anyway.
assigned =
getContainerOnHost(jobId, 4, 1024, new String[] { "h3" },
- nodeManagers[2], dispatcher, allocator, 0, 0, 0, 0, rm);
+ nodeManagers[2], allocator, 0, 0, 0, 0, rm);
Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
// Known=3, blacklisted=1, ignore should be true - assign 1
assigned =
getContainerOnHost(jobId, 5, 1024, new String[] { "h1" },
- nodeManagers[0], dispatcher, allocator, 0, 0, 0, 0, rm);
+ nodeManagers[0], allocator, 0, 0, 0, 0, rm);
Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
- nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher);
+ nodeManagers[nmNum] = registerNodeManager(nmNum++, rm);
// Known=4, blacklisted=1, ignore should be false - assign 1 anyway
assigned =
getContainerOnHost(jobId, 6, 1024, new String[] { "h4" },
- nodeManagers[3], dispatcher, allocator, 0, 0, 1, 0, rm);
+ nodeManagers[3], allocator, 0, 0, 1, 0, rm);
Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
// Test blacklisting re-enabled.
// Known=4, blacklisted=1, ignore should be false - no assignment on h1
assigned =
getContainerOnHost(jobId, 7, 1024, new String[] { "h1" },
- nodeManagers[0], dispatcher, allocator, 0, 0, 0, 0, rm);
+ nodeManagers[0], allocator, 0, 0, 0, 0, rm);
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
// RMContainerRequestor would have created a replacement request.
@@ -1450,61 +1421,61 @@ public class TestRMContainerAllocator {
// container for the same reason above.
assigned =
getContainerOnHost(jobId, 8, 1024, new String[] { "h1" },
- nodeManagers[0], dispatcher, allocator, 1, 0, 0, 2, rm);
+ nodeManagers[0], allocator, 1, 0, 0, 2, rm);
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
// Known=4, blacklisted=2, ignore should be true. Should assign 2
// containers.
assigned =
getContainerOnHost(jobId, 8, 1024, new String[] { "h1" },
- nodeManagers[0], dispatcher, allocator, 0, 0, 0, 0, rm);
+ nodeManagers[0], allocator, 0, 0, 0, 0, rm);
Assert.assertEquals("No of assignments must be 2", 2, assigned.size());
// Known=4, blacklisted=2, ignore should be true.
assigned =
getContainerOnHost(jobId, 9, 1024, new String[] { "h2" },
- nodeManagers[1], dispatcher, allocator, 0, 0, 0, 0, rm);
+ nodeManagers[1], allocator, 0, 0, 0, 0, rm);
Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
// Test blacklist while ignore blacklisting enabled
ContainerFailedEvent f3 = createFailEvent(jobId, 4, "h3", false);
allocator.sendFailure(f3);
- nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher);
+ nodeManagers[nmNum] = registerNodeManager(nmNum++, rm);
// Known=5, blacklisted=3, ignore should be true.
assigned =
getContainerOnHost(jobId, 10, 1024, new String[] { "h3" },
- nodeManagers[2], dispatcher, allocator, 0, 0, 0, 0, rm);
+ nodeManagers[2], allocator, 0, 0, 0, 0, rm);
Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
// Assign on 5 more nodes - to re-enable blacklisting
for (int i = 0; i < 5; i++) {
- nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher);
+ nodeManagers[nmNum] = registerNodeManager(nmNum++, rm);
assigned =
getContainerOnHost(jobId, 11 + i, 1024,
new String[] { String.valueOf(5 + i) }, nodeManagers[4 + i],
- dispatcher, allocator, 0, 0, (i == 4 ? 3 : 0), 0, rm);
+ allocator, 0, 0, (i == 4 ? 3 : 0), 0, rm);
Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
}
// Test h3 (blacklisted while ignoring blacklisting) is blacklisted.
assigned =
getContainerOnHost(jobId, 20, 1024, new String[] { "h3" },
- nodeManagers[2], dispatcher, allocator, 0, 0, 0, 0, rm);
+ nodeManagers[2], allocator, 0, 0, 0, 0, rm);
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
}
- private MockNM registerNodeManager(int i, MyResourceManager rm,
- DrainDispatcher dispatcher) throws Exception {
+ private MockNM registerNodeManager(int i, MyResourceManager rm)
+ throws Exception {
MockNM nm = rm.registerNode("h" + (i + 1) + ":1234", 10240);
- dispatcher.await();
+ rm.drainEvents();
return nm;
}
private
List<TaskAttemptContainerAssignedEvent> getContainerOnHost(JobId jobId,
int taskAttemptId, int memory, String[] hosts, MockNM mockNM,
- DrainDispatcher dispatcher, MyContainerAllocator allocator,
+ MyContainerAllocator allocator,
int expectedAdditions1, int expectedRemovals1,
int expectedAdditions2, int expectedRemovals2, MyResourceManager rm)
throws Exception {
@@ -1514,17 +1485,17 @@ public class TestRMContainerAllocator {
// Send the request to the RM
List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
assertBlacklistAdditionsAndRemovals(
expectedAdditions1, expectedRemovals1, rm);
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
// Heartbeat from the required nodeManager
mockNM.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
assertBlacklistAdditionsAndRemovals(
expectedAdditions2, expectedRemovals2, rm);
return assigned;
@@ -1542,21 +1513,19 @@ public class TestRMContainerAllocator {
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -1569,7 +1538,7 @@ public class TestRMContainerAllocator {
// add resources to scheduler
MockNM nodeManager1 = rm.registerNode("h1:1234", 10240);
MockNM nodeManager3 = rm.registerNode("h3:1234", 10240);
- dispatcher.await();
+ rm.drainEvents();
LOG.info("Requesting 1 Containers _1 on H1");
// create the container request
@@ -1581,17 +1550,17 @@ public class TestRMContainerAllocator {
// this tells the scheduler about the requests
// as nodes are not added, no allocations
List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
LOG.info("h1 Heartbeat (To actually schedule the containers)");
// update resources in scheduler
nodeManager1.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm.drainEvents();
LOG.info("RM Heartbeat (To process the scheduled containers)");
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
assertBlacklistAdditionsAndRemovals(0, 0, rm);
Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
@@ -1608,7 +1577,7 @@ public class TestRMContainerAllocator {
//Update the Scheduler with the new requests.
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
assertBlacklistAdditionsAndRemovals(1, 0, rm);
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
@@ -1623,11 +1592,11 @@ public class TestRMContainerAllocator {
LOG.info("h1 Heartbeat (To actually schedule the containers)");
// update resources in scheduler
nodeManager1.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm.drainEvents();
LOG.info("RM Heartbeat (To process the scheduled containers)");
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
assertBlacklistAdditionsAndRemovals(0, 0, rm);
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
@@ -1636,19 +1605,19 @@ public class TestRMContainerAllocator {
//Send a release for the p:5 container + another request.
LOG.info("RM Heartbeat (To process the re-scheduled containers)");
assigned = allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
assertBlacklistAdditionsAndRemovals(0, 0, rm);
Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
//Hearbeat from H3 to schedule on this host.
LOG.info("h3 Heartbeat (To re-schedule the containers)");
nodeManager3.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm.drainEvents();
LOG.info("RM Heartbeat (To process the re-scheduled containers for H3)");
assigned = allocator.schedule();
assertBlacklistAdditionsAndRemovals(0, 0, rm);
- dispatcher.await();
+ rm.drainEvents();
// For debugging
for (TaskAttemptContainerAssignedEvent assig : assigned) {
@@ -2229,22 +2198,20 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
final MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
// Make a node to register so as to launch the AM.
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job job = mock(Job.class);
@@ -2381,21 +2348,19 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
final MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher rmDispatcher =
- (DrainDispatcher) rm.getRMContext().getDispatcher();
// Submit the application
RMApp rmApp = rm.submitApp(1024);
- rmDispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 11264);
amNodeManager.nodeHeartbeat(true);
- rmDispatcher.await();
+ rm.drainEvents();
final ApplicationAttemptId appAttemptId =
rmApp.getCurrentAppAttempt().getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- rmDispatcher.await();
+ rm.drainEvents();
MRApp mrApp =
new MRApp(appAttemptId, ContainerId.newContainerId(appAttemptId, 0), 10,
@@ -2454,22 +2419,20 @@ public class TestRMContainerAllocator {
MyResourceManager rm1 = new MyResourceManager(conf, memStore);
rm1.start();
- DrainDispatcher dispatcher =
- (DrainDispatcher) rm1.getRMContext().getDispatcher();
// Submit the application
RMApp app = rm1.submitApp(1024);
- dispatcher.await();
+ rm1.drainEvents();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode();
nm1.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm1.drainEvents();
ApplicationAttemptId appAttemptId =
app.getCurrentAppAttempt().getAppAttemptId();
rm1.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm1.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -2498,7 +2461,7 @@ public class TestRMContainerAllocator {
// send allocate request and 1 blacklisted nodes
List<TaskAttemptContainerAssignedEvent> assignedContainers =
allocator.schedule();
- dispatcher.await();
+ rm1.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0,
assignedContainers.size());
// Why ask is 3, not 4? --> ask from blacklisted node h2 is removed
@@ -2506,11 +2469,11 @@ public class TestRMContainerAllocator {
assertBlacklistAdditionsAndRemovals(1, 0, rm1);
nm1.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm1.drainEvents();
// Step-2 : 2 containers are allocated by RM.
assignedContainers = allocator.schedule();
- dispatcher.await();
+ rm1.drainEvents();
Assert.assertEquals("No of assignments must be 2", 2,
assignedContainers.size());
assertAsksAndReleases(0, 0, rm1);
@@ -2545,7 +2508,6 @@ public class TestRMContainerAllocator {
rm2.start();
nm1.setResourceTrackerService(rm2.getResourceTrackerService());
allocator.updateSchedulerProxy(rm2);
- dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
// NM should be rebooted on heartbeat, even first heartbeat for nm2
NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
@@ -2555,7 +2517,7 @@ public class TestRMContainerAllocator {
nm1 = new MockNM("h1:1234", 10240, rm2.getResourceTrackerService());
nm1.registerNode();
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm2.drainEvents();
// Step-4 : On RM restart, AM(does not know RM is restarted) sends
// additional containerRequest(event4) and blacklisted nodes.
@@ -2576,7 +2538,7 @@ public class TestRMContainerAllocator {
// send allocate request to 2nd RM and get resync command
allocator.schedule();
- dispatcher.await();
+ rm2.drainEvents();
// Step-5 : On Resync,AM sends all outstanding
// asks,release,blacklistAaddition
@@ -2587,16 +2549,16 @@ public class TestRMContainerAllocator {
// send all outstanding request again.
assignedContainers = allocator.schedule();
- dispatcher.await();
+ rm2.drainEvents();
assertAsksAndReleases(3, 2, rm2);
assertBlacklistAdditionsAndRemovals(2, 0, rm2);
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm2.drainEvents();
// Step-6 : RM allocates containers i.e event3,event4 and cRequest5
assignedContainers = allocator.schedule();
- dispatcher.await();
+ rm2.drainEvents();
Assert.assertEquals("Number of container should be 3", 3,
assignedContainers.size());
@@ -2699,20 +2661,19 @@ public class TestRMContainerAllocator {
MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS, 0);
MyResourceManager rm1 = new MyResourceManager(conf);
rm1.start();
- DrainDispatcher dispatcher =
- (DrainDispatcher) rm1.getRMContext().getDispatcher();
+
RMApp app = rm1.submitApp(1024);
- dispatcher.await();
+ rm1.drainEvents();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode();
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm1.drainEvents();
ApplicationAttemptId appAttemptId =
app.getCurrentAppAttempt().getAppAttemptId();
rm1.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm1.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -2728,7 +2689,7 @@ public class TestRMContainerAllocator {
} catch (RMContainerAllocationException e) {
Assert.assertTrue(e.getMessage().contains("Could not contact RM after"));
}
- dispatcher.await();
+ rm1.drainEvents();
Assert.assertEquals("Should Have 1 Job Event", 1,
allocator.jobEvents.size());
JobEvent event = allocator.jobEvents.get(0);
@@ -2750,22 +2711,20 @@ public class TestRMContainerAllocator {
rm.start();
AMRMTokenSecretManager secretMgr =
rm.getRMContext().getAMRMTokenSecretManager();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
final ApplicationId appId = app.getApplicationId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
final Job mockJob = mock(Job.class);
@@ -2953,21 +2912,19 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -2989,21 +2946,19 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher =
- (DrainDispatcher) rm.getRMContext().getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 1260);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -3018,7 +2973,7 @@ public class TestRMContainerAllocator {
// Register nodes to RM.
MockNM nodeManager = rm.registerNode("h1:1234", 1024);
- dispatcher.await();
+ rm.drainEvents();
// Request 2 maps and 1 reducer(sone on nodes which are not registered).
ContainerRequestEvent event1 =
@@ -3034,7 +2989,7 @@ public class TestRMContainerAllocator {
// This will tell the scheduler about the requests but there will be no
// allocations as nodes are not added.
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// Advance clock so that maps can be considered as hanging.
clock.setTime(System.currentTimeMillis() + 500000L);
@@ -3045,15 +3000,15 @@ public class TestRMContainerAllocator {
allocator.sendRequest(event4);
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// Update resources in scheduler through node heartbeat from h1.
nodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(1024, 1));
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// One map is assigned.
Assert.assertEquals(1, allocator.getAssignedRequests().maps.size());
@@ -3087,7 +3042,7 @@ public class TestRMContainerAllocator {
// On next allocate request to scheduler, headroom reported will be 0.
rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(0, 0));
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// After allocate response from scheduler, all scheduled reduces are ramped
// down and move to pending. 3 asks are also updated with 0 containers to
// indicate ramping down of reduces to scheduler.
@@ -3155,21 +3110,19 @@ public class TestRMContainerAllocator {
Configuration conf = new Configuration();
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher =
- (DrainDispatcher) rm.getRMContext().getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 1260);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -3184,7 +3137,7 @@ public class TestRMContainerAllocator {
// Register nodes to RM.
MockNM nodeManager = rm.registerNode("h1:1234", 1024);
- dispatcher.await();
+ rm.drainEvents();
// Request 2 maps and 1 reducer(sone on nodes which are not registered).
ContainerRequestEvent event1 =
@@ -3200,7 +3153,7 @@ public class TestRMContainerAllocator {
// This will tell the scheduler about the requests but there will be no
// allocations as nodes are not added.
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// Advance clock so that maps can be considered as hanging.
clock.setTime(System.currentTimeMillis() + 500000L);
@@ -3211,15 +3164,15 @@ public class TestRMContainerAllocator {
allocator.sendRequest(event4);
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// Update resources in scheduler through node heartbeat from h1.
nodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(1024, 1));
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// One map is assigned.
Assert.assertEquals(1, allocator.getAssignedRequests().maps.size());
@@ -3256,7 +3209,7 @@ public class TestRMContainerAllocator {
// On next allocate request to scheduler, headroom reported will be 2048.
rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(2048, 0));
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// After allocate response from scheduler, all scheduled reduces are ramped
// down and move to pending. 3 asks are also updated with 0 containers to
// indicate ramping down of reduces to scheduler.
@@ -3285,21 +3238,19 @@ public class TestRMContainerAllocator {
conf.setInt(MRJobConfig.MR_JOB_REDUCER_UNCONDITIONAL_PREEMPT_DELAY_SEC, -1);
MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher =
- (DrainDispatcher) rm.getRMContext().getDispatcher();
// Submit the application
RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ rm.drainEvents();
MockNM amNodeManager = rm.registerNode("amNM:1234", 1260);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm.drainEvents();
JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
Job mockJob = mock(Job.class);
@@ -3315,10 +3266,10 @@ public class TestRMContainerAllocator {
appAttemptId, mockJob);
MockNM nodeManager = rm.registerNode("h1:1234", 4096);
- dispatcher.await();
+ rm.drainEvents();
// Register nodes to RM.
MockNM nodeManager2 = rm.registerNode("h2:1234", 1024);
- dispatcher.await();
+ rm.drainEvents();
// Request 2 maps and 1 reducer(sone on nodes which are not registered).
ContainerRequestEvent event1 =
@@ -3334,7 +3285,7 @@ public class TestRMContainerAllocator {
// This will tell the scheduler about the requests but there will be no
// allocations as nodes are not added.
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// Request for another reducer on h3 which has not registered.
ContainerRequestEvent event4 =
@@ -3342,15 +3293,15 @@ public class TestRMContainerAllocator {
allocator.sendRequest(event4);
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// Update resources in scheduler through node heartbeat from h1.
nodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(3072, 3));
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// Two maps are assigned.
Assert.assertEquals(2, allocator.getAssignedRequests().maps.size());
@@ -3363,15 +3314,15 @@ public class TestRMContainerAllocator {
Assert.assertEquals(0, allocator.getAssignedRequests().maps.size());
nodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(1024, 1));
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// h2 heartbeats.
nodeManager2.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
// Send request for one more mapper.
ContainerRequestEvent event5 =
@@ -3380,7 +3331,7 @@ public class TestRMContainerAllocator {
rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(2048, 2));
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// One reducer is assigned and one map is scheduled
Assert.assertEquals(1, allocator.getScheduledRequests().maps.size());
Assert.assertEquals(1, allocator.getAssignedRequests().reduces.size());
@@ -3388,7 +3339,7 @@ public class TestRMContainerAllocator {
// enough if scheduled reducers resources are deducted.
rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(1260, 2));
allocator.schedule();
- dispatcher.await();
+ rm.drainEvents();
// After allocate response, the one assigned reducer is preempted and killed
Assert.assertEquals(1, MyContainerAllocator.getTaskAttemptKillEvents().size());
Assert.assertEquals(RMContainerAllocator.RAMPDOWN_DIAGNOSTIC,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
index 39a7633..fa3c6af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
@@ -49,8 +49,6 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -126,22 +124,20 @@ public class TestAMRMClientOnRMRestart {
// Phase-1 Start 1st RM
MyResourceManager rm1 = new MyResourceManager(conf, memStore);
rm1.start();
- DrainDispatcher dispatcher =
- (DrainDispatcher) rm1.getRMContext().getDispatcher();
// Submit the application
RMApp app = rm1.submitApp(1024);
- dispatcher.await();
+ rm1.drainEvents();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode();
nm1.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm1.drainEvents();
ApplicationAttemptId appAttemptId =
app.getCurrentAppAttempt().getAppAttemptId();
rm1.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm1.drainEvents();
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token =
rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
@@ -176,7 +172,7 @@ public class TestAMRMClientOnRMRestart {
blacklistAdditions.remove("h2");// remove from local list
AllocateResponse allocateResponse = amClient.allocate(0.1f);
- dispatcher.await();
+ rm1.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
.getAllocatedContainers().size());
@@ -189,10 +185,10 @@ public class TestAMRMClientOnRMRestart {
// Step-2 : NM heart beat is sent.
// On 2nd AM allocate request, RM allocates 3 containers to AM
nm1.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm1.drainEvents();
allocateResponse = amClient.allocate(0.2f);
- dispatcher.await();
+ rm1.drainEvents();
// 3 containers are allocated i.e for cRequest1, cRequest2 and cRequest3.
Assert.assertEquals("No of assignments must be 0", 3, allocateResponse
.getAllocatedContainers().size());
@@ -207,7 +203,7 @@ public class TestAMRMClientOnRMRestart {
amClient.removeContainerRequest(cRequest3);
allocateResponse = amClient.allocate(0.2f);
- dispatcher.await();
+ rm1.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
.getAllocatedContainers().size());
assertAsksAndReleases(4, 0, rm1);
@@ -233,7 +229,7 @@ public class TestAMRMClientOnRMRestart {
// request
nm1.nodeHeartbeat(containerId.getApplicationAttemptId(),
containerId.getContainerId(), ContainerState.RUNNING);
- dispatcher.await();
+ rm1.drainEvents();
amClient.requestContainerUpdate(
container, UpdateContainerRequest.newInstance(
container.getVersion(), container.getId(),
@@ -242,7 +238,7 @@ public class TestAMRMClientOnRMRestart {
it.remove();
allocateResponse = amClient.allocate(0.3f);
- dispatcher.await();
+ rm1.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
.getAllocatedContainers().size());
assertAsksAndReleases(3, pendingRelease, rm1);
@@ -258,7 +254,6 @@ public class TestAMRMClientOnRMRestart {
rm2.start();
nm1.setResourceTrackerService(rm2.getResourceTrackerService());
((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
- dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
// NM should be rebooted on heartbeat, even first heartbeat for nm2
NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
@@ -274,7 +269,7 @@ public class TestAMRMClientOnRMRestart {
Collections.singletonList(
containerId.getApplicationAttemptId().getApplicationId()));
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm2.drainEvents();
blacklistAdditions.add("h3");
amClient.updateBlacklist(blacklistAdditions, null);
@@ -296,7 +291,7 @@ public class TestAMRMClientOnRMRestart {
// containerRequest and blacklisted nodes.
// Intern RM send resync command,AMRMClient resend allocate request
allocateResponse = amClient.allocate(0.3f);
- dispatcher.await();
+ rm2.drainEvents();
completedContainer =
allocateResponse.getCompletedContainersStatuses().size();
@@ -313,7 +308,7 @@ public class TestAMRMClientOnRMRestart {
// Step-5 : Allocater after resync command
allocateResponse = amClient.allocate(0.5f);
- dispatcher.await();
+ rm2.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
.getAllocatedContainers().size());
@@ -326,10 +321,10 @@ public class TestAMRMClientOnRMRestart {
int count = 5;
while (count-- > 0) {
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm2.drainEvents();
allocateResponse = amClient.allocate(0.5f);
- dispatcher.await();
+ rm2.drainEvents();
noAssignedContainer += allocateResponse.getAllocatedContainers().size();
if (noAssignedContainer == 3) {
break;
@@ -358,22 +353,20 @@ public class TestAMRMClientOnRMRestart {
// Phase-1 Start 1st RM
MyResourceManager rm1 = new MyResourceManager(conf, memStore);
rm1.start();
- DrainDispatcher dispatcher =
- (DrainDispatcher) rm1.getRMContext().getDispatcher();
// Submit the application
RMApp app = rm1.submitApp(1024);
- dispatcher.await();
+ rm1.drainEvents();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode();
nm1.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm1.drainEvents();
ApplicationAttemptId appAttemptId =
app.getCurrentAppAttempt().getAppAttemptId();
rm1.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm1.drainEvents();
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token =
rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
@@ -393,7 +386,6 @@ public class TestAMRMClientOnRMRestart {
rm2.start();
nm1.setResourceTrackerService(rm2.getResourceTrackerService());
((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
- dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
// NM should be rebooted on heartbeat, even first heartbeat for nm2
NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
@@ -409,7 +401,7 @@ public class TestAMRMClientOnRMRestart {
Priority.newInstance(0), 0);
nm1.registerNode(Arrays.asList(containerReport), null);
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm2.drainEvents();
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
null, null);
@@ -421,7 +413,6 @@ public class TestAMRMClientOnRMRestart {
amClient.stop();
rm1.stop();
rm2.stop();
-
}
@@ -439,22 +430,20 @@ public class TestAMRMClientOnRMRestart {
// start first RM
MyResourceManager2 rm1 = new MyResourceManager2(conf, memStore);
rm1.start();
- DrainDispatcher dispatcher =
- (DrainDispatcher) rm1.getRMContext().getDispatcher();
Long startTime = System.currentTimeMillis();
// Submit the application
RMApp app = rm1.submitApp(1024);
- dispatcher.await();
+ rm1.drainEvents();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode();
nm1.nodeHeartbeat(true); // Node heartbeat
- dispatcher.await();
+ rm1.drainEvents();
ApplicationAttemptId appAttemptId =
app.getCurrentAppAttempt().getAppAttemptId();
rm1.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rm1.drainEvents();
AMRMTokenSecretManager amrmTokenSecretManagerForRM1 =
rm1.getRMContext().getAMRMTokenSecretManager();
@@ -513,7 +502,6 @@ public class TestAMRMClientOnRMRestart {
rm2.start();
nm1.setResourceTrackerService(rm2.getResourceTrackerService());
((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
- dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
AMRMTokenSecretManager amrmTokenSecretManagerForRM2 =
rm2.getRMContext().getAMRMTokenSecretManager();
@@ -616,11 +604,6 @@ public class TestAMRMClientOnRMRestart {
}
@Override
- protected Dispatcher createDispatcher() {
- return new DrainDispatcher();
- }
-
- @Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
// Dispatch inline for test sanity
return new EventHandler<SchedulerEvent>() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
index fbd5ac3..100eb7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
@@ -30,14 +30,9 @@ import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.junit.Before;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
public abstract class ACLsTestBase {
protected static final String COMMON_USER = "common_user";
@@ -81,11 +76,6 @@ public abstract class ACLsTestBase {
}
@Override
- protected Dispatcher createDispatcher() {
- return new DrainDispatcher();
- }
-
- @Override
protected void doSecureLogin() throws IOException {
}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
index c9ce7d7..c95bcdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
@@ -26,22 +26,17 @@ import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -108,20 +103,9 @@ public abstract class RMHATestBase extends ClientBaseWithFixes{
}
protected void startRMs() throws IOException {
- rm1 = new MockRM(confForRM1, null, false, false){
- @Override
- protected Dispatcher createDispatcher() {
- return new DrainDispatcher();
- }
- };
- rm2 = new MockRM(confForRM2, null, false, false){
- @Override
- protected Dispatcher createDispatcher() {
- return new DrainDispatcher();
- }
- };
+ rm1 = new MockRM(confForRM1, null, false, false);
+ rm2 = new MockRM(confForRM2, null, false, false);
startRMs(rm1, confForRM1, rm2, confForRM2);
-
}
protected void startRMsWithCustomizedRMAppManager() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
index 03bc889..c8ee00e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.yarn.api.records.ReservationRequest;
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
import org.apache.hadoop.yarn.api.records.ReservationRequests;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -463,9 +462,7 @@ public class ReservationACLsTestBase extends ACLsTestBase {
int attempts = 10;
Collection<Plan> plans;
do {
- DrainDispatcher dispatcher =
- (DrainDispatcher) resourceManager.getRMContext().getDispatcher();
- dispatcher.await();
+ resourceManager.drainEvents();
LOG.info("Waiting for node capacity to be added to plan");
plans = resourceManager.getRMContext().getReservationSystem()
.getAllPlans().values();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
index c4197a1..422b7eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
@@ -40,8 +40,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
@@ -161,13 +159,7 @@ public class TestApplicationCleanup {
Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG);
- final DrainDispatcher dispatcher = new DrainDispatcher();
- MockRM rm = new MockRM() {
- @Override
- protected Dispatcher createDispatcher() {
- return dispatcher;
- }
- };
+ MockRM rm = new MockRM();
rm.start();
MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000);
@@ -185,8 +177,8 @@ public class TestApplicationCleanup {
int request = 2;
am.allocate("127.0.0.1" , 1000, request,
new ArrayList<ContainerId>());
- dispatcher.await();
-
+ rm.drainEvents();
+
//kick the scheduler
nm1.nodeHeartbeat(true);
List<Container> conts = am.allocate(new ArrayList<ResourceRequest>(),
@@ -199,7 +191,7 @@ public class TestApplicationCleanup {
Thread.sleep(100);
conts = am.allocate(new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>()).getAllocatedContainers();
- dispatcher.await();
+ rm.drainEvents();
contReceived += conts.size();
nm1.nodeHeartbeat(true);
}
@@ -209,7 +201,7 @@ public class TestApplicationCleanup {
ArrayList<ContainerId> release = new ArrayList<ContainerId>();
release.add(conts.get(0).getId());
am.allocate(new ArrayList<ResourceRequest>(), release);
- dispatcher.await();
+ rm.drainEvents();
// Send one more heartbeat with a fake running container. This is to
// simulate the situation that can happen if the NM reports that container
@@ -224,7 +216,7 @@ public class TestApplicationCleanup {
containerStatuses.put(app.getApplicationId(), containerStatusList);
NodeHeartbeatResponse resp = nm1.nodeHeartbeat(containerStatuses, true);
- waitForContainerCleanup(dispatcher, nm1, resp);
+ waitForContainerCleanup(rm, nm1, resp);
// Now to test the case when RM already gave cleanup, and NM suddenly
// realizes that the container is running.
@@ -240,17 +232,17 @@ public class TestApplicationCleanup {
resp = nm1.nodeHeartbeat(containerStatuses, true);
// The cleanup list won't be instantaneous as it is given out by scheduler
// and not RMNodeImpl.
- waitForContainerCleanup(dispatcher, nm1, resp);
+ waitForContainerCleanup(rm, nm1, resp);
rm.stop();
}
- protected void waitForContainerCleanup(DrainDispatcher dispatcher, MockNM nm,
+ protected void waitForContainerCleanup(MockRM rm, MockNM nm,
NodeHeartbeatResponse resp) throws Exception {
int waitCount = 0, cleanedConts = 0;
List<ContainerId> contsToClean;
do {
- dispatcher.await();
+ rm.drainEvents();
contsToClean = resp.getContainersToCleanup();
cleanedConts += contsToClean.size();
if (cleanedConts >= 1) {
@@ -400,13 +392,7 @@ public class TestApplicationCleanup {
memStore.init(conf);
// start RM
- final DrainDispatcher dispatcher = new DrainDispatcher();
- MockRM rm1 = new MockRM(conf, memStore) {
- @Override
- protected Dispatcher createDispatcher() {
- return dispatcher;
- }
- };
+ MockRM rm1 = new MockRM(conf, memStore);
rm1.start();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
@@ -419,13 +405,7 @@ public class TestApplicationCleanup {
rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING);
// start new RM
- final DrainDispatcher dispatcher2 = new DrainDispatcher();
- MockRM rm2 = new MockRM(conf, memStore) {
- @Override
- protected Dispatcher createDispatcher() {
- return dispatcher2;
- }
- };
+ MockRM rm2 = new MockRM(conf, memStore);
rm2.start();
// nm1 register to rm2, and do a heartbeat
@@ -437,7 +417,7 @@ public class TestApplicationCleanup {
NodeHeartbeatResponse response = nm1.nodeHeartbeat(am0
.getApplicationAttemptId(), 2, ContainerState.RUNNING);
- waitForContainerCleanup(dispatcher2, nm1, response);
+ waitForContainerCleanup(rm2, nm1, response);
rm1.stop();
rm2.stop();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
index 08b180f..9e84010 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
@@ -59,8 +59,6 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException;
@@ -260,7 +258,6 @@ public class TestApplicationMasterLauncher {
Configuration conf = new Configuration();
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
conf.setInt(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 1);
- final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new MockRMWithCustomAMLauncher(conf, null) {
@Override
protected ApplicationMasterLauncher createAMLauncher() {
@@ -284,12 +281,8 @@ public class TestApplicationMasterLauncher {
}
};
}
-
- @Override
- protected Dispatcher createDispatcher() {
- return dispatcher;
- }
};
+
rm.start();
MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120);
@@ -297,7 +290,7 @@ public class TestApplicationMasterLauncher {
// kick the scheduling
nm1.nodeHeartbeat(true);
- dispatcher.await();
+ rm.drainEvents();
MockRM.waitForState(app.getCurrentAppAttempt(),
RMAppAttemptState.LAUNCHED, 500);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1fac5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index 23bed22..18c49bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -42,8 +42,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
@@ -327,10 +325,8 @@ public class TestApplicationMasterService {
@Test(timeout=1200000)
public void testAllocateAfterUnregister() throws Exception {
- MyResourceManager rm = new MyResourceManager(conf);
+ MockRM rm = new MockRM(conf);
rm.start();
- DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext()
- .getDispatcher();
// Register node1
MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
@@ -351,7 +347,7 @@ public class TestApplicationMasterService {
AllocateResponse alloc1Response = am1.schedule();
nm1.nodeHeartbeat(true);
- rmDispatcher.await();
+ rm.drainEvents();
alloc1Response = am1.schedule();
Assert.assertEquals(0, alloc1Response.getAllocatedContainers().size());
}
@@ -474,17 +470,6 @@ public class TestApplicationMasterService {
rm.stop();
}
- private static class MyResourceManager extends MockRM {
-
- public MyResourceManager(YarnConfiguration conf) {
- super(conf);
- }
- @Override
- protected Dispatcher createDispatcher() {
- return new DrainDispatcher();
- }
- }
-
private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
RMContainer rmContainer = cs.getRMContainer(containerId);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HADOOP-14218. Replace assertThat
with assertTrue in MetricsAsserts.
Posted by ae...@apache.org.
HADOOP-14218. Replace assertThat with assertTrue in MetricsAsserts.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/448ec81f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/448ec81f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/448ec81f
Branch: refs/heads/HDFS-7240
Commit: 448ec81fd7133f413853570f116d5f6e16f68bd9
Parents: 9bae672
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Mar 28 14:32:40 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Mar 28 14:32:40 2017 +0900
----------------------------------------------------------------------
.../test/java/org/apache/hadoop/test/MetricsAsserts.java | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/448ec81f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java
index a7bbe84..b2f6054 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java
@@ -27,7 +27,6 @@ import static org.mockito.AdditionalMatchers.geq;
import static org.mockito.Mockito.*;
import org.mockito.stubbing.Answer;
-import org.mockito.internal.matchers.GreaterThan;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.ArgumentCaptor;
@@ -329,8 +328,8 @@ public class MetricsAsserts {
*/
public static void assertCounterGt(String name, long greater,
MetricsRecordBuilder rb) {
- Assert.assertThat("Bad value for metric " + name, getLongCounter(name, rb),
- new GreaterThan<Long>(greater));
+ Assert.assertTrue("Bad value for metric " + name,
+ getLongCounter(name, rb) > greater);
}
/**
@@ -352,8 +351,8 @@ public class MetricsAsserts {
*/
public static void assertGaugeGt(String name, double greater,
MetricsRecordBuilder rb) {
- Assert.assertThat("Bad value for metric " + name, getDoubleGauge(name, rb),
- new GreaterThan<Double>(greater));
+ Assert.assertTrue("Bad value for metric " + name,
+ getDoubleGauge(name, rb) > greater);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-14038. Rename ADLS
credential properties. Contributed by John Zhuge.
Posted by ae...@apache.org.
HADOOP-14038. Rename ADLS credential properties. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/253e3e78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/253e3e78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/253e3e78
Branch: refs/heads/HDFS-7240
Commit: 253e3e78abb9ebe0e8103d25538ff07875b33c84
Parents: 448ec81
Author: John Zhuge <jz...@cloudera.com>
Authored: Mon Feb 20 21:27:46 2017 -0800
Committer: John Zhuge <jz...@apache.org>
Committed: Mon Mar 27 23:05:57 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 15 +++-
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 38 +++++++--
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 4 +
.../src/site/markdown/index.md | 22 +++---
.../hadoop/fs/adl/TestAzureADTokenProvider.java | 5 +-
.../fs/adl/TestValidateConfiguration.java | 83 ++++++++++++++++++--
6 files changed, 138 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index ada8b02..da4f6ac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -751,7 +751,20 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
this.loadDefaults = other.loadDefaults;
setQuietMode(other.getQuietMode());
}
-
+
+ /**
+ * Reload existing configuration instances.
+ */
+ public static synchronized void reloadExistingConfigurations() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reloading " + REGISTRY.keySet().size()
+ + " existing configurations");
+ }
+ for (Configuration conf : REGISTRY.keySet()) {
+ conf.reloadConfiguration();
+ }
+ }
+
/**
* Add a default resource. Resources are loaded in the order of the resources
* added.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index 8fc8e00..d3a5565 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.adl;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.DeprecationDelta;
/**
* Constants.
@@ -28,25 +30,25 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Evolving
public final class AdlConfKeys {
// OAuth2 Common Configuration
- public static final String AZURE_AD_REFRESH_URL_KEY = "dfs.adls.oauth2"
- + ".refresh.url";
+ public static final String AZURE_AD_REFRESH_URL_KEY =
+ "fs.adl.oauth2.refresh.url";
// optional when provider type is refresh or client id.
public static final String AZURE_AD_TOKEN_PROVIDER_CLASS_KEY =
- "dfs.adls.oauth2.access.token.provider";
+ "fs.adl.oauth2.access.token.provider";
public static final String AZURE_AD_CLIENT_ID_KEY =
- "dfs.adls.oauth2.client.id";
+ "fs.adl.oauth2.client.id";
public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY =
- "dfs.adls.oauth2.access.token.provider.type";
+ "fs.adl.oauth2.access.token.provider.type";
// OAuth Refresh Token Configuration
public static final String AZURE_AD_REFRESH_TOKEN_KEY =
- "dfs.adls.oauth2.refresh.token";
+ "fs.adl.oauth2.refresh.token";
public static final String TOKEN_PROVIDER_TYPE_REFRESH_TOKEN = "RefreshToken";
// OAuth Client Cred Token Configuration
public static final String AZURE_AD_CLIENT_SECRET_KEY =
- "dfs.adls.oauth2.credential";
+ "fs.adl.oauth2.credential";
public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED =
"ClientCredential";
@@ -75,7 +77,7 @@ public final class AdlConfKeys {
static final int DEFAULT_WRITE_AHEAD_BUFFER_SIZE = 4 * 1024 * 1024;
static final String LATENCY_TRACKER_KEY =
- "adl.dfs.enable.client.latency.tracker";
+ "adl.enable.client.latency.tracker";
static final boolean LATENCY_TRACKER_DEFAULT = true;
static final String ADL_EXPERIMENT_POSITIONAL_READ_KEY =
@@ -90,6 +92,26 @@ public final class AdlConfKeys {
"adl.feature.ownerandgroup.enableupn";
static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
+ public static void addDeprecatedKeys() {
+ Configuration.addDeprecations(new DeprecationDelta[]{
+ new DeprecationDelta("dfs.adls.oauth2.access.token.provider.type",
+ AZURE_AD_TOKEN_PROVIDER_TYPE_KEY),
+ new DeprecationDelta("dfs.adls.oauth2.client.id",
+ AZURE_AD_CLIENT_ID_KEY),
+ new DeprecationDelta("dfs.adls.oauth2.refresh.token",
+ AZURE_AD_REFRESH_TOKEN_KEY),
+ new DeprecationDelta("dfs.adls.oauth2.refresh.url",
+ AZURE_AD_REFRESH_URL_KEY),
+ new DeprecationDelta("dfs.adls.oauth2.credential",
+ AZURE_AD_CLIENT_SECRET_KEY),
+ new DeprecationDelta("dfs.adls.oauth2.access.token.provider",
+ AZURE_AD_TOKEN_PROVIDER_CLASS_KEY),
+ new DeprecationDelta("adl.dfs.enable.client.latency.tracker",
+ LATENCY_TRACKER_KEY)
+ });
+ Configuration.reloadExistingConfigurations();
+ }
+
private AdlConfKeys() {
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index e0e273e..0b860b3 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -88,6 +88,10 @@ public class AdlFileSystem extends FileSystem {
private AccessTokenProvider tokenProvider;
private AzureADTokenProvider azureTokenProvider;
+ static {
+ AdlConfKeys.addDeprecatedKeys();
+ }
+
@Override
public String getScheme() {
return SCHEME;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index de8fda2..d2da858 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -83,7 +83,7 @@ Add the following properties to the cluster's `core-site.xml`
```xml
<property>
- <name>dfs.adls.oauth2.access.token.provider.type</name>
+ <name>fs.adl.oauth2.access.token.provider.type</name>
<value>RefreshToken</value>
</property>
```
@@ -95,12 +95,12 @@ service associated with the client id. See [*Active Directory Library For Java*]
```xml
<property>
- <name>dfs.adls.oauth2.client.id</name>
+ <name>fs.adl.oauth2.client.id</name>
<value></value>
</property>
<property>
- <name>dfs.adls.oauth2.refresh.token</name>
+ <name>fs.adl.oauth2.refresh.token</name>
<value></value>
</property>
```
@@ -133,22 +133,22 @@ Add the following properties to your `core-site.xml`
```xml
<property>
- <name>dfs.adls.oauth2.access.token.provider.type</name>
+ <name>fs.adl.oauth2.access.token.provider.type</name>
<value>ClientCredential</value>
</property>
<property>
- <name>dfs.adls.oauth2.refresh.url</name>
+ <name>fs.adl.oauth2.refresh.url</name>
<value>TOKEN ENDPOINT FROM STEP 7 ABOVE</value>
</property>
<property>
- <name>dfs.adls.oauth2.client.id</name>
+ <name>fs.adl.oauth2.client.id</name>
<value>CLIENT ID FROM STEP 7 ABOVE</value>
</property>
<property>
- <name>dfs.adls.oauth2.credential</name>
+ <name>fs.adl.oauth2.credential</name>
<value>PASSWORD FROM STEP 7 ABOVE</value>
</property>
```
@@ -166,9 +166,9 @@ For additional reading on the credential provider API, see
##### Provisioning
```bash
-hadoop credential create dfs.adls.oauth2.client.id -value 123
+hadoop credential create fs.adl.oauth2.client.id -value 123
-provider localjceks://file/home/foo/adls.jceks
-hadoop credential create dfs.adls.oauth2.refresh.token -value 123
+hadoop credential create fs.adl.oauth2.refresh.token -value 123
-provider localjceks://file/home/foo/adls.jceks
```
@@ -176,7 +176,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123
```xml
<property>
- <name>dfs.adls.oauth2.access.token.provider.type</name>
+ <name>fs.adl.oauth2.access.token.provider.type</name>
<value>RefreshToken</value>
</property>
<property>
@@ -190,7 +190,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123
```bash
hadoop distcp
- [-D dfs.adls.oauth2.access.token.provider.type=RefreshToken
+ [-D fs.adl.oauth2.access.token.provider.type=RefreshToken
-D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks]
hdfs://<NameNode Hostname>:9001/user/foo/srcDir
adl://<Account Name>.azuredatalakestore.net/tgtDir/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
index 70f2a7f..3867e74 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -46,6 +46,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -121,8 +122,8 @@ public class TestAzureADTokenProvider {
Assert.fail("Initialization should have failed due no token provider "
+ "configuration");
} catch (IllegalArgumentException e) {
- Assert.assertTrue(
- e.getMessage().contains("dfs.adls.oauth2.access.token.provider"));
+ GenericTestUtils.assertExceptionContains(
+ AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, e);
}
conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
CustomMockTokenProvider.class, AzureADTokenProvider.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
index 4cabaa3..3d51b42 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.fs.adl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -56,6 +58,11 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys
.TOKEN_PROVIDER_TYPE_REFRESH_TOKEN;
import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
/**
* Validate configuration keys defined for adl storage file system instance.
*/
@@ -64,18 +71,18 @@ public class TestValidateConfiguration {
@Test
public void validateConfigurationKeys() {
Assert
- .assertEquals("dfs.adls.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
- Assert.assertEquals("dfs.adls.oauth2.access.token.provider",
+ .assertEquals("fs.adl.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
+ Assert.assertEquals("fs.adl.oauth2.access.token.provider",
AZURE_AD_TOKEN_PROVIDER_CLASS_KEY);
- Assert.assertEquals("dfs.adls.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
- Assert.assertEquals("dfs.adls.oauth2.refresh.token",
+ Assert.assertEquals("fs.adl.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
+ Assert.assertEquals("fs.adl.oauth2.refresh.token",
AZURE_AD_REFRESH_TOKEN_KEY);
Assert
- .assertEquals("dfs.adls.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
+ .assertEquals("fs.adl.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
Assert.assertEquals("adl.debug.override.localuserasfileowner",
ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER);
- Assert.assertEquals("dfs.adls.oauth2.access.token.provider.type",
+ Assert.assertEquals("fs.adl.oauth2.access.token.provider.type",
AZURE_AD_TOKEN_PROVIDER_TYPE_KEY);
Assert.assertEquals("adl.feature.client.cache.readahead",
@@ -88,7 +95,7 @@ public class TestValidateConfiguration {
Assert.assertEquals("ClientCredential", TOKEN_PROVIDER_TYPE_CLIENT_CRED);
- Assert.assertEquals("adl.dfs.enable.client.latency.tracker",
+ Assert.assertEquals("adl.enable.client.latency.tracker",
LATENCY_TRACKER_KEY);
Assert.assertEquals(true, LATENCY_TRACKER_DEFAULT);
@@ -109,4 +116,66 @@ public class TestValidateConfiguration {
Assert.assertEquals(false,
ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
}
+
+ @Test
+ public void testSetDeprecatedKeys() throws ClassNotFoundException {
+ Configuration conf = new Configuration(true);
+ setDeprecatedKeys(conf);
+
+ // Force AdlFileSystem static initialization to register deprecated keys.
+ Class.forName(AdlFileSystem.class.getName());
+
+ assertDeprecatedKeys(conf);
+ }
+
+ @Test
+ public void testLoadDeprecatedKeys()
+ throws IOException, ClassNotFoundException {
+ Configuration saveConf = new Configuration(false);
+ setDeprecatedKeys(saveConf);
+
+ final File testRootDir = GenericTestUtils.getTestDir();
+ File confXml = new File(testRootDir, "testLoadDeprecatedKeys.xml");
+ OutputStream out = new FileOutputStream(confXml);
+ saveConf.writeXml(out);
+ out.close();
+
+ Configuration conf = new Configuration(true);
+ conf.addResource(confXml.toURI().toURL());
+
+ // Trigger loading the configuration resources by getting any key.
+ conf.get("dummy.key");
+
+ // Force AdlFileSystem static initialization to register deprecated keys.
+ Class.forName(AdlFileSystem.class.getName());
+
+ assertDeprecatedKeys(conf);
+ }
+
+ private void setDeprecatedKeys(Configuration conf) {
+ conf.set("dfs.adls.oauth2.access.token.provider.type", "dummyType");
+ conf.set("dfs.adls.oauth2.client.id", "dummyClientId");
+ conf.set("dfs.adls.oauth2.refresh.token", "dummyRefreshToken");
+ conf.set("dfs.adls.oauth2.refresh.url", "dummyRefreshUrl");
+ conf.set("dfs.adls.oauth2.credential", "dummyCredential");
+ conf.set("dfs.adls.oauth2.access.token.provider", "dummyClass");
+ conf.set("adl.dfs.enable.client.latency.tracker", "dummyTracker");
+ }
+
+ private void assertDeprecatedKeys(Configuration conf) {
+ Assert.assertEquals("dummyType",
+ conf.get(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY));
+ Assert.assertEquals("dummyClientId",
+ conf.get(AZURE_AD_CLIENT_ID_KEY));
+ Assert.assertEquals("dummyRefreshToken",
+ conf.get(AZURE_AD_REFRESH_TOKEN_KEY));
+ Assert.assertEquals("dummyRefreshUrl",
+ conf.get(AZURE_AD_REFRESH_URL_KEY));
+ Assert.assertEquals("dummyCredential",
+ conf.get(AZURE_AD_CLIENT_SECRET_KEY));
+ Assert.assertEquals("dummyClass",
+ conf.get(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY));
+ Assert.assertEquals("dummyTracker",
+ conf.get(LATENCY_TRACKER_KEY));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDFS-11560. Expose slow disks via
NameNode JMX. Contributed by Hanisha Koneru.
Posted by ae...@apache.org.
HDFS-11560. Expose slow disks via NameNode JMX. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/73835c73
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/73835c73
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/73835c73
Branch: refs/heads/HDFS-7240
Commit: 73835c73e2d34b3854a71dd29d88c8303d698ac8
Parents: 5485d93
Author: Hanisha Koneru <ha...@apache.org>
Authored: Fri Mar 31 13:50:29 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Mar 31 13:50:29 2017 -0700
----------------------------------------------------------------------
.../server/blockmanagement/DatanodeManager.java | 9 +++
.../server/blockmanagement/SlowDiskTracker.java | 3 +
.../hadoop/hdfs/server/namenode/NameNode.java | 6 ++
.../server/namenode/NameNodeStatusMXBean.java | 8 +++
.../blockmanagement/TestSlowDiskTracker.java | 13 +----
.../namenode/TestNameNodeStatusMXBean.java | 59 +++++++++++++++++++-
6 files changed, 85 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73835c73/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 18135a8..c7bdca9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -1907,5 +1907,14 @@ public class DatanodeManager {
public SlowDiskTracker getSlowDiskTracker() {
return slowDiskTracker;
}
+ /**
+ * Retrieve information about slow disks as a JSON.
+ * Returns null if we are not tracking slow disks.
+ * @return
+ */
+ public String getSlowDisksReport() {
+ return slowDiskTracker != null ?
+ slowDiskTracker.getSlowDiskReportAsJsonString() : null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73835c73/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
index 25920a2..52fce5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
@@ -256,6 +256,9 @@ public class SlowDiskTracker {
public String getSlowDiskReportAsJsonString() {
ObjectMapper objectMapper = new ObjectMapper();
try {
+ if (slowDisksReport.isEmpty()) {
+ return null;
+ }
return objectMapper.writeValueAsString(slowDisksReport);
} catch (JsonProcessingException e) {
// Failed to serialize. Don't log the exception call stack.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73835c73/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index e7841f0..32d268a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -1826,6 +1826,12 @@ public class NameNode extends ReconfigurableBase implements
.getSlowPeersReport();
}
+ @Override //NameNodeStatusMXBean
+ public String getSlowDisksReport() {
+ return namesystem.getBlockManager().getDatanodeManager()
+ .getSlowDisksReport();
+ }
+
/**
* Shutdown the NN immediately in an ungraceful way. Used when it would be
* unsafe for the NN to continue operating, e.g. during a failed HA state
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73835c73/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeStatusMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeStatusMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeStatusMXBean.java
index f46b9ae..ed1b96b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeStatusMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeStatusMXBean.java
@@ -75,4 +75,12 @@ public interface NameNodeStatusMXBean {
* enabled. The report is in a JSON format.
*/
String getSlowPeersReport();
+
+
+ /**
+ * Gets the topN slow disks in the cluster, if the feature is enabled.
+ *
+ * @return JSON string of list of diskIDs and latencies
+ */
+ String getSlowDisksReport();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73835c73/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java
index e96b96a..16dfab2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java
@@ -393,18 +393,9 @@ public class TestSlowDiskTracker {
timer.advance(reportValidityMs);
tracker.updateSlowDiskReportAsync(timer.monotonicNow());
+ Thread.sleep(OUTLIERS_REPORT_INTERVAL*2);
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- return tracker.getSlowDiskReportAsJsonString() != null;
- }
- }, 500, 5000);
-
- ArrayList<DiskLatency> jsonReport = getAndDeserializeJson(
- tracker.getSlowDiskReportAsJsonString());
-
- assertTrue(jsonReport.isEmpty());
+ assertTrue(tracker.getSlowDiskReportAsJsonString() == null);
}
private boolean isDiskInReports(ArrayList<DiskLatency> reports,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73835c73/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java
index c03dc20..8fe734e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java
@@ -17,17 +17,23 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import com.google.common.base.Supplier;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.datanode.TestDataNodeMXBean;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
import javax.management.MBeanServer;
import javax.management.ObjectName;
import java.lang.management.ManagementFactory;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
/**
* Class for testing {@link NameNodeStatusMXBean} implementation.
@@ -38,7 +44,7 @@ public class TestNameNodeStatusMXBean {
TestNameNodeStatusMXBean.class);
@Test(timeout = 120000L)
- public void testDataNodeMXBean() throws Exception {
+ public void testNameNodeStatusMXBean() throws Exception {
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
@@ -84,6 +90,55 @@ public class TestNameNodeStatusMXBean {
String slowPeersReport = (String)mbs.getAttribute(mxbeanName,
"SlowPeersReport");
Assert.assertEquals(nn.getSlowPeersReport(), slowPeersReport);
+
+ // Get attribute "SlowDisksReport"
+ String slowDisksReport = (String)mbs.getAttribute(mxbeanName,
+ "SlowDisksReport");
+ Assert.assertEquals(nn.getSlowDisksReport(), slowDisksReport);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testNameNodeMXBeanSlowDisksEnabled() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setDouble(
+ DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
+ conf.setTimeDuration(
+ DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
+ 1000, TimeUnit.MILLISECONDS);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+
+ try {
+ List<DataNode> datanodes = cluster.getDataNodes();
+ Assert.assertEquals(datanodes.size(), 1);
+ DataNode datanode = datanodes.get(0);
+ String slowDiskPath = "test/data1/slowVolume";
+ datanode.getDiskMetrics().addSlowDiskForTesting(slowDiskPath, null);
+
+ NameNode nn = cluster.getNameNode();
+ DatanodeManager datanodeManager = nn.getNamesystem().getBlockManager()
+ .getDatanodeManager();
+
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName mxbeanName = new ObjectName(
+ "Hadoop:service=NameNode,name=NameNodeStatus");
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return (datanodeManager.getSlowDisksReport() != null);
+ }
+ }, 1000, 100000);
+
+ String slowDisksReport = (String)mbs.getAttribute(
+ mxbeanName, "SlowDisksReport");
+ Assert.assertEquals(datanodeManager.getSlowDisksReport(),
+ slowDisksReport);
+ Assert.assertTrue(slowDisksReport.contains(slowDiskPath));
} finally {
if (cluster != null) {
cluster.shutdown();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HADOOP-14216. Addendum to Improve
Configuration XML Parsing Performance (jeagles)
Posted by ae...@apache.org.
HADOOP-14216. Addendum to Improve Configuration XML Parsing Performance (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1309c585
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1309c585
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1309c585
Branch: refs/heads/HDFS-7240
Commit: 1309c585fb9f632f7c649464ecbe358c5130b142
Parents: 8c591b8
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Thu Mar 30 14:14:43 2017 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Thu Mar 30 14:14:43 2017 -0500
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/conf/Configuration.java | 9 ---------
1 file changed, 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1309c585/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 36845e3..2ac52cb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -2756,9 +2756,6 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
token.setLength(0);
break;
case "include":
- if (!"xi".equals(reader.getPrefix())) {
- break;
- }
// Determine href for xi:include
String confInclude = null;
attrCount = reader.getAttributeCount();
@@ -2795,9 +2792,6 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
break;
case "fallback":
- if (!"xi".equals(reader.getPrefix())) {
- break;
- }
fallbackEntered = true;
break;
case "configuration":
@@ -2833,9 +2827,6 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
confSource.add(StringInterner.weakIntern(token.toString()));
break;
case "include":
- if (!"xi".equals(reader.getPrefix())) {
- break;
- }
if (fallbackAllowed && !fallbackEntered) {
throw new IOException("Fetch fail on include with no "
+ "fallback while loading '" + name + "'");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDFS-11598. Improve -setrep for
Erasure Coded files. Contributed by Yiqun Lin.
Posted by ae...@apache.org.
HDFS-11598. Improve -setrep for Erasure Coded files. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbd68478
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbd68478
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbd68478
Branch: refs/heads/HDFS-7240
Commit: bbd68478d5743b3b2911bf3febed7daa89479e45
Parents: bc7aff7
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Apr 3 07:57:28 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Apr 3 07:57:28 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/shell/SetReplication.java | 17 ++++++--
.../hadoop/hdfs/TestSetrepIncreasing.java | 44 ++++++++++++++++++++
2 files changed, 57 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbd68478/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
index 2231c58..16e6e92 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
@@ -85,11 +85,20 @@ class SetReplication extends FsCommand {
}
if (item.stat.isFile()) {
- if (!item.fs.setReplication(item.path, newRep)) {
- throw new IOException("Could not set replication for: " + item);
+ // Do the checking if the file is erasure coded since
+ // replication factor for an EC file is meaningless.
+ if (!item.stat.isErasureCoded()) {
+ if (!item.fs.setReplication(item.path, newRep)) {
+ throw new IOException("Could not set replication for: " + item);
+ }
+ out.println("Replication " + newRep + " set: " + item);
+ if (waitOpt) {
+ waitList.add(item);
+ }
+ } else {
+ out.println("Did not set replication for: " + item
+ + ", because it's an erasure coded file.");
}
- out.println("Replication " + newRep + " set: " + item);
- if (waitOpt) waitList.add(item);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbd68478/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
index fee30b5..50d7b27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
@@ -20,7 +20,9 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.PrintStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
@@ -28,6 +30,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.junit.Test;
@@ -102,4 +105,45 @@ public class TestSetrepIncreasing {
cluster.shutdown();
}
}
+
+ @Test
+ public void testSetRepOnECFile() throws Exception {
+ ClientProtocol client;
+ Configuration conf = new HdfsConfiguration();
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
+ StripedFileTestUtil.getDefaultECPolicy().getName());
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+ .build();
+ cluster.waitActive();
+ client = NameNodeProxies.createProxy(conf,
+ cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
+ client.setErasureCodingPolicy("/",
+ StripedFileTestUtil.getDefaultECPolicy().getName());
+
+ FileSystem dfs = cluster.getFileSystem();
+ try {
+ Path d = new Path("/tmp");
+ dfs.mkdirs(d);
+ Path f = new Path(d, "foo");
+ dfs.createNewFile(f);
+ FileStatus file = dfs.getFileStatus(f);
+ assertTrue(file.isErasureCoded());
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(out));
+ String[] args = {"-setrep", "2", "" + f};
+ FsShell shell = new FsShell();
+ shell.setConf(conf);
+ assertEquals(0, shell.run(args));
+ assertTrue(
+ out.toString().contains("Did not set replication for: /tmp/foo"));
+
+ // verify the replication factor of the EC file
+ file = dfs.getFileStatus(f);
+ assertEquals(1, file.getReplication());
+ } finally {
+ dfs.close();
+ cluster.shutdown();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: YARN-5685. RM configuration allows
all failover methods to disabled when automatic failover is enabled
Posted by ae...@apache.org.
YARN-5685. RM configuration allows all failover methods to disabled when automatic failover is enabled
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/640ba1d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/640ba1d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/640ba1d2
Branch: refs/heads/HDFS-7240
Commit: 640ba1d23fe8b8105bae6d342ddc1c839302f8e5
Parents: 13c766b
Author: Daniel Templeton <te...@apache.org>
Authored: Wed Mar 29 12:36:15 2017 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Wed Mar 29 12:38:11 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/conf/HAUtil.java | 30 ++++++-
.../hadoop/yarn/conf/YarnConfiguration.java | 16 +++-
.../org/apache/hadoop/yarn/conf/TestHAUtil.java | 86 +++++++++++-------
.../resourcemanager/TestRMAdminService.java | 95 ++++++++++++++++++++
4 files changed, 191 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/640ba1d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
index e4948e7..133b377 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
@@ -34,6 +34,7 @@ import java.util.Collection;
public class HAUtil {
private static Log LOG = LogFactory.getLog(HAUtil.class);
+ @VisibleForTesting
public static final String BAD_CONFIG_MESSAGE_PREFIX =
"Invalid configuration! ";
@@ -79,6 +80,7 @@ public class HAUtil {
throws YarnRuntimeException {
verifyAndSetRMHAIdsList(conf);
verifyAndSetCurrentRMHAId(conf);
+ verifyLeaderElection(conf);
verifyAndSetAllServiceAddresses(conf);
}
@@ -117,7 +119,7 @@ public class HAUtil {
msg.append("Can not find valid RM_HA_ID. None of ");
for (String id : conf
.getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS)) {
- msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id) + " ");
+ msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id)).append(" ");
}
msg.append(" are matching" +
" the local address OR " + YarnConfiguration.RM_HA_ID + " is not" +
@@ -133,6 +135,32 @@ public class HAUtil {
conf.set(YarnConfiguration.RM_HA_ID, rmId);
}
+ /**
+ * This method validates that some leader election service is enabled. YARN
+ * allows leadership election to be disabled in the configuration, which
+ * breaks automatic failover. If leadership election is disabled, this
+ * method will throw an exception via
+ * {@link #throwBadConfigurationException(java.lang.String)}.
+ *
+ * @param conf the {@link Configuration} to validate
+ */
+ private static void verifyLeaderElection(Configuration conf) {
+ if (isAutomaticFailoverEnabled(conf) &&
+ !conf.getBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR,
+ YarnConfiguration.DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED) &&
+ !isAutomaticFailoverEmbedded(conf)) {
+ throwBadConfigurationException(NO_LEADER_ELECTION_MESSAGE);
+ }
+ }
+
+ @VisibleForTesting
+ static final String NO_LEADER_ELECTION_MESSAGE =
+ "The yarn.resourcemanager.ha.automatic-failover.embedded "
+ + "and yarn.resourcemanager.ha.curator-leader-elector.enabled "
+ + "properties are both false. One of these two properties must "
+ + "be true when yarn.resourcemanager.ha.automatic-failover.enabled "
+ + "is true";
+
private static void verifyAndSetConfValue(String prefix, Configuration conf) {
String confKey = null;
String confValue = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/640ba1d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b366855..f52e487 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -627,8 +627,22 @@ public class YarnConfiguration extends Configuration {
AUTO_FAILOVER_PREFIX + "enabled";
public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true;
+ /**
+ * This property controls whether {@link ActiveStandbyElector} leader
+ * election should be used when {@link #CURATOR_LEADER_ELECTOR} is
+ * {@code false}.
+ *
+ * @deprecated This property should never be set to {@code false}.
+ */
+ @Deprecated
public static final String AUTO_FAILOVER_EMBEDDED =
AUTO_FAILOVER_PREFIX + "embedded";
+ /**
+ * The default value for {@link #AUTO_FAILOVER_EMBEDDED}.
+ *
+ * @deprecated The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.
+ */
+ @Deprecated
public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true;
public static final String AUTO_FAILOVER_ZK_BASE_PATH =
@@ -667,7 +681,7 @@ public class YarnConfiguration extends Configuration {
/**
- * Whether to use curator-based elector for leader election.
+ * Whether to use the Curator-based elector for leader election.
*
* @deprecated Eventually, we want to default to the curator-based
* implementation and remove the {@link ActiveStandbyElector} based
http://git-wip-us.apache.org/repos/asf/hadoop/blob/640ba1d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
index 6ced5f2..fc2c1d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
@@ -85,44 +85,47 @@ public class TestHAUtil {
@Test
public void testVerifyAndSetConfiguration() throws Exception {
+ Configuration myConf = new Configuration(conf);
+
try {
- HAUtil.verifyAndSetConfiguration(conf);
+ HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
fail("Should not throw any exceptions.");
}
assertEquals("Should be saved as Trimmed collection",
- StringUtils.getStringCollection(RM_NODE_IDS), HAUtil.getRMHAIds(conf));
+ StringUtils.getStringCollection(RM_NODE_IDS),
+ HAUtil.getRMHAIds(myConf));
assertEquals("Should be saved as Trimmed string",
- RM1_NODE_ID, HAUtil.getRMHAId(conf));
- for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
+ RM1_NODE_ID, HAUtil.getRMHAId(myConf));
+ for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) {
assertEquals("RPC address not set for " + confKey,
- RM1_ADDRESS, conf.get(confKey));
+ RM1_ADDRESS, myConf.get(confKey));
}
- conf.clear();
- conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID);
+ myConf = new Configuration(conf);
+ myConf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID);
try {
- HAUtil.verifyAndSetConfiguration(conf);
+ HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
assertEquals("YarnRuntimeException by verifyAndSetRMHAIds()",
HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
HAUtil.getInvalidValueMessage(YarnConfiguration.RM_HA_IDS,
- conf.get(YarnConfiguration.RM_HA_IDS) +
+ myConf.get(YarnConfiguration.RM_HA_IDS) +
"\nHA mode requires atleast two RMs"),
e.getMessage());
}
- conf.clear();
+ myConf = new Configuration(conf);
// simulate the case YarnConfiguration.RM_HA_ID is not set
- conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
+ myConf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
+ RM2_NODE_ID);
- for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
- conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
- conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
+ for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) {
+ myConf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
+ myConf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
}
try {
- HAUtil.verifyAndSetConfiguration(conf);
+ HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
assertEquals("YarnRuntimeException by getRMId()",
HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
@@ -130,16 +133,16 @@ public class TestHAUtil {
e.getMessage());
}
- conf.clear();
- conf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID);
- conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + ","
+ myConf = new Configuration(conf);
+ myConf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID);
+ myConf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + ","
+ RM1_NODE_ID);
- for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
+ for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) {
// simulate xml with invalid node id
- conf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID);
+ myConf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID);
}
try {
- HAUtil.verifyAndSetConfiguration(conf);
+ HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
assertEquals("YarnRuntimeException by addSuffix()",
HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
@@ -148,12 +151,12 @@ public class TestHAUtil {
e.getMessage());
}
- conf.clear();
+ myConf = new Configuration();
// simulate the case HAUtil.RM_RPC_ADDRESS_CONF_KEYS are not set
- conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
- conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
+ myConf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
+ myConf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
try {
- HAUtil.verifyAndSetConfiguration(conf);
+ HAUtil.verifyAndSetConfiguration(myConf);
fail("Should throw YarnRuntimeException. by Configuration#set()");
} catch (YarnRuntimeException e) {
String confKey =
@@ -166,21 +169,36 @@ public class TestHAUtil {
// simulate the case YarnConfiguration.RM_HA_IDS doesn't contain
// the value of YarnConfiguration.RM_HA_ID
- conf.clear();
- conf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID);
- conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED);
- for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
- conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
- conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
- conf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
+ myConf = new Configuration(conf);
+ myConf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID);
+ myConf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED);
+ for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) {
+ myConf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
+ myConf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
+ myConf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
}
try {
- HAUtil.verifyAndSetConfiguration(conf);
+ HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
assertEquals("YarnRuntimeException by getRMId()'s validation",
HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
HAUtil.getRMHAIdNeedToBeIncludedMessage("[rm2, rm3]", RM1_NODE_ID),
- e.getMessage());
+ e.getMessage());
+ }
+
+ // simulate the case that no leader election is enabled
+ myConf = new Configuration(conf);
+ myConf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ myConf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
+ myConf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, false);
+ myConf.setBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR, false);
+
+ try {
+ HAUtil.verifyAndSetConfiguration(myConf);
+ } catch (YarnRuntimeException e) {
+ assertEquals("YarnRuntimeException by getRMId()'s validation",
+ HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.NO_LEADER_ELECTION_MESSAGE,
+ e.getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/640ba1d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index 9ae28c2..8d00ae0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -78,6 +78,8 @@ import org.junit.Test;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import static org.junit.Assert.assertTrue;
public class TestRMAdminService {
@@ -841,6 +843,78 @@ public class TestRMAdminService {
}
}
+ /**
+ * Test that a configuration with no leader election configured fails.
+ */
+ @Test
+ public void testHAConfWithoutLeaderElection() {
+ Configuration conf = new Configuration(configuration);
+
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, false);
+ conf.setBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR, false);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
+
+ int base = 100;
+
+ for (String confKey :
+ YarnConfiguration.getServiceAddressConfKeys(configuration)) {
+ conf.set(HAUtil.addSuffix(confKey, "rm1"), "0.0.0.0:"
+ + (base + 20));
+ conf.set(HAUtil.addSuffix(confKey, "rm2"), "0.0.0.0:"
+ + (base + 40));
+ base = base * 2;
+ }
+
+ conf.set(YarnConfiguration.RM_HA_ID, "rm1");
+
+ checkBadConfiguration(conf);
+ }
+
+ /**
+ * Test that a configuration with a single RM fails.
+ */
+ @Test
+ public void testHAConfWithSingleRMID() {
+ Configuration conf = new Configuration(configuration);
+
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1");
+
+ int base = 100;
+
+ for (String confKey :
+ YarnConfiguration.getServiceAddressConfKeys(configuration)) {
+ conf.set(HAUtil.addSuffix(confKey, "rm1"), "0.0.0.0:"
+ + (base + 20));
+ base = base * 2;
+ }
+
+ conf.set(YarnConfiguration.RM_HA_ID, "rm1");
+
+ checkBadConfiguration(conf);
+ }
+
+ /**
+ * Test that a configuration with no service information fails.
+ */
+ @Test
+ public void testHAConfWithoutServiceInfo() {
+ Configuration conf = new Configuration(configuration);
+
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
+
+ conf.set(YarnConfiguration.RM_HA_ID, "rm1");
+
+ checkBadConfiguration(conf);
+ }
+
@Test
public void testRMStartsWithoutConfigurationFilesProvided() {
// enable FileSystemBasedConfigurationProvider without uploading
@@ -1368,4 +1442,25 @@ public class TestRMAdminService {
base = base * 2;
}
}
+
+ /**
+ * This method initializes an RM with the given configuration and expects it
+ * to fail with a configuration error.
+ *
+ * @param conf the {@link Configuration} to use
+ */
+ private void checkBadConfiguration(Configuration conf) {
+ MockRM rm1 = null;
+
+ conf.set(YarnConfiguration.RM_HA_ID, "rm1");
+
+ try {
+ rm1 = new MockRM(conf);
+ rm1.init(conf);
+ fail("The RM allowed an invalid configuration");
+ } catch (YarnRuntimeException e) {
+ assertTrue("The RM initialization threw an unexpected exception",
+ e.getMessage().startsWith(HAUtil.BAD_CONFIG_MESSAGE_PREFIX));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HADOOP-14223. Extend
FileStatus#toString() to include details like Erasure Coding and Encryption.
Contributed by Manoj Govindassamy.
Posted by ae...@apache.org.
HADOOP-14223. Extend FileStatus#toString() to include details like Erasure Coding and Encryption. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4966a6e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4966a6e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4966a6e2
Branch: refs/heads/HDFS-7240
Commit: 4966a6e26e45d7dc36e0b270066ff7c87bcd00cc
Parents: 640ba1d
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Mar 29 14:37:21 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Mar 29 14:37:21 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileStatus.java | 12 ++++++++++
.../src/site/markdown/filesystem/filesystem.md | 19 ++++++----------
.../org/apache/hadoop/fs/TestFileStatus.java | 10 ++++++---
.../hadoop/fs/viewfs/TestViewfsFileStatus.java | 3 +++
.../org/apache/hadoop/hdfs/TestFileStatus.java | 3 +++
.../hadoop/hdfs/TestFileStatusWithECPolicy.java | 7 +++++-
.../hdfs/server/namenode/FSAclBaseTest.java | 23 ++++++++++++++++++++
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 12 ++++++++++
.../hadoop/fs/s3a/TestS3AGetFileStatus.java | 6 +++++
.../apache/hadoop/fs/adl/TestGetFileStatus.java | 7 ++++++
10 files changed, 86 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
index 26d3962..f5111ef 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
@@ -208,6 +208,15 @@ public class FileStatus implements Writable, Comparable<FileStatus>,
}
/**
+ * Tell whether the underlying file or directory has ACLs set.
+ *
+ * @return true if the underlying file or directory has ACLs set.
+ */
+ public boolean hasAcl() {
+ return permission.getAclBit();
+ }
+
+ /**
* Tell whether the underlying file or directory is encrypted or not.
*
* @return true if the underlying file is encrypted.
@@ -399,6 +408,9 @@ public class FileStatus implements Writable, Comparable<FileStatus>,
if(isSymlink()) {
sb.append("; symlink=" + symlink);
}
+ sb.append("; hasAcl=" + hasAcl());
+ sb.append("; isEncrypted=" + isEncrypted());
+ sb.append("; isErasureCoded=" + isErasureCoded());
sb.append("}");
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index 97bc7d1..b464941 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -86,20 +86,15 @@ Get the status of a path
stat.length = 0
stat.isdir = False
stat.symlink = FS.Symlinks[p]
- if inEncryptionZone(FS, p) :
- stat.isEncrypted = True
- else
- stat.isEncrypted = False
- if isErasureCoded(FS, p) :
- stat.isErasureCoded = True
- else
- stat.isErasureCoded = False
+ stat.hasAcl = hasACL(FS, p)
+ stat.isEncrypted = inEncryptionZone(FS, p)
+ stat.isErasureCoded = isErasureCoded(FS, p)
The returned `FileStatus` status of the path additionally carries details on
-Encryption and Erasure Coding information. `getFileStatus(Path p).isEncrypted()`
-can be queried to find if the path is Encrypted.
-Likewise, `getFileStatus(Path p).isErasureCoded()` will tell if the path is
-Erasure Coded or not.
+ACL, encryption and erasure coding information. `getFileStatus(Path p).hasAcl()`
+can be queried to find if the path has an ACL. `getFileStatus(Path p).isEncrypted()`
+can be queried to find if the path is encrypted. `getFileStatus(Path p).isErasureCoded()`
+will tell if the path is erasure coded or not.
### `Path getHomeDirectory()`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
index 35f2bad..d29b1a4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
@@ -295,11 +295,15 @@ public class TestFileStatus {
expected.append("permission=").append(fileStatus.getPermission()).append("; ");
if(fileStatus.isSymlink()) {
expected.append("isSymlink=").append(true).append("; ");
- expected.append("symlink=").append(fileStatus.getSymlink()).append("}");
+ expected.append("symlink=").append(fileStatus.getSymlink()).append("; ");
} else {
- expected.append("isSymlink=").append(false).append("}");
+ expected.append("isSymlink=").append(false).append("; ");
}
-
+ expected.append("hasAcl=").append(fileStatus.hasAcl()).append("; ");
+ expected.append("isEncrypted=").append(
+ fileStatus.isEncrypted()).append("; ");
+ expected.append("isErasureCoded=").append(
+ fileStatus.isErasureCoded()).append("}");
assertEquals(expected.toString(), fileStatus.toString());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
index 449e2e3..0c31c8e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
@@ -75,6 +75,9 @@ public class TestViewfsFileStatus {
FileStatus stat = vfs.getFileStatus(path);
assertEquals(content.length, stat.getLen());
ContractTestUtils.assertNotErasureCoded(vfs, path);
+ assertTrue(path + " should have erasure coding unset in " +
+ "FileStatus#toString(): " + stat,
+ stat.toString().contains("isErasureCoded=false"));
// check serialization/deserialization
DataOutputBuffer dob = new DataOutputBuffer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
index 7631e7d..c74bb63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
@@ -139,6 +139,9 @@ public class TestFileStatus {
assertEquals(file1.makeQualified(fs.getUri(),
fs.getWorkingDirectory()).toString(),
status.getPath().toString());
+ assertTrue(file1 + " should have erasure coding unset in " +
+ "FileStatus#toString(): " + status,
+ status.toString().contains("isErasureCoded=false"));
}
/** Test the FileStatus obtained calling listStatus on a file */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
index 0e270ff..e04f957 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -85,7 +86,7 @@ public class TestFileStatusWithECPolicy {
assertNotNull(ecPolicy2);
assertTrue(ecPolicy1.equals(ecPolicy2));
- // test file doesn't have an EC policy
+ // test file with EC policy
fs.create(file).close();
final ErasureCodingPolicy ecPolicy3 =
fs.getClient().getFileInfo(file.toUri().getPath())
@@ -93,5 +94,9 @@ public class TestFileStatusWithECPolicy {
assertNotNull(ecPolicy3);
assertTrue(ecPolicy1.equals(ecPolicy3));
ContractTestUtils.assertErasureCoded(fs, file);
+ FileStatus status = fs.getFileStatus(file);
+ assertTrue(file + " should have erasure coding set in " +
+ "FileStatus#toString(): " + status,
+ status.toString().contains("isErasureCoded=true"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
index 81270e6..60b0ab1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -120,10 +121,16 @@ public abstract class FSAclBaseTest {
aclEntry(ACCESS, OTHER, NONE),
aclEntry(DEFAULT, USER, "foo", ALL));
fs.setAcl(path, aclSpec);
+ Assert.assertTrue(path + " should have ACLs in FileStatus!",
+ fs.getFileStatus(path).hasAcl());
+
aclSpec = Lists.newArrayList(
aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
aclEntry(DEFAULT, USER, "foo", READ_EXECUTE));
fs.modifyAclEntries(path, aclSpec);
+ Assert.assertTrue(path + " should have ACLs in FileStatus!",
+ fs.getFileStatus(path).hasAcl());
+
AclStatus s = fs.getAclStatus(path);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(new AclEntry[] {
@@ -561,8 +568,18 @@ public abstract class FSAclBaseTest {
aclEntry(ACCESS, GROUP, READ_EXECUTE),
aclEntry(ACCESS, OTHER, NONE),
aclEntry(DEFAULT, USER, "foo", ALL));
+
fs.setAcl(path, aclSpec);
+ Assert.assertTrue(path + " should have ACLs in FileStatus!",
+ fs.getFileStatus(path).hasAcl());
+ Assert.assertTrue(path + " should have ACLs in FileStatus#toString()!",
+ fs.getFileStatus(path).toString().contains("hasAcl=true"));
fs.removeAcl(path);
+ Assert.assertFalse(path + " should not have ACLs in FileStatus!",
+ fs.getFileStatus(path).hasAcl());
+ Assert.assertTrue(path + " should not have ACLs in FileStatus#toString()!",
+ fs.getFileStatus(path).toString().contains("hasAcl=false"));
+
AclStatus s = fs.getAclStatus(path);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(new AclEntry[] { }, returned);
@@ -968,8 +985,14 @@ public abstract class FSAclBaseTest {
List<AclEntry> aclSpec = Lists.newArrayList(
aclEntry(DEFAULT, USER, "foo", ALL));
fs.setAcl(path, aclSpec);
+ Assert.assertTrue(path + " should have ACLs in FileStatus!",
+ fs.getFileStatus(path).hasAcl());
+
Path dirPath = new Path(path, "dir1");
fs.mkdirs(dirPath);
+ Assert.assertTrue(dirPath + " should have ACLs in FileStatus!",
+ fs.getFileStatus(dirPath).hasAcl());
+
AclStatus s = fs.getAclStatus(dirPath);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(new AclEntry[] {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 3c366c3..46a8a56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -547,24 +547,36 @@ public class TestWebHDFS {
Assert.assertEquals(expectedECDirStatus.isErasureCoded(),
actualECDirStatus.isErasureCoded());
ContractTestUtils.assertErasureCoded(dfs, ecDir);
+ assertTrue(ecDir+ " should have erasure coding set in " +
+ "FileStatus#toString(): " + actualECDirStatus,
+ actualECDirStatus.toString().contains("isErasureCoded=true"));
FileStatus expectedECFileStatus = dfs.getFileStatus(ecFile);
FileStatus actualECFileStatus = webHdfs.getFileStatus(ecFile);
Assert.assertEquals(expectedECFileStatus.isErasureCoded(),
actualECFileStatus.isErasureCoded());
ContractTestUtils.assertErasureCoded(dfs, ecFile);
+ assertTrue(ecFile+ " should have erasure coding set in " +
+ "FileStatus#toString(): " + actualECFileStatus,
+ actualECFileStatus.toString().contains("isErasureCoded=true"));
FileStatus expectedNormalDirStatus = dfs.getFileStatus(normalDir);
FileStatus actualNormalDirStatus = webHdfs.getFileStatus(normalDir);
Assert.assertEquals(expectedNormalDirStatus.isErasureCoded(),
actualNormalDirStatus.isErasureCoded());
ContractTestUtils.assertNotErasureCoded(dfs, normalDir);
+ assertTrue(normalDir + " should have erasure coding unset in " +
+ "FileStatus#toString(): " + actualNormalDirStatus,
+ actualNormalDirStatus.toString().contains("isErasureCoded=false"));
FileStatus expectedNormalFileStatus = dfs.getFileStatus(normalFile);
FileStatus actualNormalFileStatus = webHdfs.getFileStatus(normalDir);
Assert.assertEquals(expectedNormalFileStatus.isErasureCoded(),
actualNormalFileStatus.isErasureCoded());
ContractTestUtils.assertNotErasureCoded(dfs, normalFile);
+ assertTrue(normalFile + " should have erasure coding unset in " +
+ "FileStatus#toString(): " + actualNormalFileStatus,
+ actualNormalFileStatus.toString().contains("isErasureCoded=false"));
} finally {
if (cluster != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
index deb3963..58e4d30 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
@@ -62,6 +62,9 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest {
assertEquals(meta.getContentLength(), stat.getLen());
assertEquals(meta.getLastModified().getTime(), stat.getModificationTime());
ContractTestUtils.assertNotErasureCoded(fs, path);
+ assertTrue(path + " should have erasure coding unset in " +
+ "FileStatus#toString(): " + stat,
+ stat.toString().contains("isErasureCoded=false"));
}
@Test
@@ -101,6 +104,9 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest {
assertEquals(fs.makeQualified(path), stat.getPath());
assertTrue(stat.isDirectory());
ContractTestUtils.assertNotErasureCoded(fs, path);
+ assertTrue(path + " should have erasure coding unset in " +
+ "FileStatus#toString(): " + stat,
+ stat.toString().contains("isErasureCoded=false"));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
index df39a40..0ea4b86 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
@@ -65,6 +65,9 @@ public class TestGetFileStatus extends AdlMockWebServer {
Assert.assertEquals(new FsPermission("777"), fileStatus.getPermission());
Assert.assertEquals("NotSupportYet", fileStatus.getOwner());
Assert.assertEquals("NotSupportYet", fileStatus.getGroup());
+ Assert.assertTrue(path + " should have Acl!", fileStatus.hasAcl());
+ Assert.assertFalse(path + " should not be encrypted!",
+ fileStatus.isEncrypted());
Assert.assertFalse(path + " should not be erasure coded!",
fileStatus.isErasureCoded());
}
@@ -82,6 +85,8 @@ public class TestGetFileStatus extends AdlMockWebServer {
LOG.debug("Time : " + (endTime - startTime));
Assert.assertTrue(fileStatus.isFile());
Assert.assertEquals(true, fileStatus.getPermission().getAclBit());
+ Assert.assertEquals(fileStatus.hasAcl(),
+ fileStatus.getPermission().getAclBit());
// With ACLBIT set to false
getMockServer().enqueue(new MockResponse().setResponseCode(200)
@@ -93,5 +98,7 @@ public class TestGetFileStatus extends AdlMockWebServer {
LOG.debug("Time : " + (endTime - startTime));
Assert.assertTrue(fileStatus.isFile());
Assert.assertEquals(false, fileStatus.getPermission().getAclBit());
+ Assert.assertEquals(fileStatus.hasAcl(),
+ fileStatus.getPermission().getAclBit());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: MAPREDUCE-6836. exception thrown
when accessing the job configuration web UI. Contributed by Haibo Chen
Posted by ae...@apache.org.
MAPREDUCE-6836. exception thrown when accessing the job configuration web UI. Contributed by Haibo Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9b7ce92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9b7ce92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9b7ce92
Branch: refs/heads/HDFS-7240
Commit: c9b7ce92737f0a829408936ed9a361220d3e9680
Parents: 7297e82
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Thu Mar 30 12:54:11 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Thu Mar 30 12:55:32 2017 -0500
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b7ce92/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
index 532c2bd..98a2ce1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
@@ -70,7 +70,7 @@ public class ConfBlock extends HtmlBlock {
try {
ConfInfo info = new ConfInfo(job);
- html.div().a("/jobhistory/downloadconf/" + jid, confPath.toString());
+ html.div().a("/jobhistory/downloadconf/" + jid, confPath.toString())._();
TBODY<TABLE<Hamlet>> tbody = html.
// Tasks table
table("#conf").
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-6359.
TestRM#testApplicationKillAtAcceptedState fails rarely due to race condition.
Contributed by Robert Kanter
Posted by ae...@apache.org.
YARN-6359. TestRM#testApplicationKillAtAcceptedState fails rarely due to race condition. Contributed by Robert Kanter
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdf8f8eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdf8f8eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdf8f8eb
Branch: refs/heads/HDFS-7240
Commit: fdf8f8ebca9987a1956ce464fe33ea6a3ad28d72
Parents: 253e3e7
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Tue Mar 28 09:33:26 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Tue Mar 28 09:33:26 2017 -0500
----------------------------------------------------------------------
.../apache/hadoop/yarn/server/resourcemanager/TestRM.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf8f8eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
index cdf582e..39313d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import com.google.common.base.Supplier;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.junit.Before;
import static org.mockito.Matchers.argThat;
@@ -631,7 +633,13 @@ public class TestRM extends ParameterizedSchedulerTestBase {
rm.waitForState(application.getApplicationId(), RMAppState.KILLED);
// test metrics
- metrics = rm.getResourceScheduler().getRootQueueMetrics();
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return appsKilled + 1 == metrics.getAppsKilled()
+ && appsSubmitted + 1 == metrics.getAppsSubmitted();
+ }
+ }, 100, 10000);
Assert.assertEquals(appsKilled + 1, metrics.getAppsKilled());
Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-6357. Implement
putEntitiesAsync API in TimelineCollector (Haibo Chen via Varun Saxena)
Posted by ae...@apache.org.
YARN-6357. Implement putEntitiesAsync API in TimelineCollector (Haibo Chen via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/063b513b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/063b513b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/063b513b
Branch: refs/heads/HDFS-7240
Commit: 063b513b1c10987461caab3d26c8543c6e657bf7
Parents: 01aca54
Author: Varun Saxena <va...@apache.org>
Authored: Wed Mar 29 03:48:03 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Wed Mar 29 03:48:03 2017 +0530
----------------------------------------------------------------------
.../collector/TimelineCollector.java | 31 ++++++++--
.../collector/TimelineCollectorWebService.java | 12 ++--
.../collector/TestTimelineCollector.java | 63 ++++++++++++++++++++
3 files changed, 96 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/063b513b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
index 2fc3033..353066b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -133,19 +133,35 @@ public abstract class TimelineCollector extends CompositeService {
public TimelineWriteResponse putEntities(TimelineEntities entities,
UserGroupInformation callerUgi) throws IOException {
if (LOG.isDebugEnabled()) {
- LOG.debug("SUCCESS - TIMELINE V2 PROTOTYPE");
LOG.debug("putEntities(entities=" + entities + ", callerUgi="
+ callerUgi + ")");
}
- TimelineCollectorContext context = getTimelineEntityContext();
+ TimelineWriteResponse response = writeTimelineEntities(entities);
+ flushBufferedTimelineEntities();
+
+ return response;
+ }
+
+ private TimelineWriteResponse writeTimelineEntities(
+ TimelineEntities entities) throws IOException {
// Update application metrics for aggregation
updateAggregateStatus(entities, aggregationGroups,
getEntityTypesSkipAggregation());
+ final TimelineCollectorContext context = getTimelineEntityContext();
return writer.write(context.getClusterId(), context.getUserId(),
- context.getFlowName(), context.getFlowVersion(), context.getFlowRunId(),
- context.getAppId(), entities);
+ context.getFlowName(), context.getFlowVersion(),
+ context.getFlowRunId(), context.getAppId(), entities);
+ }
+
+ /**
+ * Flush buffered timeline entities, if any.
+ * @throws IOException if there is any exception encountered while
+ * flushing buffered entities.
+ */
+ private void flushBufferedTimelineEntities() throws IOException {
+ writer.flush();
}
/**
@@ -158,14 +174,17 @@ public abstract class TimelineCollector extends CompositeService {
*
* @param entities entities to post
* @param callerUgi the caller UGI
+ * @throws IOException if there is any exception encounted while putting
+ * entities.
*/
public void putEntitiesAsync(TimelineEntities entities,
- UserGroupInformation callerUgi) {
- // TODO implement
+ UserGroupInformation callerUgi) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("putEntitiesAsync(entities=" + entities + ", callerUgi=" +
callerUgi + ")");
}
+
+ writeTimelineEntities(entities);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/063b513b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
index f36c636..fe04b7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
@@ -152,9 +152,6 @@ public class TimelineCollectorWebService {
throw new ForbiddenException(msg);
}
- // TODO how to express async posts and handle them
- boolean isAsync = async != null && async.trim().equalsIgnoreCase("true");
-
try {
ApplicationId appID = parseApplicationId(appId);
if (appID == null) {
@@ -169,7 +166,14 @@ public class TimelineCollectorWebService {
throw new NotFoundException(); // different exception?
}
- collector.putEntities(processTimelineEntities(entities), callerUgi);
+ boolean isAsync = async != null && async.trim().equalsIgnoreCase("true");
+ if (isAsync) {
+ collector.putEntitiesAsync(
+ processTimelineEntities(entities), callerUgi);
+ } else {
+ collector.putEntities(processTimelineEntities(entities), callerUgi);
+ }
+
return Response.ok().build();
} catch (Exception e) {
LOG.error("Error putting entities", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/063b513b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java
index 5b4dc50..a55f227 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java
@@ -18,17 +18,27 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
import org.junit.Test;
+import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
public class TestTimelineCollector {
@@ -124,4 +134,57 @@ public class TestTimelineCollector {
}
}
+
+ /**
+ * Test TimelineCollector's interaction with TimelineWriter upon
+ * putEntity() calls.
+ */
+ @Test
+ public void testPutEntity() throws IOException {
+ TimelineWriter writer = mock(TimelineWriter.class);
+ TimelineCollector collector = new TimelineCollectorForTest(writer);
+
+ TimelineEntities entities = generateTestEntities(1, 1);
+ collector.putEntities(
+ entities, UserGroupInformation.createRemoteUser("test-user"));
+
+ verify(writer, times(1)).write(
+ anyString(), anyString(), anyString(), anyString(), anyLong(),
+ anyString(), any(TimelineEntities.class));
+ verify(writer, times(1)).flush();
+ }
+
+ /**
+ * Test TimelineCollector's interaction with TimelineWriter upon
+ * putEntityAsync() calls.
+ */
+ @Test
+ public void testPutEntityAsync() throws IOException {
+ TimelineWriter writer = mock(TimelineWriter.class);
+ TimelineCollector collector = new TimelineCollectorForTest(writer);
+
+ TimelineEntities entities = generateTestEntities(1, 1);
+ collector.putEntitiesAsync(
+ entities, UserGroupInformation.createRemoteUser("test-user"));
+
+ verify(writer, times(1)).write(
+ anyString(), anyString(), anyString(), anyString(), anyLong(),
+ anyString(), any(TimelineEntities.class));
+ verify(writer, never()).flush();
+ }
+
+ private static class TimelineCollectorForTest extends TimelineCollector {
+ private final TimelineCollectorContext context =
+ new TimelineCollectorContext();
+
+ TimelineCollectorForTest(TimelineWriter writer) {
+ super("TimelineCollectorForTest");
+ setWriter(writer);
+ }
+
+ @Override
+ public TimelineCollectorContext getTimelineEntityContext() {
+ return context;
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-6354. LeveldbRMStateStore can
parse invalid keys when recovering reservations. Contributed by Jason Lowe
Posted by ae...@apache.org.
YARN-6354. LeveldbRMStateStore can parse invalid keys when recovering reservations. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/318bfb01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/318bfb01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/318bfb01
Branch: refs/heads/HDFS-7240
Commit: 318bfb01bc6793da09e32e9cc292eb63224b6ca2
Parents: 4d1fac5
Author: Eric Payne <ep...@apache.org>
Authored: Fri Mar 31 12:30:35 2017 -0500
Committer: Eric Payne <ep...@apache.org>
Committed: Fri Mar 31 12:30:35 2017 -0500
----------------------------------------------------------------------
.../recovery/LeveldbRMStateStore.java | 8 ++++++++
.../recovery/TestLeveldbRMStateStore.java | 19 +++++++++++++++----
2 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/318bfb01/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
index 02f90dd..2ca53db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
@@ -214,6 +214,11 @@ public class LeveldbRMStateStore extends RMStateStore {
return db == null;
}
+ @VisibleForTesting
+ DB getDatabase() {
+ return db;
+ }
+
@Override
protected Version loadVersion() throws Exception {
Version version = null;
@@ -284,6 +289,9 @@ public class LeveldbRMStateStore extends RMStateStore {
while (iter.hasNext()) {
Entry<byte[],byte[]> entry = iter.next();
String key = asString(entry.getKey());
+ if (!key.startsWith(RM_RESERVATION_KEY_PREFIX)) {
+ break;
+ }
String planReservationString =
key.substring(RM_RESERVATION_KEY_PREFIX.length());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/318bfb01/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
index 4297e73..51adbe1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.DB;
import org.junit.After;
import org.junit.Before;
@@ -125,17 +126,27 @@ public class TestLeveldbRMStateStore extends RMStateStoreTestBase {
public void testCompactionCycle() throws Exception {
final DB mockdb = mock(DB.class);
conf.setLong(YarnConfiguration.RM_LEVELDB_COMPACTION_INTERVAL_SECS, 1);
- LeveldbRMStateStore store = new LeveldbRMStateStore() {
+ stateStore = new LeveldbRMStateStore() {
@Override
protected DB openDatabase() throws Exception {
return mockdb;
}
};
- store.init(conf);
- store.start();
+ stateStore.init(conf);
+ stateStore.start();
verify(mockdb, timeout(10000)).compactRange(
(byte[]) isNull(), (byte[]) isNull());
- store.close();
+ }
+
+ @Test
+ public void testBadKeyIteration() throws Exception {
+ stateStore = new LeveldbRMStateStore();
+ stateStore.init(conf);
+ stateStore.start();
+ DB db = stateStore.getDatabase();
+ // add an entry that appears at the end of the database when iterating
+ db.put(JniDBFactory.bytes("zzz"), JniDBFactory.bytes("z"));
+ stateStore.loadState();
}
class LeveldbStateStoreTester implements RMStateStoreHelper {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-5654. Not be able to run SLS
with FairScheduler (yufeigu via rkanter)
Posted by ae...@apache.org.
YARN-5654. Not be able to run SLS with FairScheduler (yufeigu via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a5516c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a5516c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a5516c2
Branch: refs/heads/HDFS-7240
Commit: 6a5516c2381f107d96b8326939514de3c6e53d3d
Parents: 4966a6e
Author: Robert Kanter <rk...@apache.org>
Authored: Wed Mar 29 16:18:13 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Mar 29 16:18:13 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 22 +-
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 24 +-
.../sls/scheduler/ResourceSchedulerWrapper.java | 969 -------------------
.../sls/scheduler/SLSCapacityScheduler.java | 685 ++-----------
.../yarn/sls/scheduler/SLSFairScheduler.java | 339 +++++++
.../yarn/sls/scheduler/SchedulerMetrics.java | 533 +++++++++-
.../yarn/sls/scheduler/SchedulerWrapper.java | 18 +-
.../hadoop/yarn/sls/scheduler/Tracker.java | 46 +
.../apache/hadoop/yarn/sls/utils/SLSUtils.java | 1 +
.../apache/hadoop/yarn/sls/web/SLSWebApp.java | 12 +-
.../yarn/sls/appmaster/TestAMSimulator.java | 53 +-
.../yarn/sls/nodemanager/TestNMSimulator.java | 32 +-
12 files changed, 1103 insertions(+), 1631 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 61b7f36..ba43816 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -59,16 +59,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.sls.appmaster.AMSimulator;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.sls.nodemanager.NMSimulator;
import org.apache.hadoop.yarn.sls.resourcemanager.MockAMLauncher;
-import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
-import org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper;
-import org.apache.hadoop.yarn.sls.scheduler.SLSCapacityScheduler;
-import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper;
-import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
+import org.apache.hadoop.yarn.sls.scheduler.*;
import org.apache.hadoop.yarn.sls.utils.SLSUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.log4j.Logger;
@@ -152,9 +150,9 @@ public class SLSRunner {
// start application masters
startAM();
// set queue & tracked apps information
- ((SchedulerWrapper) rm.getResourceScheduler())
+ ((SchedulerWrapper) rm.getResourceScheduler()).getTracker()
.setQueueSet(this.queueAppNumMap.keySet());
- ((SchedulerWrapper) rm.getResourceScheduler())
+ ((SchedulerWrapper) rm.getResourceScheduler()).getTracker()
.setTrackedAppSet(this.trackedApps);
// print out simulation info
printSimulationInfo();
@@ -164,7 +162,7 @@ public class SLSRunner {
runner.start();
}
- private void startRM() throws IOException, ClassNotFoundException {
+ private void startRM() throws Exception {
Configuration rmConf = new YarnConfiguration();
String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER);
@@ -175,10 +173,12 @@ public class SLSRunner {
if(Class.forName(schedulerClass) == CapacityScheduler.class) {
rmConf.set(YarnConfiguration.RM_SCHEDULER,
SLSCapacityScheduler.class.getName());
- } else {
+ } else if (Class.forName(schedulerClass) == FairScheduler.class) {
rmConf.set(YarnConfiguration.RM_SCHEDULER,
- ResourceSchedulerWrapper.class.getName());
- rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass);
+ SLSFairScheduler.class.getName());
+ } else if (Class.forName(schedulerClass) == FifoScheduler.class){
+ // TODO add support for FifoScheduler
+ throw new Exception("Fifo Scheduler is not supported yet.");
}
rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
index 0573bae..a62f2b6 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.sls.scheduler.SchedulerMetrics;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.log4j.Logger;
@@ -219,10 +220,12 @@ public abstract class AMSimulator extends TaskRunner.Task {
simulateFinishTimeMS = System.currentTimeMillis() -
SLSRunner.getRunner().getStartTimeMS();
// record job running information
- ((SchedulerWrapper)rm.getResourceScheduler())
- .addAMRuntime(appId,
- traceStartTimeMS, traceFinishTimeMS,
- simulateStartTimeMS, simulateFinishTimeMS);
+ SchedulerMetrics schedulerMetrics =
+ ((SchedulerWrapper)rm.getResourceScheduler()).getSchedulerMetrics();
+ if (schedulerMetrics != null) {
+ schedulerMetrics.addAMRuntime(appId, traceStartTimeMS, traceFinishTimeMS,
+ simulateStartTimeMS, simulateFinishTimeMS);
+ }
}
protected ResourceRequest createResourceRequest(
@@ -330,13 +333,20 @@ public abstract class AMSimulator extends TaskRunner.Task {
private void trackApp() {
if (isTracked) {
- ((SchedulerWrapper) rm.getResourceScheduler())
- .addTrackedApp(appId, oldAppId);
+ SchedulerMetrics schedulerMetrics =
+ ((SchedulerWrapper)rm.getResourceScheduler()).getSchedulerMetrics();
+ if (schedulerMetrics != null) {
+ schedulerMetrics.addTrackedApp(appId, oldAppId);
+ }
}
}
public void untrackApp() {
if (isTracked) {
- ((SchedulerWrapper) rm.getResourceScheduler()).removeTrackedApp(oldAppId);
+ SchedulerMetrics schedulerMetrics =
+ ((SchedulerWrapper)rm.getResourceScheduler()).getSchedulerMetrics();
+ if (schedulerMetrics != null) {
+ schedulerMetrics.removeTrackedApp(oldAppId);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
deleted file mode 100644
index a4b8e64..0000000
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
+++ /dev/null
@@ -1,969 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.sls.scheduler;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.QueueACL;
-import org.apache.hadoop.yarn.api.records.QueueInfo;
-import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
-import org.apache.hadoop.yarn.sls.SLSRunner;
-import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
-import org.apache.hadoop.yarn.sls.web.SLSWebApp;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.log4j.Logger;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.CsvReporter;
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.SlidingWindowReservoir;
-import com.codahale.metrics.Timer;
-
-@Private
-@Unstable
-final public class ResourceSchedulerWrapper
- extends AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>
- implements SchedulerWrapper, ResourceScheduler, Configurable {
- private static final String EOL = System.getProperty("line.separator");
- private static final int SAMPLING_SIZE = 60;
- private ScheduledExecutorService pool;
- // counters for scheduler allocate/handle operations
- private Counter schedulerAllocateCounter;
- private Counter schedulerHandleCounter;
- private Map<SchedulerEventType, Counter> schedulerHandleCounterMap;
- // Timers for scheduler allocate/handle operations
- private Timer schedulerAllocateTimer;
- private Timer schedulerHandleTimer;
- private Map<SchedulerEventType, Timer> schedulerHandleTimerMap;
- private List<Histogram> schedulerHistogramList;
- private Map<Histogram, Timer> histogramTimerMap;
- private Lock samplerLock;
- private Lock queueLock;
-
- private Configuration conf;
- private ResourceScheduler scheduler;
- private Map<ApplicationId, String> appQueueMap =
- new ConcurrentHashMap<ApplicationId, String>();
- private BufferedWriter jobRuntimeLogBW;
-
- // Priority of the ResourceSchedulerWrapper shutdown hook.
- public static final int SHUTDOWN_HOOK_PRIORITY = 30;
-
- // web app
- private SLSWebApp web;
-
- private Map<ContainerId, Resource> preemptionContainerMap =
- new ConcurrentHashMap<ContainerId, Resource>();
-
- // metrics
- private MetricRegistry metrics;
- private SchedulerMetrics schedulerMetrics;
- private boolean metricsON;
- private String metricsOutputDir;
- private BufferedWriter metricsLogBW;
- private boolean running = false;
- private static Map<Class, Class> defaultSchedulerMetricsMap =
- new HashMap<Class, Class>();
- static {
- defaultSchedulerMetricsMap.put(FairScheduler.class,
- FairSchedulerMetrics.class);
- defaultSchedulerMetricsMap.put(FifoScheduler.class,
- FifoSchedulerMetrics.class);
- defaultSchedulerMetricsMap.put(CapacityScheduler.class,
- CapacitySchedulerMetrics.class);
- }
- // must set by outside
- private Set<String> queueSet;
- private Set<String> trackedAppSet;
-
- public final Logger LOG = Logger.getLogger(ResourceSchedulerWrapper.class);
-
- public ResourceSchedulerWrapper() {
- super(ResourceSchedulerWrapper.class.getName());
- samplerLock = new ReentrantLock();
- queueLock = new ReentrantLock();
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.conf = conf;
- // set scheduler
- Class<? extends ResourceScheduler> klass = conf.getClass(
- SLSConfiguration.RM_SCHEDULER, null, ResourceScheduler.class);
-
- scheduler = ReflectionUtils.newInstance(klass, conf);
- // start metrics
- metricsON = conf.getBoolean(SLSConfiguration.METRICS_SWITCH, true);
- if (metricsON) {
- try {
- initMetrics();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- ShutdownHookManager.get().addShutdownHook(new Runnable() {
- @Override
- public void run() {
- try {
- if (metricsLogBW != null) {
- metricsLogBW.write("]");
- metricsLogBW.close();
- }
- if (web != null) {
- web.stop();
- }
- tearDown();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }, SHUTDOWN_HOOK_PRIORITY);
- }
-
- @Override
- public Allocation allocate(ApplicationAttemptId attemptId,
- List<ResourceRequest> resourceRequests, List<ContainerId> containerIds,
- List<String> strings, List<String> strings2,
- ContainerUpdates updateRequests) {
- if (metricsON) {
- final Timer.Context context = schedulerAllocateTimer.time();
- Allocation allocation = null;
- try {
- allocation = scheduler.allocate(attemptId, resourceRequests,
- containerIds, strings, strings2, updateRequests);
- return allocation;
- } finally {
- context.stop();
- schedulerAllocateCounter.inc();
- try {
- updateQueueWithAllocateRequest(allocation, attemptId,
- resourceRequests, containerIds);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- } else {
- return scheduler.allocate(attemptId,
- resourceRequests, containerIds, strings, strings2, updateRequests);
- }
- }
-
- @Override
- public void handle(SchedulerEvent schedulerEvent) {
- // metrics off
- if (! metricsON) {
- scheduler.handle(schedulerEvent);
- return;
- }
- if(!running) running = true;
-
- // metrics on
- Timer.Context handlerTimer = null;
- Timer.Context operationTimer = null;
-
- NodeUpdateSchedulerEventWrapper eventWrapper;
- try {
- //if (schedulerEvent instanceof NodeUpdateSchedulerEvent) {
- if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE
- && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
- eventWrapper = new NodeUpdateSchedulerEventWrapper(
- (NodeUpdateSchedulerEvent)schedulerEvent);
- schedulerEvent = eventWrapper;
- updateQueueWithNodeUpdate(eventWrapper);
- } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
- && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
- // check if having AM Container, update resource usage information
- AppAttemptRemovedSchedulerEvent appRemoveEvent =
- (AppAttemptRemovedSchedulerEvent) schedulerEvent;
- ApplicationAttemptId appAttemptId =
- appRemoveEvent.getApplicationAttemptID();
- String queue = appQueueMap.get(appAttemptId.getApplicationId());
- SchedulerAppReport app = scheduler.getSchedulerAppInfo(appAttemptId);
- if (! app.getLiveContainers().isEmpty()) { // have 0 or 1
- // should have one container which is AM container
- RMContainer rmc = app.getLiveContainers().iterator().next();
- updateQueueMetrics(queue,
- rmc.getContainer().getResource().getMemorySize(),
- rmc.getContainer().getResource().getVirtualCores());
- }
- }
-
- handlerTimer = schedulerHandleTimer.time();
- operationTimer = schedulerHandleTimerMap
- .get(schedulerEvent.getType()).time();
-
- scheduler.handle(schedulerEvent);
- } finally {
- if (handlerTimer != null) handlerTimer.stop();
- if (operationTimer != null) operationTimer.stop();
- schedulerHandleCounter.inc();
- schedulerHandleCounterMap.get(schedulerEvent.getType()).inc();
-
- if (schedulerEvent.getType() == SchedulerEventType.APP_REMOVED
- && schedulerEvent instanceof AppRemovedSchedulerEvent) {
- SLSRunner.decreaseRemainingApps();
- AppRemovedSchedulerEvent appRemoveEvent =
- (AppRemovedSchedulerEvent) schedulerEvent;
- appQueueMap.remove(appRemoveEvent.getApplicationID());
- } else if (schedulerEvent.getType() == SchedulerEventType.APP_ADDED
- && schedulerEvent instanceof AppAddedSchedulerEvent) {
- AppAddedSchedulerEvent appAddEvent =
- (AppAddedSchedulerEvent) schedulerEvent;
- String queueName = appAddEvent.getQueue();
- appQueueMap.put(appAddEvent.getApplicationId(), queueName);
- }
- }
- }
-
- private void updateQueueWithNodeUpdate(
- NodeUpdateSchedulerEventWrapper eventWrapper) {
- RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
- List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
- for (UpdatedContainerInfo info : containerList) {
- for (ContainerStatus status : info.getCompletedContainers()) {
- ContainerId containerId = status.getContainerId();
- SchedulerAppReport app = scheduler.getSchedulerAppInfo(
- containerId.getApplicationAttemptId());
-
- if (app == null) {
- // this happens for the AM container
- // The app have already removed when the NM sends the release
- // information.
- continue;
- }
-
- String queue =
- appQueueMap.get(containerId.getApplicationAttemptId()
- .getApplicationId());
- int releasedMemory = 0, releasedVCores = 0;
- if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
- for (RMContainer rmc : app.getLiveContainers()) {
- if (rmc.getContainerId() == containerId) {
- releasedMemory += rmc.getContainer().getResource().getMemorySize();
- releasedVCores += rmc.getContainer()
- .getResource().getVirtualCores();
- break;
- }
- }
- } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
- if (preemptionContainerMap.containsKey(containerId)) {
- Resource preResource = preemptionContainerMap.get(containerId);
- releasedMemory += preResource.getMemorySize();
- releasedVCores += preResource.getVirtualCores();
- preemptionContainerMap.remove(containerId);
- }
- }
- // update queue counters
- updateQueueMetrics(queue, releasedMemory, releasedVCores);
- }
- }
- }
-
- private void updateQueueWithAllocateRequest(Allocation allocation,
- ApplicationAttemptId attemptId,
- List<ResourceRequest> resourceRequests,
- List<ContainerId> containerIds) throws IOException {
- // update queue information
- Resource pendingResource = Resources.createResource(0, 0);
- Resource allocatedResource = Resources.createResource(0, 0);
- String queueName = appQueueMap.get(attemptId.getApplicationId());
- // container requested
- for (ResourceRequest request : resourceRequests) {
- if (request.getResourceName().equals(ResourceRequest.ANY)) {
- Resources.addTo(pendingResource,
- Resources.multiply(request.getCapability(),
- request.getNumContainers()));
- }
- }
- // container allocated
- for (Container container : allocation.getContainers()) {
- Resources.addTo(allocatedResource, container.getResource());
- Resources.subtractFrom(pendingResource, container.getResource());
- }
- // container released from AM
- SchedulerAppReport report = scheduler.getSchedulerAppInfo(attemptId);
- for (ContainerId containerId : containerIds) {
- Container container = null;
- for (RMContainer c : report.getLiveContainers()) {
- if (c.getContainerId().equals(containerId)) {
- container = c.getContainer();
- break;
- }
- }
- if (container != null) {
- // released allocated containers
- Resources.subtractFrom(allocatedResource, container.getResource());
- } else {
- for (RMContainer c : report.getReservedContainers()) {
- if (c.getContainerId().equals(containerId)) {
- container = c.getContainer();
- break;
- }
- }
- if (container != null) {
- // released reserved containers
- Resources.subtractFrom(pendingResource, container.getResource());
- }
- }
- }
- // containers released/preemption from scheduler
- Set<ContainerId> preemptionContainers = new HashSet<ContainerId>();
- if (allocation.getContainerPreemptions() != null) {
- preemptionContainers.addAll(allocation.getContainerPreemptions());
- }
- if (allocation.getStrictContainerPreemptions() != null) {
- preemptionContainers.addAll(allocation.getStrictContainerPreemptions());
- }
- if (! preemptionContainers.isEmpty()) {
- for (ContainerId containerId : preemptionContainers) {
- if (! preemptionContainerMap.containsKey(containerId)) {
- Container container = null;
- for (RMContainer c : report.getLiveContainers()) {
- if (c.getContainerId().equals(containerId)) {
- container = c.getContainer();
- break;
- }
- }
- if (container != null) {
- preemptionContainerMap.put(containerId, container.getResource());
- }
- }
-
- }
- }
-
- // update metrics
- SortedMap<String, Counter> counterMap = metrics.getCounters();
- String names[] = new String[]{
- "counter.queue." + queueName + ".pending.memory",
- "counter.queue." + queueName + ".pending.cores",
- "counter.queue." + queueName + ".allocated.memory",
- "counter.queue." + queueName + ".allocated.cores"};
- long values[] = new long[]{pendingResource.getMemorySize(),
- pendingResource.getVirtualCores(),
- allocatedResource.getMemorySize(), allocatedResource.getVirtualCores()};
- for (int i = names.length - 1; i >= 0; i --) {
- if (! counterMap.containsKey(names[i])) {
- metrics.counter(names[i]);
- counterMap = metrics.getCounters();
- }
- counterMap.get(names[i]).inc(values[i]);
- }
-
- queueLock.lock();
- try {
- if (! schedulerMetrics.isTracked(queueName)) {
- schedulerMetrics.trackQueue(queueName);
- }
- } finally {
- queueLock.unlock();
- }
- }
-
- private void tearDown() throws IOException {
- // close job runtime writer
- if (jobRuntimeLogBW != null) {
- jobRuntimeLogBW.close();
- }
- // shut pool
- if (pool != null) pool.shutdown();
- }
-
- @SuppressWarnings("unchecked")
- private void initMetrics() throws Exception {
- metrics = new MetricRegistry();
- // configuration
- metricsOutputDir = conf.get(SLSConfiguration.METRICS_OUTPUT_DIR);
- int metricsWebAddressPort = conf.getInt(
- SLSConfiguration.METRICS_WEB_ADDRESS_PORT,
- SLSConfiguration.METRICS_WEB_ADDRESS_PORT_DEFAULT);
- // create SchedulerMetrics for current scheduler
- String schedulerMetricsType = conf.get(scheduler.getClass().getName());
- Class schedulerMetricsClass = schedulerMetricsType == null?
- defaultSchedulerMetricsMap.get(scheduler.getClass()) :
- Class.forName(schedulerMetricsType);
- schedulerMetrics = (SchedulerMetrics)ReflectionUtils
- .newInstance(schedulerMetricsClass, new Configuration());
- schedulerMetrics.init(scheduler, metrics);
-
- // register various metrics
- registerJvmMetrics();
- registerClusterResourceMetrics();
- registerContainerAppNumMetrics();
- registerSchedulerMetrics();
-
- // .csv output
- initMetricsCSVOutput();
-
- // start web app to provide real-time tracking
- web = new SLSWebApp(this, metricsWebAddressPort);
- web.start();
-
- // a thread to update histogram timer
- pool = new ScheduledThreadPoolExecutor(2);
- pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000,
- TimeUnit.MILLISECONDS);
-
- // a thread to output metrics for real-tiem tracking
- pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000,
- TimeUnit.MILLISECONDS);
-
- // application running information
- jobRuntimeLogBW =
- new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
- metricsOutputDir + "/jobruntime.csv"), "UTF-8"));
- jobRuntimeLogBW.write("JobID,real_start_time,real_end_time," +
- "simulate_start_time,simulate_end_time" + EOL);
- jobRuntimeLogBW.flush();
- }
-
- private void registerJvmMetrics() {
- // add JVM gauges
- metrics.register("variable.jvm.free.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- return Runtime.getRuntime().freeMemory();
- }
- }
- );
- metrics.register("variable.jvm.max.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- return Runtime.getRuntime().maxMemory();
- }
- }
- );
- metrics.register("variable.jvm.total.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- return Runtime.getRuntime().totalMemory();
- }
- }
- );
- }
-
- private void registerClusterResourceMetrics() {
- metrics.register("variable.cluster.allocated.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
- return 0L;
- } else {
- return scheduler.getRootQueueMetrics().getAllocatedMB();
- }
- }
- }
- );
- metrics.register("variable.cluster.allocated.vcores",
- new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
- return 0;
- } else {
- return scheduler.getRootQueueMetrics().getAllocatedVirtualCores();
- }
- }
- }
- );
- metrics.register("variable.cluster.available.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
- return 0L;
- } else {
- return scheduler.getRootQueueMetrics().getAvailableMB();
- }
- }
- }
- );
- metrics.register("variable.cluster.available.vcores",
- new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
- return 0;
- } else {
- return scheduler.getRootQueueMetrics().getAvailableVirtualCores();
- }
- }
- }
- );
- }
-
- private void registerContainerAppNumMetrics() {
- metrics.register("variable.running.application",
- new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- if (scheduler == null || scheduler.getRootQueueMetrics() == null) {
- return 0;
- } else {
- return scheduler.getRootQueueMetrics().getAppsRunning();
- }
- }
- }
- );
- metrics.register("variable.running.container",
- new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
- return 0;
- } else {
- return scheduler.getRootQueueMetrics().getAllocatedContainers();
- }
- }
- }
- );
- }
-
- private void registerSchedulerMetrics() {
- samplerLock.lock();
- try {
- // counters for scheduler operations
- schedulerAllocateCounter = metrics.counter(
- "counter.scheduler.operation.allocate");
- schedulerHandleCounter = metrics.counter(
- "counter.scheduler.operation.handle");
- schedulerHandleCounterMap = new HashMap<SchedulerEventType, Counter>();
- for (SchedulerEventType e : SchedulerEventType.values()) {
- Counter counter = metrics.counter(
- "counter.scheduler.operation.handle." + e);
- schedulerHandleCounterMap.put(e, counter);
- }
- // timers for scheduler operations
- int timeWindowSize = conf.getInt(
- SLSConfiguration.METRICS_TIMER_WINDOW_SIZE,
- SLSConfiguration.METRICS_TIMER_WINDOW_SIZE_DEFAULT);
- schedulerAllocateTimer = new Timer(
- new SlidingWindowReservoir(timeWindowSize));
- schedulerHandleTimer = new Timer(
- new SlidingWindowReservoir(timeWindowSize));
- schedulerHandleTimerMap = new HashMap<SchedulerEventType, Timer>();
- for (SchedulerEventType e : SchedulerEventType.values()) {
- Timer timer = new Timer(new SlidingWindowReservoir(timeWindowSize));
- schedulerHandleTimerMap.put(e, timer);
- }
- // histogram for scheduler operations (Samplers)
- schedulerHistogramList = new ArrayList<Histogram>();
- histogramTimerMap = new HashMap<Histogram, Timer>();
- Histogram schedulerAllocateHistogram = new Histogram(
- new SlidingWindowReservoir(SAMPLING_SIZE));
- metrics.register("sampler.scheduler.operation.allocate.timecost",
- schedulerAllocateHistogram);
- schedulerHistogramList.add(schedulerAllocateHistogram);
- histogramTimerMap.put(schedulerAllocateHistogram, schedulerAllocateTimer);
- Histogram schedulerHandleHistogram = new Histogram(
- new SlidingWindowReservoir(SAMPLING_SIZE));
- metrics.register("sampler.scheduler.operation.handle.timecost",
- schedulerHandleHistogram);
- schedulerHistogramList.add(schedulerHandleHistogram);
- histogramTimerMap.put(schedulerHandleHistogram, schedulerHandleTimer);
- for (SchedulerEventType e : SchedulerEventType.values()) {
- Histogram histogram = new Histogram(
- new SlidingWindowReservoir(SAMPLING_SIZE));
- metrics.register(
- "sampler.scheduler.operation.handle." + e + ".timecost",
- histogram);
- schedulerHistogramList.add(histogram);
- histogramTimerMap.put(histogram, schedulerHandleTimerMap.get(e));
- }
- } finally {
- samplerLock.unlock();
- }
- }
-
- private void initMetricsCSVOutput() {
- int timeIntervalMS = conf.getInt(
- SLSConfiguration.METRICS_RECORD_INTERVAL_MS,
- SLSConfiguration.METRICS_RECORD_INTERVAL_MS_DEFAULT);
- File dir = new File(metricsOutputDir + "/metrics");
- if(! dir.exists()
- && ! dir.mkdirs()) {
- LOG.error("Cannot create directory " + dir.getAbsoluteFile());
- }
- final CsvReporter reporter = CsvReporter.forRegistry(metrics)
- .formatFor(Locale.US)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build(new File(metricsOutputDir + "/metrics"));
- reporter.start(timeIntervalMS, TimeUnit.MILLISECONDS);
- }
-
- class HistogramsRunnable implements Runnable {
- @Override
- public void run() {
- samplerLock.lock();
- try {
- for (Histogram histogram : schedulerHistogramList) {
- Timer timer = histogramTimerMap.get(histogram);
- histogram.update((int) timer.getSnapshot().getMean());
- }
- } finally {
- samplerLock.unlock();
- }
- }
- }
-
- class MetricsLogRunnable implements Runnable {
- private boolean firstLine = true;
- public MetricsLogRunnable() {
- try {
- metricsLogBW =
- new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
- metricsOutputDir + "/realtimetrack.json"), "UTF-8"));
- metricsLogBW.write("[");
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public void run() {
- if(running) {
- // all WebApp to get real tracking json
- String metrics = web.generateRealTimeTrackingMetrics();
- // output
- try {
- if(firstLine) {
- metricsLogBW.write(metrics + EOL);
- firstLine = false;
- } else {
- metricsLogBW.write("," + metrics + EOL);
- }
- metricsLogBW.flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- // the following functions are used by AMSimulator
- public void addAMRuntime(ApplicationId appId,
- long traceStartTimeMS, long traceEndTimeMS,
- long simulateStartTimeMS, long simulateEndTimeMS) {
- if (metricsON) {
- try {
- // write job runtime information
- StringBuilder sb = new StringBuilder();
- sb.append(appId).append(",").append(traceStartTimeMS).append(",")
- .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
- .append(",").append(simulateEndTimeMS);
- jobRuntimeLogBW.write(sb.toString() + EOL);
- jobRuntimeLogBW.flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
-
- private void updateQueueMetrics(String queue,
- long releasedMemory, int releasedVCores) {
- // update queue counters
- SortedMap<String, Counter> counterMap = metrics.getCounters();
- if (releasedMemory != 0) {
- String name = "counter.queue." + queue + ".allocated.memory";
- if (! counterMap.containsKey(name)) {
- metrics.counter(name);
- counterMap = metrics.getCounters();
- }
- counterMap.get(name).inc(-releasedMemory);
- }
- if (releasedVCores != 0) {
- String name = "counter.queue." + queue + ".allocated.cores";
- if (! counterMap.containsKey(name)) {
- metrics.counter(name);
- counterMap = metrics.getCounters();
- }
- counterMap.get(name).inc(-releasedVCores);
- }
- }
-
- public void setQueueSet(Set<String> queues) {
- this.queueSet = queues;
- }
-
- public Set<String> getQueueSet() {
- return this.queueSet;
- }
-
- public void setTrackedAppSet(Set<String> apps) {
- this.trackedAppSet = apps;
- }
-
- public Set<String> getTrackedAppSet() {
- return this.trackedAppSet;
- }
-
- public MetricRegistry getMetrics() {
- return metrics;
- }
-
- public SchedulerMetrics getSchedulerMetrics() {
- return schedulerMetrics;
- }
-
- // API open to out classes
- public void addTrackedApp(ApplicationId appId, String oldAppId) {
- if (metricsON) {
- schedulerMetrics.trackApp(appId, oldAppId);
- }
- }
-
- public void removeTrackedApp(String oldAppId) {
- if (metricsON) {
- schedulerMetrics.untrackApp(oldAppId);
- }
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
- scheduler).init(conf);
- super.serviceInit(conf);
- initScheduler(conf);
- }
-
- private synchronized void initScheduler(Configuration configuration) throws
- IOException {
- this.applications =
- new ConcurrentHashMap<ApplicationId,
- SchedulerApplication<SchedulerApplicationAttempt>>();
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void serviceStart() throws Exception {
- ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
- scheduler).start();
- super.serviceStart();
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void serviceStop() throws Exception {
- ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
- scheduler).stop();
- super.serviceStop();
- }
-
- @Override
- public void setRMContext(RMContext rmContext) {
- scheduler.setRMContext(rmContext);
- }
-
- @Override
- public void reinitialize(Configuration conf, RMContext rmContext)
- throws IOException {
- scheduler.reinitialize(conf, rmContext);
- }
-
- @Override
- public void recover(RMStateStore.RMState rmState) throws Exception {
- scheduler.recover(rmState);
- }
-
- @Override
- public QueueInfo getQueueInfo(String s, boolean b, boolean b2)
- throws IOException {
- return scheduler.getQueueInfo(s, b, b2);
- }
-
- @Override
- public List<QueueUserACLInfo> getQueueUserAclInfo() {
- return scheduler.getQueueUserAclInfo();
- }
-
- @Override
- public Resource getMinimumResourceCapability() {
- return scheduler.getMinimumResourceCapability();
- }
-
- @Override
- public Resource getMaximumResourceCapability() {
- return scheduler.getMaximumResourceCapability();
- }
-
- @Override
- public ResourceCalculator getResourceCalculator() {
- return scheduler.getResourceCalculator();
- }
-
- @Override
- public int getNumClusterNodes() {
- return scheduler.getNumClusterNodes();
- }
-
- @Override
- public SchedulerNodeReport getNodeReport(NodeId nodeId) {
- return scheduler.getNodeReport(nodeId);
- }
-
- @Override
- public SchedulerAppReport getSchedulerAppInfo(
- ApplicationAttemptId attemptId) {
- return scheduler.getSchedulerAppInfo(attemptId);
- }
-
- @Override
- public QueueMetrics getRootQueueMetrics() {
- return scheduler.getRootQueueMetrics();
- }
-
- @Override
- public synchronized boolean checkAccess(UserGroupInformation callerUGI,
- QueueACL acl, String queueName) {
- return scheduler.checkAccess(callerUGI, acl, queueName);
- }
-
- @Override
- public ApplicationResourceUsageReport getAppResourceUsageReport(
- ApplicationAttemptId appAttemptId) {
- return scheduler.getAppResourceUsageReport(appAttemptId);
- }
-
- @Override
- public List<ApplicationAttemptId> getAppsInQueue(String queue) {
- return scheduler.getAppsInQueue(queue);
- }
-
- @Override
- public RMContainer getRMContainer(ContainerId containerId) {
- return null;
- }
-
- @Override
- public String moveApplication(ApplicationId appId, String newQueue)
- throws YarnException {
- return scheduler.moveApplication(appId, newQueue);
- }
-
- @Override
- @LimitedPrivate("yarn")
- @Unstable
- public Resource getClusterResource() {
- return super.getClusterResource();
- }
-
- @Override
- public synchronized List<Container> getTransferredContainers(
- ApplicationAttemptId currentAttempt) {
- return new ArrayList<Container>();
- }
-
- @Override
- public Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>>
- getSchedulerApplications() {
- return new HashMap<ApplicationId,
- SchedulerApplication<SchedulerApplicationAttempt>>();
- }
-
- @Override
- protected void completedContainerInternal(RMContainer rmContainer,
- ContainerStatus containerStatus, RMContainerEventType event) {
- // do nothing
- }
-
- @Override
- public Priority checkAndGetApplicationPriority(Priority priority,
- UserGroupInformation user, String queueName, ApplicationId applicationId)
- throws YarnException {
- // TODO Dummy implementation.
- return Priority.newInstance(0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index 6ea2ab0..7c37465 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -17,34 +17,19 @@
*/
package org.apache.hadoop.yarn.sls.scheduler;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.Set;
-import java.util.SortedMap;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -65,117 +50,63 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptR
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
-import org.apache.hadoop.yarn.sls.web.SLSWebApp;
+import org.apache.hadoop.yarn.sls.utils.SLSUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.log4j.Logger;
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.CsvReporter;
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.SlidingWindowReservoir;
import com.codahale.metrics.Timer;
@Private
@Unstable
public class SLSCapacityScheduler extends CapacityScheduler implements
SchedulerWrapper,Configurable {
- private static final String EOL = System.getProperty("line.separator");
- private static final String QUEUE_COUNTER_PREFIX = "counter.queue.";
- private static final int SAMPLING_SIZE = 60;
- private ScheduledExecutorService pool;
- // counters for scheduler allocate/handle operations
- private Counter schedulerAllocateCounter;
- private Counter schedulerHandleCounter;
- private Map<SchedulerEventType, Counter> schedulerHandleCounterMap;
- // Timers for scheduler allocate/handle operations
- private Timer schedulerAllocateTimer;
- private Timer schedulerHandleTimer;
- private Map<SchedulerEventType, Timer> schedulerHandleTimerMap;
- private List<Histogram> schedulerHistogramList;
- private Map<Histogram, Timer> histogramTimerMap;
- private Lock samplerLock;
- private Lock queueLock;
private Configuration conf;
private Map<ApplicationAttemptId, String> appQueueMap =
new ConcurrentHashMap<ApplicationAttemptId, String>();
- private BufferedWriter jobRuntimeLogBW;
-
- // Priority of the ResourceSchedulerWrapper shutdown hook.
- public static final int SHUTDOWN_HOOK_PRIORITY = 30;
-
- // web app
- private SLSWebApp web;
private Map<ContainerId, Resource> preemptionContainerMap =
new ConcurrentHashMap<ContainerId, Resource>();
// metrics
- private MetricRegistry metrics;
private SchedulerMetrics schedulerMetrics;
private boolean metricsON;
- private String metricsOutputDir;
- private BufferedWriter metricsLogBW;
- private boolean running = false;
- private static Map<Class, Class> defaultSchedulerMetricsMap =
- new HashMap<Class, Class>();
- static {
- defaultSchedulerMetricsMap.put(FairScheduler.class,
- FairSchedulerMetrics.class);
- defaultSchedulerMetricsMap.put(FifoScheduler.class,
- FifoSchedulerMetrics.class);
- defaultSchedulerMetricsMap.put(CapacityScheduler.class,
- CapacitySchedulerMetrics.class);
- }
- // must set by outside
- private Set<String> queueSet;
- private Set<String> trackedAppSet;
+ private Tracker tracker;
- public final Logger LOG = Logger.getLogger(SLSCapacityScheduler.class);
+ public Tracker getTracker() {
+ return tracker;
+ }
public SLSCapacityScheduler() {
- samplerLock = new ReentrantLock();
- queueLock = new ReentrantLock();
+ tracker = new Tracker();
}
@Override
public void setConf(Configuration conf) {
this.conf = conf;
super.setConf(conf);
- // start metrics
metricsON = conf.getBoolean(SLSConfiguration.METRICS_SWITCH, true);
if (metricsON) {
try {
- initMetrics();
+ schedulerMetrics = SchedulerMetrics.getInstance(conf,
+ CapacityScheduler.class);
+ schedulerMetrics.init(this, conf);
} catch (Exception e) {
e.printStackTrace();
}
- }
- ShutdownHookManager.get().addShutdownHook(new Runnable() {
- @Override
- public void run() {
- try {
- if (metricsLogBW != null) {
- metricsLogBW.write("]");
- metricsLogBW.close();
- }
- if (web != null) {
- web.stop();
+ ShutdownHookManager.get().addShutdownHook(new Runnable() {
+ @Override public void run() {
+ try {
+ schedulerMetrics.tearDown();
+ } catch (Exception e) {
+ e.printStackTrace();
}
- tearDown();
- } catch (Exception e) {
- e.printStackTrace();
}
- }
- }, SHUTDOWN_HOOK_PRIORITY);
+ }, SLSUtils.SHUTDOWN_HOOK_PRIORITY);
+ }
}
@Override
@@ -184,7 +115,8 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
List<String> strings, List<String> strings2,
ContainerUpdates updateRequests) {
if (metricsON) {
- final Timer.Context context = schedulerAllocateTimer.time();
+ final Timer.Context context = schedulerMetrics.getSchedulerAllocateTimer()
+ .time();
Allocation allocation = null;
try {
allocation = super
@@ -193,7 +125,7 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
return allocation;
} finally {
context.stop();
- schedulerAllocateCounter.inc();
+ schedulerMetrics.increaseSchedulerAllocationCounter();
try {
updateQueueWithAllocateRequest(allocation, attemptId,
resourceRequests, containerIds);
@@ -209,74 +141,76 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
@Override
public void handle(SchedulerEvent schedulerEvent) {
- // metrics off
- if (! metricsON) {
- super.handle(schedulerEvent);
- return;
- }
- if(!running) running = true;
+ if (!metricsON) {
+ super.handle(schedulerEvent);
+ return;
+ }
- // metrics on
- Timer.Context handlerTimer = null;
- Timer.Context operationTimer = null;
+ if (!schedulerMetrics.isRunning()) {
+ schedulerMetrics.setRunning(true);
+ }
- NodeUpdateSchedulerEventWrapper eventWrapper;
- try {
- //if (schedulerEvent instanceof NodeUpdateSchedulerEvent) {
- if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE
- && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
- eventWrapper = new NodeUpdateSchedulerEventWrapper(
- (NodeUpdateSchedulerEvent)schedulerEvent);
- schedulerEvent = eventWrapper;
- updateQueueWithNodeUpdate(eventWrapper);
- } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
- && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
- // check if having AM Container, update resource usage information
- AppAttemptRemovedSchedulerEvent appRemoveEvent =
- (AppAttemptRemovedSchedulerEvent) schedulerEvent;
- ApplicationAttemptId appAttemptId =
- appRemoveEvent.getApplicationAttemptID();
- String queue = appQueueMap.get(appAttemptId);
- SchedulerAppReport app = super.getSchedulerAppInfo(appAttemptId);
- if (! app.getLiveContainers().isEmpty()) { // have 0 or 1
- // should have one container which is AM container
- RMContainer rmc = app.getLiveContainers().iterator().next();
- updateQueueMetrics(queue,
- rmc.getContainer().getResource().getMemorySize(),
- rmc.getContainer().getResource().getVirtualCores());
- }
- }
+ Timer.Context handlerTimer = null;
+ Timer.Context operationTimer = null;
- handlerTimer = schedulerHandleTimer.time();
- operationTimer = schedulerHandleTimerMap
- .get(schedulerEvent.getType()).time();
+ NodeUpdateSchedulerEventWrapper eventWrapper;
+ try {
+ if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE
+ && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
+ eventWrapper = new NodeUpdateSchedulerEventWrapper(
+ (NodeUpdateSchedulerEvent)schedulerEvent);
+ schedulerEvent = eventWrapper;
+ updateQueueWithNodeUpdate(eventWrapper);
+ } else if (schedulerEvent.getType() ==
+ SchedulerEventType.APP_ATTEMPT_REMOVED
+ && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
+ // check if having AM Container, update resource usage information
+ AppAttemptRemovedSchedulerEvent appRemoveEvent =
+ (AppAttemptRemovedSchedulerEvent) schedulerEvent;
+ ApplicationAttemptId appAttemptId =
+ appRemoveEvent.getApplicationAttemptID();
+ String queue = appQueueMap.get(appAttemptId);
+ SchedulerAppReport app = super.getSchedulerAppInfo(appAttemptId);
+ if (!app.getLiveContainers().isEmpty()) { // have 0 or 1
+ // should have one container which is AM container
+ RMContainer rmc = app.getLiveContainers().iterator().next();
+ schedulerMetrics.updateQueueMetricsByRelease(
+ rmc.getContainer().getResource(), queue);
+ }
+ }
- super.handle(schedulerEvent);
- } finally {
- if (handlerTimer != null) handlerTimer.stop();
- if (operationTimer != null) operationTimer.stop();
- schedulerHandleCounter.inc();
- schedulerHandleCounterMap.get(schedulerEvent.getType()).inc();
+ handlerTimer = schedulerMetrics.getSchedulerHandleTimer().time();
+ operationTimer = schedulerMetrics.getSchedulerHandleTimer(
+ schedulerEvent.getType()).time();
- if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
- && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
- SLSRunner.decreaseRemainingApps();
- AppAttemptRemovedSchedulerEvent appRemoveEvent =
- (AppAttemptRemovedSchedulerEvent) schedulerEvent;
- ApplicationAttemptId appAttemptId =
- appRemoveEvent.getApplicationAttemptID();
- appQueueMap.remove(appRemoveEvent.getApplicationAttemptID());
- } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_ADDED
- && schedulerEvent instanceof AppAttemptAddedSchedulerEvent) {
- AppAttemptAddedSchedulerEvent appAddEvent =
- (AppAttemptAddedSchedulerEvent) schedulerEvent;
- SchedulerApplication app =
- applications.get(appAddEvent.getApplicationAttemptId()
+ super.handle(schedulerEvent);
+ } finally {
+ if (handlerTimer != null) {
+ handlerTimer.stop();
+ }
+ if (operationTimer != null) {
+ operationTimer.stop();
+ }
+ schedulerMetrics.increaseSchedulerHandleCounter(schedulerEvent.getType());
+
+ if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
+ && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
+ SLSRunner.decreaseRemainingApps();
+ AppAttemptRemovedSchedulerEvent appRemoveEvent =
+ (AppAttemptRemovedSchedulerEvent) schedulerEvent;
+ appQueueMap.remove(appRemoveEvent.getApplicationAttemptID());
+ } else if (schedulerEvent.getType() ==
+ SchedulerEventType.APP_ATTEMPT_ADDED
+ && schedulerEvent instanceof AppAttemptAddedSchedulerEvent) {
+ AppAttemptAddedSchedulerEvent appAddEvent =
+ (AppAttemptAddedSchedulerEvent) schedulerEvent;
+ SchedulerApplication app =
+ applications.get(appAddEvent.getApplicationAttemptId()
.getApplicationId());
- appQueueMap.put(appAddEvent.getApplicationAttemptId(), app.getQueue()
- .getQueueName());
- }
- }
+ appQueueMap.put(appAddEvent.getApplicationAttemptId(), app.getQueue()
+ .getQueueName());
+ }
+ }
}
private void updateQueueWithNodeUpdate(
@@ -316,7 +250,8 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
}
}
// update queue counters
- updateQueueMetrics(queue, releasedMemory, releasedVCores);
+ schedulerMetrics.updateQueueMetricsByRelease(
+ Resource.newInstance(releasedMemory, releasedVCores), queue);
}
}
}
@@ -395,410 +330,13 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
}
// update metrics
- SortedMap<String, Counter> counterMap = metrics.getCounters();
- String names[] = new String[]{
- "counter.queue." + queueName + ".pending.memory",
- "counter.queue." + queueName + ".pending.cores",
- "counter.queue." + queueName + ".allocated.memory",
- "counter.queue." + queueName + ".allocated.cores"};
- long values[] = new long[]{pendingResource.getMemorySize(),
- pendingResource.getVirtualCores(),
- allocatedResource.getMemorySize(), allocatedResource.getVirtualCores()};
- for (int i = names.length - 1; i >= 0; i --) {
- if (! counterMap.containsKey(names[i])) {
- metrics.counter(names[i]);
- counterMap = metrics.getCounters();
- }
- counterMap.get(names[i]).inc(values[i]);
- }
-
- queueLock.lock();
- try {
- if (! schedulerMetrics.isTracked(queueName)) {
- schedulerMetrics.trackQueue(queueName);
- }
- } finally {
- queueLock.unlock();
- }
- }
-
- private void tearDown() throws IOException {
- // close job runtime writer
- if (jobRuntimeLogBW != null) {
- jobRuntimeLogBW.close();
- }
- // shut pool
- if (pool != null) pool.shutdown();
- }
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- private void initMetrics() throws Exception {
- metrics = new MetricRegistry();
- // configuration
- metricsOutputDir = conf.get(SLSConfiguration.METRICS_OUTPUT_DIR);
- int metricsWebAddressPort = conf.getInt(
- SLSConfiguration.METRICS_WEB_ADDRESS_PORT,
- SLSConfiguration.METRICS_WEB_ADDRESS_PORT_DEFAULT);
- // create SchedulerMetrics for current scheduler
- String schedulerMetricsType = conf.get(CapacityScheduler.class.getName());
- Class schedulerMetricsClass = schedulerMetricsType == null?
- defaultSchedulerMetricsMap.get(CapacityScheduler.class) :
- Class.forName(schedulerMetricsType);
- schedulerMetrics = (SchedulerMetrics)ReflectionUtils
- .newInstance(schedulerMetricsClass, new Configuration());
- schedulerMetrics.init(this, metrics);
-
- // register various metrics
- registerJvmMetrics();
- registerClusterResourceMetrics();
- registerContainerAppNumMetrics();
- registerSchedulerMetrics();
-
- // .csv output
- initMetricsCSVOutput();
-
- // start web app to provide real-time tracking
- web = new SLSWebApp(this, metricsWebAddressPort);
- web.start();
-
- // a thread to update histogram timer
- pool = new ScheduledThreadPoolExecutor(2);
- pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000,
- TimeUnit.MILLISECONDS);
-
- // a thread to output metrics for real-tiem tracking
- pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000,
- TimeUnit.MILLISECONDS);
-
- // application running information
- jobRuntimeLogBW =
- new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
- metricsOutputDir + "/jobruntime.csv"), "UTF-8"));
- jobRuntimeLogBW.write("JobID,real_start_time,real_end_time," +
- "simulate_start_time,simulate_end_time" + EOL);
- jobRuntimeLogBW.flush();
- }
-
- private void registerJvmMetrics() {
- // add JVM gauges
- metrics.register("variable.jvm.free.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- return Runtime.getRuntime().freeMemory();
- }
- }
- );
- metrics.register("variable.jvm.max.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- return Runtime.getRuntime().maxMemory();
- }
- }
- );
- metrics.register("variable.jvm.total.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- return Runtime.getRuntime().totalMemory();
- }
- }
- );
- }
-
- private void registerClusterResourceMetrics() {
- metrics.register("variable.cluster.allocated.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- if( getRootQueueMetrics() == null) {
- return 0L;
- } else {
- return getRootQueueMetrics().getAllocatedMB();
- }
- }
- }
- );
- metrics.register("variable.cluster.allocated.vcores",
- new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- if(getRootQueueMetrics() == null) {
- return 0;
- } else {
- return getRootQueueMetrics().getAllocatedVirtualCores();
- }
- }
- }
- );
- metrics.register("variable.cluster.available.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- if(getRootQueueMetrics() == null) {
- return 0L;
- } else {
- return getRootQueueMetrics().getAvailableMB();
- }
- }
- }
- );
- metrics.register("variable.cluster.available.vcores",
- new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- if(getRootQueueMetrics() == null) {
- return 0;
- } else {
- return getRootQueueMetrics().getAvailableVirtualCores();
- }
- }
- }
- );
- metrics.register("variable.cluster.reserved.memory",
- new Gauge<Long>() {
- @Override
- public Long getValue() {
- if(getRootQueueMetrics() == null) {
- return 0L;
- } else {
- return getRootQueueMetrics().getReservedMB();
- }
- }
- }
- );
- metrics.register("variable.cluster.reserved.vcores",
- new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- if(getRootQueueMetrics() == null) {
- return 0;
- } else {
- return getRootQueueMetrics().getReservedVirtualCores();
- }
- }
- }
- );
- }
-
- private void registerContainerAppNumMetrics() {
- metrics.register("variable.running.application",
- new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- if(getRootQueueMetrics() == null) {
- return 0;
- } else {
- return getRootQueueMetrics().getAppsRunning();
- }
- }
- }
- );
- metrics.register("variable.running.container",
- new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- if(getRootQueueMetrics() == null) {
- return 0;
- } else {
- return getRootQueueMetrics().getAllocatedContainers();
- }
- }
- }
- );
- }
-
- private void registerSchedulerMetrics() {
- samplerLock.lock();
- try {
- // counters for scheduler operations
- schedulerAllocateCounter = metrics.counter(
- "counter.scheduler.operation.allocate");
- schedulerHandleCounter = metrics.counter(
- "counter.scheduler.operation.handle");
- schedulerHandleCounterMap = new HashMap<SchedulerEventType, Counter>();
- for (SchedulerEventType e : SchedulerEventType.values()) {
- Counter counter = metrics.counter(
- "counter.scheduler.operation.handle." + e);
- schedulerHandleCounterMap.put(e, counter);
- }
- // timers for scheduler operations
- int timeWindowSize = conf.getInt(
- SLSConfiguration.METRICS_TIMER_WINDOW_SIZE,
- SLSConfiguration.METRICS_TIMER_WINDOW_SIZE_DEFAULT);
- schedulerAllocateTimer = new Timer(
- new SlidingWindowReservoir(timeWindowSize));
- schedulerHandleTimer = new Timer(
- new SlidingWindowReservoir(timeWindowSize));
- schedulerHandleTimerMap = new HashMap<SchedulerEventType, Timer>();
- for (SchedulerEventType e : SchedulerEventType.values()) {
- Timer timer = new Timer(new SlidingWindowReservoir(timeWindowSize));
- schedulerHandleTimerMap.put(e, timer);
- }
- // histogram for scheduler operations (Samplers)
- schedulerHistogramList = new ArrayList<Histogram>();
- histogramTimerMap = new HashMap<Histogram, Timer>();
- Histogram schedulerAllocateHistogram = new Histogram(
- new SlidingWindowReservoir(SAMPLING_SIZE));
- metrics.register("sampler.scheduler.operation.allocate.timecost",
- schedulerAllocateHistogram);
- schedulerHistogramList.add(schedulerAllocateHistogram);
- histogramTimerMap.put(schedulerAllocateHistogram, schedulerAllocateTimer);
- Histogram schedulerHandleHistogram = new Histogram(
- new SlidingWindowReservoir(SAMPLING_SIZE));
- metrics.register("sampler.scheduler.operation.handle.timecost",
- schedulerHandleHistogram);
- schedulerHistogramList.add(schedulerHandleHistogram);
- histogramTimerMap.put(schedulerHandleHistogram, schedulerHandleTimer);
- for (SchedulerEventType e : SchedulerEventType.values()) {
- Histogram histogram = new Histogram(
- new SlidingWindowReservoir(SAMPLING_SIZE));
- metrics.register(
- "sampler.scheduler.operation.handle." + e + ".timecost",
- histogram);
- schedulerHistogramList.add(histogram);
- histogramTimerMap.put(histogram, schedulerHandleTimerMap.get(e));
- }
- } finally {
- samplerLock.unlock();
- }
- }
-
- private void initMetricsCSVOutput() {
- int timeIntervalMS = conf.getInt(
- SLSConfiguration.METRICS_RECORD_INTERVAL_MS,
- SLSConfiguration.METRICS_RECORD_INTERVAL_MS_DEFAULT);
- File dir = new File(metricsOutputDir + "/metrics");
- if(! dir.exists()
- && ! dir.mkdirs()) {
- LOG.error("Cannot create directory " + dir.getAbsoluteFile());
- }
- final CsvReporter reporter = CsvReporter.forRegistry(metrics)
- .formatFor(Locale.US)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build(new File(metricsOutputDir + "/metrics"));
- reporter.start(timeIntervalMS, TimeUnit.MILLISECONDS);
- }
-
- class HistogramsRunnable implements Runnable {
- @Override
- public void run() {
- samplerLock.lock();
- try {
- for (Histogram histogram : schedulerHistogramList) {
- Timer timer = histogramTimerMap.get(histogram);
- histogram.update((int) timer.getSnapshot().getMean());
- }
- } finally {
- samplerLock.unlock();
- }
- }
- }
-
- class MetricsLogRunnable implements Runnable {
- private boolean firstLine = true;
- public MetricsLogRunnable() {
- try {
- metricsLogBW =
- new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
- metricsOutputDir + "/realtimetrack.json"), "UTF-8"));
- metricsLogBW.write("[");
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public void run() {
- if(running) {
- // all WebApp to get real tracking json
- String metrics = web.generateRealTimeTrackingMetrics();
- // output
- try {
- if(firstLine) {
- metricsLogBW.write(metrics + EOL);
- firstLine = false;
- } else {
- metricsLogBW.write("," + metrics + EOL);
- }
- metricsLogBW.flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- // the following functions are used by AMSimulator
- public void addAMRuntime(ApplicationId appId,
- long traceStartTimeMS, long traceEndTimeMS,
- long simulateStartTimeMS, long simulateEndTimeMS) {
-
- if (metricsON) {
- try {
- // write job runtime information
- StringBuilder sb = new StringBuilder();
- sb.append(appId).append(",").append(traceStartTimeMS).append(",")
- .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
- .append(",").append(simulateEndTimeMS);
- jobRuntimeLogBW.write(sb.toString() + EOL);
- jobRuntimeLogBW.flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
-
- private void updateQueueMetrics(String queue,
- long releasedMemory, int releasedVCores) {
- // update queue counters
- SortedMap<String, Counter> counterMap = metrics.getCounters();
- if (releasedMemory != 0) {
- String name = "counter.queue." + queue + ".allocated.memory";
- if (! counterMap.containsKey(name)) {
- metrics.counter(name);
- counterMap = metrics.getCounters();
- }
- counterMap.get(name).inc(-releasedMemory);
- }
- if (releasedVCores != 0) {
- String name = "counter.queue." + queue + ".allocated.cores";
- if (! counterMap.containsKey(name)) {
- metrics.counter(name);
- counterMap = metrics.getCounters();
- }
- counterMap.get(name).inc(-releasedVCores);
- }
+ schedulerMetrics.updateQueueMetrics(pendingResource, allocatedResource,
+ queueName);
}
private void initQueueMetrics(CSQueue queue) {
if (queue instanceof LeafQueue) {
- SortedMap<String, Counter> counterMap = metrics.getCounters();
- String queueName = queue.getQueueName();
- String[] names = new String[]{
- QUEUE_COUNTER_PREFIX + queueName + ".pending.memory",
- QUEUE_COUNTER_PREFIX + queueName + ".pending.cores",
- QUEUE_COUNTER_PREFIX + queueName + ".allocated.memory",
- QUEUE_COUNTER_PREFIX + queueName + ".allocated.cores" };
-
- for (int i = names.length - 1; i >= 0; i--) {
- if (!counterMap.containsKey(names[i])) {
- metrics.counter(names[i]);
- counterMap = metrics.getCounters();
- }
- }
-
- queueLock.lock();
- try {
- if (!schedulerMetrics.isTracked(queueName)) {
- schedulerMetrics.trackQueue(queueName);
- }
- } finally {
- queueLock.unlock();
- }
-
+ schedulerMetrics.initQueueMetric(queue.getQueueName());
return;
}
@@ -811,54 +349,17 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
public void serviceInit(Configuration configuration) throws Exception {
super.serviceInit(configuration);
- initQueueMetrics(getRootQueue());
- }
-
- public void setQueueSet(Set<String> queues) {
- this.queueSet = queues;
- }
-
- public Set<String> getQueueSet() {
- return this.queueSet;
- }
-
- public void setTrackedAppSet(Set<String> apps) {
- this.trackedAppSet = apps;
- }
-
- public Set<String> getTrackedAppSet() {
- return this.trackedAppSet;
- }
-
- public MetricRegistry getMetrics() {
- return metrics;
- }
-
- public SchedulerMetrics getSchedulerMetrics() {
- return schedulerMetrics;
- }
-
- // API open to out classes
- public void addTrackedApp(ApplicationId appId,
- String oldAppId) {
if (metricsON) {
- schedulerMetrics.trackApp(appId, oldAppId);
+ initQueueMetrics(getRootQueue());
}
}
- public void removeTrackedApp(String oldAppId) {
- if (metricsON) {
- schedulerMetrics.untrackApp(oldAppId);
- }
+ public SchedulerMetrics getSchedulerMetrics() {
+ return schedulerMetrics;
}
@Override
public Configuration getConf() {
return conf;
}
-
-
-
-
-}
-
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
new file mode 100644
index 0000000..572dacf
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -0,0 +1,339 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.sls.scheduler;
+
+import com.codahale.metrics.Timer;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.sls.SLSRunner;
+import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
+import org.apache.hadoop.yarn.sls.utils.SLSUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+@Private
+@Unstable
+public class SLSFairScheduler extends FairScheduler
+ implements SchedulerWrapper, Configurable {
+ private SchedulerMetrics schedulerMetrics;
+ private boolean metricsON;
+ private Tracker tracker;
+
+ private Map<ContainerId, Resource> preemptionContainerMap =
+ new ConcurrentHashMap<>();
+
+ public SchedulerMetrics getSchedulerMetrics() {
+ return schedulerMetrics;
+ }
+
+ public Tracker getTracker() {
+ return tracker;
+ }
+
+ public SLSFairScheduler() {
+ tracker = new Tracker();
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConfig(conf);
+
+ metricsON = conf.getBoolean(SLSConfiguration.METRICS_SWITCH, true);
+ if (metricsON) {
+ try {
+ schedulerMetrics = SchedulerMetrics.getInstance(conf,
+ FairScheduler.class);
+ schedulerMetrics.init(this, conf);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ ShutdownHookManager.get().addShutdownHook(new Runnable() {
+ @Override public void run() {
+ try {
+ schedulerMetrics.tearDown();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }, SLSUtils.SHUTDOWN_HOOK_PRIORITY);
+ }
+ }
+
+ @Override
+ public Allocation allocate(ApplicationAttemptId attemptId,
+ List<ResourceRequest> resourceRequests, List<ContainerId> containerIds,
+ List<String> blacklistAdditions, List<String> blacklistRemovals,
+ ContainerUpdates updateRequests) {
+ if (metricsON) {
+ final Timer.Context context = schedulerMetrics.getSchedulerAllocateTimer()
+ .time();
+ Allocation allocation = null;
+ try {
+ allocation = super.allocate(attemptId, resourceRequests, containerIds,
+ blacklistAdditions, blacklistRemovals, updateRequests);
+ return allocation;
+ } finally {
+ context.stop();
+ schedulerMetrics.increaseSchedulerAllocationCounter();
+ try {
+ updateQueueWithAllocateRequest(allocation, attemptId,
+ resourceRequests, containerIds);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ } else {
+ return super.allocate(attemptId, resourceRequests, containerIds,
+ blacklistAdditions, blacklistRemovals, updateRequests);
+ }
+ }
+
+ @Override
+ public void handle(SchedulerEvent schedulerEvent) {
+ // metrics off
+ if (!metricsON) {
+ super.handle(schedulerEvent);
+ return;
+ }
+
+ // metrics on
+ if(!schedulerMetrics.isRunning()) {
+ schedulerMetrics.setRunning(true);
+ }
+
+ Timer.Context handlerTimer = null;
+ Timer.Context operationTimer = null;
+
+ NodeUpdateSchedulerEventWrapper eventWrapper;
+ try {
+ if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE
+ && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
+ eventWrapper = new NodeUpdateSchedulerEventWrapper(
+ (NodeUpdateSchedulerEvent)schedulerEvent);
+ schedulerEvent = eventWrapper;
+ updateQueueWithNodeUpdate(eventWrapper);
+ } else if (
+ schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
+ && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
+ // check if having AM Container, update resource usage information
+ AppAttemptRemovedSchedulerEvent appRemoveEvent =
+ (AppAttemptRemovedSchedulerEvent) schedulerEvent;
+ ApplicationAttemptId appAttemptId =
+ appRemoveEvent.getApplicationAttemptID();
+ String queueName = getSchedulerApp(appAttemptId).getQueue().getName();
+ SchedulerAppReport app = getSchedulerAppInfo(appAttemptId);
+ if (!app.getLiveContainers().isEmpty()) { // have 0 or 1
+ // should have one container which is AM container
+ RMContainer rmc = app.getLiveContainers().iterator().next();
+ schedulerMetrics.updateQueueMetricsByRelease(
+ rmc.getContainer().getResource(), queueName);
+ }
+ }
+
+ handlerTimer = schedulerMetrics.getSchedulerHandleTimer().time();
+ operationTimer = schedulerMetrics.getSchedulerHandleTimer(
+ schedulerEvent.getType()).time();
+
+ super.handle(schedulerEvent);
+ } finally {
+ if (handlerTimer != null) {
+ handlerTimer.stop();
+ }
+ if (operationTimer != null) {
+ operationTimer.stop();
+ }
+ schedulerMetrics.increaseSchedulerHandleCounter(schedulerEvent.getType());
+
+ if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
+ && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
+ SLSRunner.decreaseRemainingApps();
+ }
+ }
+ }
+
+ private void updateQueueWithNodeUpdate(
+ NodeUpdateSchedulerEventWrapper eventWrapper) {
+ RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
+ List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
+ for (UpdatedContainerInfo info : containerList) {
+ for (ContainerStatus status : info.getCompletedContainers()) {
+ ContainerId containerId = status.getContainerId();
+ SchedulerAppReport app = super.getSchedulerAppInfo(
+ containerId.getApplicationAttemptId());
+
+ if (app == null) {
+ // this happens for the AM container
+ // The app have already removed when the NM sends the release
+ // information.
+ continue;
+ }
+
+ int releasedMemory = 0, releasedVCores = 0;
+ if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
+ for (RMContainer rmc : app.getLiveContainers()) {
+ if (rmc.getContainerId() == containerId) {
+ Resource resource = rmc.getContainer().getResource();
+ releasedMemory += resource.getMemorySize();
+ releasedVCores += resource.getVirtualCores();
+ break;
+ }
+ }
+ } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
+ if (preemptionContainerMap.containsKey(containerId)) {
+ Resource preResource = preemptionContainerMap.get(containerId);
+ releasedMemory += preResource.getMemorySize();
+ releasedVCores += preResource.getVirtualCores();
+ preemptionContainerMap.remove(containerId);
+ }
+ }
+ // update queue counters
+ String queue = getSchedulerApp(containerId.getApplicationAttemptId()).
+ getQueueName();
+ schedulerMetrics.updateQueueMetricsByRelease(
+ Resource.newInstance(releasedMemory, releasedVCores), queue);
+ }
+ }
+ }
+
+ private void updateQueueWithAllocateRequest(Allocation allocation,
+ ApplicationAttemptId attemptId,
+ List<ResourceRequest> resourceRequests,
+ List<ContainerId> containerIds) throws IOException {
+ // update queue information
+ Resource pendingResource = Resources.createResource(0, 0);
+ Resource allocatedResource = Resources.createResource(0, 0);
+ // container requested
+ for (ResourceRequest request : resourceRequests) {
+ if (request.getResourceName().equals(ResourceRequest.ANY)) {
+ Resources.addTo(pendingResource,
+ Resources.multiply(request.getCapability(),
+ request.getNumContainers()));
+ }
+ }
+ // container allocated
+ for (Container container : allocation.getContainers()) {
+ Resources.addTo(allocatedResource, container.getResource());
+ Resources.subtractFrom(pendingResource, container.getResource());
+ }
+ // container released from AM
+ SchedulerAppReport report = super.getSchedulerAppInfo(attemptId);
+ for (ContainerId containerId : containerIds) {
+ Container container = null;
+ for (RMContainer c : report.getLiveContainers()) {
+ if (c.getContainerId().equals(containerId)) {
+ container = c.getContainer();
+ break;
+ }
+ }
+ if (container != null) {
+ // released allocated containers
+ Resources.subtractFrom(allocatedResource, container.getResource());
+ } else {
+ for (RMContainer c : report.getReservedContainers()) {
+ if (c.getContainerId().equals(containerId)) {
+ container = c.getContainer();
+ break;
+ }
+ }
+ if (container != null) {
+ // released reserved containers
+ Resources.subtractFrom(pendingResource, container.getResource());
+ }
+ }
+ }
+ // containers released/preemption from scheduler
+ Set<ContainerId> preemptionContainers = new HashSet<ContainerId>();
+ if (allocation.getContainerPreemptions() != null) {
+ preemptionContainers.addAll(allocation.getContainerPreemptions());
+ }
+ if (allocation.getStrictContainerPreemptions() != null) {
+ preemptionContainers.addAll(allocation.getStrictContainerPreemptions());
+ }
+ if (!preemptionContainers.isEmpty()) {
+ for (ContainerId containerId : preemptionContainers) {
+ if (!preemptionContainerMap.containsKey(containerId)) {
+ Container container = null;
+ for (RMContainer c : report.getLiveContainers()) {
+ if (c.getContainerId().equals(containerId)) {
+ container = c.getContainer();
+ break;
+ }
+ }
+ if (container != null) {
+ preemptionContainerMap.put(containerId, container.getResource());
+ }
+ }
+
+ }
+ }
+
+ // update metrics
+ String queueName = getSchedulerApp(attemptId).getQueueName();
+ schedulerMetrics.updateQueueMetrics(pendingResource, allocatedResource,
+ queueName);
+ }
+
+ private void initQueueMetrics(FSQueue queue) {
+ if (queue instanceof FSLeafQueue) {
+ schedulerMetrics.initQueueMetric(queue.getQueueName());
+ return;
+ }
+
+ for (FSQueue child : queue.getChildQueues()) {
+ initQueueMetrics(child);
+ }
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ if (metricsON) {
+ initQueueMetrics(getQueueManager().getRootQueue());
+ }
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: MAPREDUCE-6850. Shuffle Handler
keep-alive connections are closed from the server side. Contributed by
Jonathan Eagles
Posted by ae...@apache.org.
MAPREDUCE-6850. Shuffle Handler keep-alive connections are closed from the server side. Contributed by Jonathan Eagles
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8bd5fc7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8bd5fc7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8bd5fc7
Branch: refs/heads/HDFS-7240
Commit: c8bd5fc7a86f9890ceaa37a89491ab650e7e9a64
Parents: 0d053ee
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Thu Mar 30 10:57:19 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Thu Mar 30 10:57:19 2017 -0500
----------------------------------------------------------------------
.../apache/hadoop/mapred/ShuffleHandler.java | 68 ++++++++++++++++++--
.../hadoop/mapred/TestShuffleHandler.java | 31 +++++++++
2 files changed, 92 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bd5fc7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 15a1b89..863da7e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -104,6 +104,7 @@ import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFactory;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.ChannelHandler;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
@@ -126,7 +127,13 @@ import org.jboss.netty.handler.codec.http.HttpResponseStatus;
import org.jboss.netty.handler.codec.http.QueryStringDecoder;
import org.jboss.netty.handler.ssl.SslHandler;
import org.jboss.netty.handler.stream.ChunkedWriteHandler;
+import org.jboss.netty.handler.timeout.IdleState;
+import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler;
+import org.jboss.netty.handler.timeout.IdleStateEvent;
+import org.jboss.netty.handler.timeout.IdleStateHandler;
import org.jboss.netty.util.CharsetUtil;
+import org.jboss.netty.util.HashedWheelTimer;
+import org.jboss.netty.util.Timer;
import org.eclipse.jetty.http.HttpHeader;
import com.google.common.annotations.VisibleForTesting;
@@ -240,6 +247,7 @@ public class ShuffleHandler extends AuxiliaryService {
public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true;
public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED =
false;
+ private static final String TIMEOUT_HANDLER = "timeout";
/* the maximum number of files a single GET request can
open simultaneously during shuffle
@@ -249,8 +257,9 @@ public class ShuffleHandler extends AuxiliaryService {
public static final int DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES = 3;
boolean connectionKeepAliveEnabled = false;
- int connectionKeepAliveTimeOut;
- int mapOutputMetaInfoCacheSize;
+ private int connectionKeepAliveTimeOut;
+ private int mapOutputMetaInfoCacheSize;
+ private Timer timer;
@Metrics(about="Shuffle output metrics", context="mapred")
static class ShuffleMetrics implements ChannelFutureListener {
@@ -293,7 +302,15 @@ public class ShuffleHandler extends AuxiliaryService {
int waitCount = this.reduceContext.getMapsToWait().decrementAndGet();
if (waitCount == 0) {
metrics.operationComplete(future);
- future.getChannel().close();
+ // Let the idle timer handler close keep-alive connections
+ if (reduceContext.getKeepAlive()) {
+ ChannelPipeline pipeline = future.getChannel().getPipeline();
+ TimeoutHandler timeoutHandler =
+ (TimeoutHandler)pipeline.get(TIMEOUT_HANDLER);
+ timeoutHandler.setEnabledTimeout(true);
+ } else {
+ future.getChannel().close();
+ }
} else {
pipelineFact.getSHUFFLE().sendMap(reduceContext);
}
@@ -314,11 +331,12 @@ public class ShuffleHandler extends AuxiliaryService {
private String user;
private Map<String, Shuffle.MapOutputInfo> infoMap;
private String jobId;
+ private final boolean keepAlive;
public ReduceContext(List<String> mapIds, int rId,
ChannelHandlerContext context, String usr,
Map<String, Shuffle.MapOutputInfo> mapOutputInfoMap,
- String jobId) {
+ String jobId, boolean keepAlive) {
this.mapIds = mapIds;
this.reduceId = rId;
@@ -339,6 +357,7 @@ public class ShuffleHandler extends AuxiliaryService {
this.user = usr;
this.infoMap = mapOutputInfoMap;
this.jobId = jobId;
+ this.keepAlive = keepAlive;
}
public int getReduceId() {
@@ -372,6 +391,10 @@ public class ShuffleHandler extends AuxiliaryService {
public AtomicInteger getMapsToWait() {
return mapsToWait;
}
+
+ public boolean getKeepAlive() {
+ return keepAlive;
+ }
}
ShuffleHandler(MetricsSystem ms) {
@@ -508,8 +531,10 @@ public class ShuffleHandler extends AuxiliaryService {
secretManager = new JobTokenSecretManager();
recoverState(conf);
ServerBootstrap bootstrap = new ServerBootstrap(selector);
+ // Timer is shared across entire factory and must be released separately
+ timer = new HashedWheelTimer();
try {
- pipelineFact = new HttpPipelineFactory(conf);
+ pipelineFact = new HttpPipelineFactory(conf, timer);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
@@ -549,6 +574,10 @@ public class ShuffleHandler extends AuxiliaryService {
if (pipelineFact != null) {
pipelineFact.destroy();
}
+ if (timer != null) {
+ // Release this shared timer resource
+ timer.stop();
+ }
if (stateDb != null) {
stateDb.close();
}
@@ -755,12 +784,29 @@ public class ShuffleHandler extends AuxiliaryService {
}
}
+ static class TimeoutHandler extends IdleStateAwareChannelHandler {
+
+ private boolean enabledTimeout;
+
+ void setEnabledTimeout(boolean enabledTimeout) {
+ this.enabledTimeout = enabledTimeout;
+ }
+
+ @Override
+ public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) {
+ if (e.getState() == IdleState.WRITER_IDLE && enabledTimeout) {
+ e.getChannel().close();
+ }
+ }
+ }
+
class HttpPipelineFactory implements ChannelPipelineFactory {
final Shuffle SHUFFLE;
private SSLFactory sslFactory;
+ private final ChannelHandler idleStateHandler;
- public HttpPipelineFactory(Configuration conf) throws Exception {
+ public HttpPipelineFactory(Configuration conf, Timer timer) throws Exception {
SHUFFLE = getShuffle(conf);
if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY,
MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) {
@@ -768,6 +814,7 @@ public class ShuffleHandler extends AuxiliaryService {
sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf);
sslFactory.init();
}
+ this.idleStateHandler = new IdleStateHandler(timer, 0, connectionKeepAliveTimeOut, 0);
}
public Shuffle getSHUFFLE() {
@@ -791,6 +838,8 @@ public class ShuffleHandler extends AuxiliaryService {
pipeline.addLast("encoder", new HttpResponseEncoder());
pipeline.addLast("chunking", new ChunkedWriteHandler());
pipeline.addLast("shuffle", SHUFFLE);
+ pipeline.addLast("idle", idleStateHandler);
+ pipeline.addLast(TIMEOUT_HANDLER, new TimeoutHandler());
return pipeline;
// TODO factor security manager into pipeline
// TODO factor out encode/decode to permit binary shuffle
@@ -981,6 +1030,10 @@ public class ShuffleHandler extends AuxiliaryService {
Map<String, MapOutputInfo> mapOutputInfoMap =
new HashMap<String, MapOutputInfo>();
Channel ch = evt.getChannel();
+ ChannelPipeline pipeline = ch.getPipeline();
+ TimeoutHandler timeoutHandler =
+ (TimeoutHandler)pipeline.get(TIMEOUT_HANDLER);
+ timeoutHandler.setEnabledTimeout(false);
String user = userRsrc.get(jobId);
try {
@@ -995,8 +1048,9 @@ public class ShuffleHandler extends AuxiliaryService {
}
ch.write(response);
//Initialize one ReduceContext object per messageReceived call
+ boolean keepAlive = keepAliveParam || connectionKeepAliveEnabled;
ReduceContext reduceContext = new ReduceContext(mapIds, reduceId, ctx,
- user, mapOutputInfoMap, jobId);
+ user, mapOutputInfoMap, jobId, keepAlive);
for (int i = 0; i < Math.min(maxSessionOpenFiles, mapIds.size()); i++) {
ChannelFuture nextMap = sendMap(reduceContext);
if(nextMap == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bd5fc7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 1e43937..7fb2051 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -37,6 +37,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.net.SocketAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -79,6 +80,7 @@ import org.apache.hadoop.yarn.server.records.Version;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.socket.SocketChannel;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.AbstractChannel;
@@ -309,6 +311,15 @@ public class TestShuffleHandler {
Assert.assertTrue("sendError called when client closed connection",
failures.size() == 0);
}
+ static class LastSocketAddress {
+ SocketAddress lastAddress;
+ void setAddress(SocketAddress lastAddress) {
+ this.lastAddress = lastAddress;
+ }
+ SocketAddress getSocketAddres() {
+ return lastAddress;
+ }
+ }
@Test(timeout = 10000)
public void testKeepAlive() throws Exception {
@@ -318,6 +329,8 @@ public class TestShuffleHandler {
conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true);
// try setting to -ve keep alive timeout.
conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT, -100);
+ final LastSocketAddress lastSocketAddress = new LastSocketAddress();
+
ShuffleHandler shuffleHandler = new ShuffleHandler() {
@Override
protected Shuffle getShuffle(final Configuration conf) {
@@ -363,6 +376,7 @@ public class TestShuffleHandler {
protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
Channel ch, String user, String mapId, int reduce,
MapOutputInfo info) throws IOException {
+ lastSocketAddress.setAddress(ch.getRemoteAddress());
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
// send a shuffle header and a lot of data down the channel
@@ -422,6 +436,9 @@ public class TestShuffleHandler {
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
ShuffleHeader header = new ShuffleHeader();
header.readFields(input);
+ byte[] buffer = new byte[1024];
+ while (input.read(buffer) != -1) {}
+ SocketAddress firstAddress = lastSocketAddress.getSocketAddres();
input.close();
// For keepAlive via URL
@@ -443,6 +460,14 @@ public class TestShuffleHandler {
header = new ShuffleHeader();
header.readFields(input);
input.close();
+ SocketAddress secondAddress = lastSocketAddress.getSocketAddres();
+ Assert.assertNotNull("Initial shuffle address should not be null",
+ firstAddress);
+ Assert.assertNotNull("Keep-Alive shuffle address should not be null",
+ secondAddress);
+ Assert.assertEquals("Initial shuffle address and keep-alive shuffle "
+ + "address should be the same", firstAddress, secondAddress);
+
}
@Test(timeout = 10000)
@@ -1058,14 +1083,20 @@ public class TestShuffleHandler {
Mockito.mock(ChannelHandlerContext.class);
final MessageEvent mockEvt = Mockito.mock(MessageEvent.class);
final Channel mockCh = Mockito.mock(AbstractChannel.class);
+ final ChannelPipeline mockPipeline = Mockito.mock(ChannelPipeline.class);
// Mock HttpRequest and ChannelFuture
final HttpRequest mockHttpRequest = createMockHttpRequest();
final ChannelFuture mockFuture = createMockChannelFuture(mockCh,
listenerList);
+ final ShuffleHandler.TimeoutHandler timerHandler =
+ new ShuffleHandler.TimeoutHandler();
// Mock Netty Channel Context and Channel behavior
Mockito.doReturn(mockCh).when(mockCtx).getChannel();
+ Mockito.when(mockCh.getPipeline()).thenReturn(mockPipeline);
+ Mockito.when(mockPipeline.get(
+ Mockito.any(String.class))).thenReturn(timerHandler);
Mockito.when(mockCtx.getChannel()).thenReturn(mockCh);
Mockito.doReturn(mockFuture).when(mockCh).write(Mockito.any(Object.class));
Mockito.when(mockCh.write(Object.class)).thenReturn(mockFuture);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDFS-10974. Document replication
factor for EC files. Contributed by Yiqun Lin.
Posted by ae...@apache.org.
HDFS-10974. Document replication factor for EC files. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c591b8d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c591b8d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c591b8d
Branch: refs/heads/HDFS-7240
Commit: 8c591b8d199325f49b5bba29240ca25610cf80a0
Parents: c9b7ce9
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Mar 30 11:16:05 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Mar 30 11:16:05 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/shell/SetReplication.java | 13 +++++++------
.../hadoop-common/src/site/markdown/FileSystemShell.md | 2 +-
.../hadoop-common/src/test/resources/testConf.xml | 2 +-
.../hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md | 1 +
.../hadoop-distcp/src/site/markdown/DistCp.md.vm | 2 +-
5 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c591b8d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
index fab0349..2231c58 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
@@ -41,12 +41,13 @@ class SetReplication extends FsCommand {
public static final String NAME = "setrep";
public static final String USAGE = "[-R] [-w] <rep> <path> ...";
public static final String DESCRIPTION =
- "Set the replication level of a file. If <path> is a directory " +
- "then the command recursively changes the replication factor of " +
- "all files under the directory tree rooted at <path>.\n" +
- "-w: It requests that the command waits for the replication " +
- "to complete. This can potentially take a very long time.\n" +
- "-R: It is accepted for backwards compatibility. It has no effect.";
+ "Set the replication level of a file. If <path> is a directory " +
+ "then the command recursively changes the replication factor of " +
+ "all files under the directory tree rooted at <path>. " +
+ "The EC files will be ignored here.\n" +
+ "-w: It requests that the command waits for the replication " +
+ "to complete. This can potentially take a very long time.\n" +
+ "-R: It is accepted for backwards compatibility. It has no effect.";
protected short newRep = 0;
protected List<PathData> waitList = new LinkedList<PathData>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c591b8d/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 42fddc9..5db96eb 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -647,7 +647,7 @@ setrep
Usage: `hadoop fs -setrep [-R] [-w] <numReplicas> <path> `
-Changes the replication factor of a file. If *path* is a directory then the command recursively changes the replication factor of all files under the directory tree rooted at *path*.
+Changes the replication factor of a file. If *path* is a directory then the command recursively changes the replication factor of all files under the directory tree rooted at *path*. The EC files will be ignored when executing this command.
Options:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c591b8d/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 112aea0..6347aa0 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -778,7 +778,7 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*rooted at <path>\.( )*</expected-output>
+ <expected-output>^\s*rooted at <path>\. The EC files will be ignored here\.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c591b8d/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 04acdce..f0c487d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -23,6 +23,7 @@ Purpose
However, for warm and cold datasets with relatively low I/O activities, additional block replicas are rarely accessed during normal operations, but still consume the same amount of resources as the first replica.
Therefore, a natural improvement is to use Erasure Coding (EC) in place of replication, which provides the same level of fault-tolerance with much less storage space. In typical Erasure Coding (EC) setups, the storage overhead is no more than 50%.
+ Replication factor of an EC file is meaningless. It is always 1 and cannot be changed via -setrep command.
Background
----------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c591b8d/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
index dbf0e8d..41a6e94 100644
--- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
+++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
@@ -217,7 +217,7 @@ Command Line Options
Flag | Description | Notes
----------------- | ------------------------------------ | --------
-`-p[rbugpcaxt]` | Preserve r: replication number b: block size u: user g: group p: permission c: checksum-type a: ACL x: XAttr t: timestamp | When `-update` is specified, status updates will **not** be synchronized unless the file sizes also differ (i.e. unless the file is re-created). If -pa is specified, DistCp preserves the permissions also because ACLs are a super-set of permissions.
+`-p[rbugpcaxt]` | Preserve r: replication number b: block size u: user g: group p: permission c: checksum-type a: ACL x: XAttr t: timestamp | When `-update` is specified, status updates will **not** be synchronized unless the file sizes also differ (i.e. unless the file is re-created). If -pa is specified, DistCp preserves the permissions also because ACLs are a super-set of permissions. The option -pr is only valid if both source and target directory are not erasure coded.
`-i` | Ignore failures | As explained in the Appendix, this option will keep more accurate statistics about the copy than the default case. It also preserves logs from failed copies, which can be valuable for debugging. Finally, a failing map will not cause the job to fail before all splits are attempted.
`-log <logdir>` | Write logs to \<logdir\> | DistCp keeps logs of each file it attempts to copy as map output. If a map fails, the log output will not be retained if it is re-executed.
`-m <num_maps>` | Maximum number of simultaneous copies | Specify the number of maps to copy data. Note that more maps may not necessarily improve throughput.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: HADOOP-14273. Azure:
NativeAzureFileSystem should respect config for kerberosSupportEnabled flag.
Contributed by Santhosh G Nayak
Posted by ae...@apache.org.
HADOOP-14273. Azure: NativeAzureFileSystem should respect config for kerberosSupportEnabled flag. Contributed by Santhosh G Nayak
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56ab02ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56ab02ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56ab02ee
Branch: refs/heads/HDFS-7240
Commit: 56ab02eed9b61e1c80605104dfc4c87fc6abac96
Parents: 27a44b6
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Apr 4 11:10:28 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Apr 4 11:10:28 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56ab02ee/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 1e6f00e..5469944 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -1287,6 +1287,8 @@ public class NativeAzureFileSystem extends FileSystem {
this.azureAuthorization = useSecureMode &&
conf.getBoolean(KEY_AZURE_AUTHORIZATION, DEFAULT_AZURE_AUTHORIZATION);
+ this.kerberosSupportEnabled =
+ conf.getBoolean(Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
if (this.azureAuthorization) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HADOOP-14272. Azure:
WasbRemoteCallHelper should use String equals for comparison. Contributed by
Santhosh G Nayak
Posted by ae...@apache.org.
HADOOP-14272. Azure: WasbRemoteCallHelper should use String equals for comparison. Contributed by Santhosh G Nayak
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27a44b60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27a44b60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27a44b60
Branch: refs/heads/HDFS-7240
Commit: 27a44b60c1a55cabbf1873bca62db4e321c901ca
Parents: 6eba792
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Apr 4 11:03:59 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Apr 4 11:03:59 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a44b60/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
index 09ea084..b43e5ae 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
@@ -88,7 +88,8 @@ class WasbRemoteCallHelper {
}
Header contentTypeHeader = response.getFirstHeader("Content-Type");
- if (contentTypeHeader == null || contentTypeHeader.getValue() != APPLICATION_JSON) {
+ if (contentTypeHeader == null
+ || !APPLICATION_JSON.equals(contentTypeHeader.getValue())) {
throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" +
"Content-Type mismatch: expected: " + APPLICATION_JSON +
", got " + ((contentTypeHeader!=null) ? contentTypeHeader.getValue() : "NULL")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDFS-11541. Call RawErasureEncoder
and RawErasureDecoder release() methods. Contributed by SammiChen.
Posted by ae...@apache.org.
HDFS-11541. Call RawErasureEncoder and RawErasureDecoder release() methods. Contributed by SammiChen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84d787b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84d787b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84d787b9
Branch: refs/heads/HDFS-7240
Commit: 84d787b9d51196010495d51dc5ebf66c01c340ab
Parents: 0e6f8e4
Author: Rakesh Radhakrishnan <ra...@apache.org>
Authored: Wed Mar 29 11:41:48 2017 +0530
Committer: Rakesh Radhakrishnan <ra...@apache.org>
Committed: Wed Mar 29 11:41:48 2017 +0530
----------------------------------------------------------------------
.../hadoop/hdfs/DFSStripedInputStream.java | 20 ++++----
.../hadoop/hdfs/DFSStripedOutputStream.java | 1 +
.../StripedBlockChecksumReconstructor.java | 48 +++++++++++---------
.../erasurecode/StripedBlockReconstructor.java | 1 +
.../erasurecode/StripedReconstructor.java | 6 +++
5 files changed, 46 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 922f74e..07e6609 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -177,14 +177,18 @@ public class DFSStripedInputStream extends DFSInputStream {
@Override
public synchronized void close() throws IOException {
- super.close();
- if (curStripeBuf != null) {
- BUFFER_POOL.putBuffer(curStripeBuf);
- curStripeBuf = null;
- }
- if (parityBuf != null) {
- BUFFER_POOL.putBuffer(parityBuf);
- parityBuf = null;
+ try {
+ super.close();
+ } finally {
+ if (curStripeBuf != null) {
+ BUFFER_POOL.putBuffer(curStripeBuf);
+ curStripeBuf = null;
+ }
+ if (parityBuf != null) {
+ BUFFER_POOL.putBuffer(parityBuf);
+ parityBuf = null;
+ }
+ decoder.release();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 52fc5eb..22b30e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -1033,6 +1033,7 @@ public class DFSStripedOutputStream extends DFSOutputStream {
setClosed();
// shutdown executor of flushAll tasks
flushAllExecutor.shutdownNow();
+ encoder.release();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
index 9555618..6917317 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
@@ -75,29 +75,33 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
public void reconstruct() throws IOException {
MessageDigest digester = MD5Hash.getDigester();
long maxTargetLength = getMaxTargetLength();
- while (requestedLen > 0 && getPositionInBlock() < maxTargetLength) {
- long remaining = maxTargetLength - getPositionInBlock();
- final int toReconstructLen = (int) Math
- .min(getStripedReader().getBufferSize(), remaining);
- // step1: read from minimum source DNs required for reconstruction.
- // The returned success list is the source DNs we do real read from
- getStripedReader().readMinimumSources(toReconstructLen);
-
- // step2: decode to reconstruct targets
- reconstructTargets(toReconstructLen);
-
- // step3: calculate checksum
- checksumDataLen += checksumWithTargetOutput(targetBuffer.array(),
- toReconstructLen, digester);
-
- updatePositionInBlock(toReconstructLen);
- requestedLen -= toReconstructLen;
- clearBuffers();
- }
+ try {
+ while (requestedLen > 0 && getPositionInBlock() < maxTargetLength) {
+ long remaining = maxTargetLength - getPositionInBlock();
+ final int toReconstructLen = (int) Math
+ .min(getStripedReader().getBufferSize(), remaining);
+ // step1: read from minimum source DNs required for reconstruction.
+ // The returned success list is the source DNs we do real read from
+ getStripedReader().readMinimumSources(toReconstructLen);
+
+ // step2: decode to reconstruct targets
+ reconstructTargets(toReconstructLen);
+
+ // step3: calculate checksum
+ checksumDataLen += checksumWithTargetOutput(targetBuffer.array(),
+ toReconstructLen, digester);
+
+ updatePositionInBlock(toReconstructLen);
+ requestedLen -= toReconstructLen;
+ clearBuffers();
+ }
- byte[] digest = digester.digest();
- md5 = new MD5Hash(digest);
- md5.write(checksumWriter);
+ byte[] digest = digester.digest();
+ md5 = new MD5Hash(digest);
+ md5.write(checksumWriter);
+ } finally {
+ cleanup();
+ }
}
private long checksumWithTargetOutput(byte[] outputData, int toReconstructLen,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index a1da536..1119bbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -74,6 +74,7 @@ class StripedBlockReconstructor extends StripedReconstructor
metrics.incrECReconstructionBytesWritten(getBytesWritten());
getStripedReader().close();
stripedWriter.close();
+ cleanup();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
index cd17864..b8433c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
@@ -253,6 +253,12 @@ abstract class StripedReconstructor {
return decoder;
}
+ void cleanup() {
+ if (decoder != null) {
+ decoder.release();
+ }
+ }
+
StripedReader getStripedReader() {
return stripedReader;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Posted by ae...@apache.org.
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adc65104
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adc65104
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adc65104
Branch: refs/heads/HDFS-7240
Commit: adc651044c0cacff62a3afa1ef1f833ab47e0413
Parents: 38dddad 56ab02e
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Apr 4 12:50:22 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Apr 4 12:50:22 2017 -0700
----------------------------------------------------------------------
LICENSE.txt | 22 +-
.../jdiff/Apache_Hadoop_Common_2.8.0.xml | 37921 +++++++++++++++++
hadoop-common-project/hadoop-common/pom.xml | 15 +-
.../src/main/conf/log4j.properties | 10 +
.../org/apache/hadoop/conf/Configuration.java | 395 +-
.../apache/hadoop/fs/AbstractFileSystem.java | 16 +-
.../java/org/apache/hadoop/fs/ChecksumFs.java | 13 +-
.../apache/hadoop/fs/DelegateToFileSystem.java | 6 +
.../hadoop/fs/FSDataOutputStreamBuilder.java | 142 +
.../java/org/apache/hadoop/fs/FileStatus.java | 23 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 9 +
.../org/apache/hadoop/fs/FilterFileSystem.java | 5 +
.../java/org/apache/hadoop/fs/FilterFs.java | 8 +-
.../org/apache/hadoop/fs/HarFileSystem.java | 5 +
.../java/org/apache/hadoop/fs/ftp/FtpFs.java | 7 +
.../org/apache/hadoop/fs/local/RawLocalFs.java | 8 +
.../hadoop/fs/permission/FsPermission.java | 7 +
.../hadoop/fs/permission/PermissionStatus.java | 13 -
.../apache/hadoop/fs/shell/SetReplication.java | 30 +-
.../apache/hadoop/fs/shell/XAttrCommands.java | 2 +-
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 23 +-
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 97 +-
.../apache/hadoop/ha/ZKFailoverController.java | 3 +-
.../hadoop/io/retry/RetryInvocationHandler.java | 17 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 4 +-
.../apache/hadoop/metrics2/AbstractMetric.java | 3 +-
.../org/apache/hadoop/metrics2/MetricsTag.java | 3 +-
.../metrics2/impl/AbstractMetricsRecord.java | 3 +-
.../org/apache/hadoop/metrics2/impl/MsInfo.java | 4 +-
.../hadoop/metrics2/lib/MetricsInfoImpl.java | 3 +-
.../hadoop/metrics2/lib/MetricsRegistry.java | 4 +-
.../hadoop/metrics2/source/JvmMetricsInfo.java | 6 +-
.../hadoop/metrics2/util/MetricsCache.java | 4 +-
.../org/apache/hadoop/net/InnerNodeImpl.java | 8 +-
.../org/apache/hadoop/net/NetworkTopology.java | 2 +-
.../net/NetworkTopologyWithNodeGroup.java | 2 +-
.../java/org/apache/hadoop/net/NodeBase.java | 2 +-
.../hadoop/net/unix/DomainSocketWatcher.java | 12 +-
.../alias/CredentialProviderFactory.java | 16 +-
.../java/org/apache/hadoop/util/ConfTest.java | 2 +-
.../org/apache/hadoop/util/Progressable.java | 2 +-
.../org/apache/hadoop/util/StringUtils.java | 8 +-
.../apache/hadoop/util/UTF8ByteArrayUtils.java | 4 +-
.../apache/hadoop/util/curator/ChildReaper.java | 4 +-
.../src/main/native/gtest/gtest-all.cc | 2 +-
.../src/main/native/gtest/include/gtest/gtest.h | 2 +-
.../src/main/resources/core-default.xml | 6 +
.../src/site/markdown/FileSystemShell.md | 2 +-
.../src/site/markdown/filesystem/filesystem.md | 14 +-
.../site/markdown/filesystem/introduction.md | 85 +
.../markdown/release/2.8.0/CHANGES.2.8.0.md | 2986 ++
.../release/2.8.0/RELEASENOTES.2.8.0.md | 1105 +
.../org/apache/hadoop/cli/CLITestHelper.java | 17 +-
.../apache/hadoop/cli/util/CommandExecutor.java | 4 +-
.../apache/hadoop/cli/util/FSCmdExecutor.java | 4 +-
.../apache/hadoop/conf/TestConfiguration.java | 62 +-
.../fs/FileContextMainOperationsBaseTest.java | 21 +-
.../hadoop/fs/FileSystemContractBaseTest.java | 246 +-
.../org/apache/hadoop/fs/TestAfsCheckPath.java | 1 +
.../org/apache/hadoop/fs/TestFileStatus.java | 10 +-
.../java/org/apache/hadoop/fs/TestFilterFs.java | 12 +
.../apache/hadoop/fs/TestLocalFileSystem.java | 54 +
.../fs/TestRawLocalFileSystemContract.java | 24 +-
.../hadoop/fs/contract/ContractTestUtils.java | 27 +
.../hadoop/fs/viewfs/TestViewfsFileStatus.java | 10 +-
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 141 +-
.../org/apache/hadoop/test/MetricsAsserts.java | 9 +-
.../hadoop/test/MultithreadedTestUtil.java | 2 +-
.../src/test/resources/testConf.xml | 2 +-
.../hadoop-kms/src/site/markdown/index.md.vm | 2 +-
.../apache/hadoop/oncrpc/SimpleTcpServer.java | 3 +-
.../java/org/apache/hadoop/portmap/Portmap.java | 1 +
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 -
.../main/java/org/apache/hadoop/fs/Hdfs.java | 6 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 10 +-
.../hadoop/hdfs/DFSInotifyEventInputStream.java | 4 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 2 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 22 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 1 +
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 2 +-
.../hadoop/hdfs/DistributedFileSystem.java | 81 +
.../apache/hadoop/hdfs/FileChecksumHelper.java | 42 +-
.../hdfs/client/HdfsClientConfigKeys.java | 2 +
.../hadoop/hdfs/protocol/DatanodeInfo.java | 1 +
.../hdfs/protocol/FsPermissionExtension.java | 17 +-
.../hdfs/server/protocol/SlowDiskReports.java | 146 +
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 13 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 7 +-
.../hadoop/fs/http/server/FSOperations.java | 39 +-
.../src/site/markdown/index.md | 8 +-
.../fs/http/client/BaseTestHttpFSWith.java | 24 +-
.../hadoop/fs/http/server/TestHttpFSServer.java | 18 +
.../org/apache/hadoop/test/TestHdfsHelper.java | 18 +-
.../dev-support/findbugsExcludeFile.xml | 12 +
.../jdiff/Apache_Hadoop_HDFS_2.8.0.xml | 312 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +
.../hadoop/hdfs/net/DFSNetworkTopology.java | 349 +
.../hadoop/hdfs/net/DFSTopologyNodeImpl.java | 302 +
.../DatanodeProtocolClientSideTranslatorPB.java | 7 +-
.../DatanodeProtocolServerSideTranslatorPB.java | 3 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 68 +
.../server/blockmanagement/BlockManager.java | 12 +-
.../blockmanagement/DatanodeDescriptor.java | 9 +
.../server/blockmanagement/DatanodeManager.java | 46 +-
.../server/blockmanagement/SlowDiskTracker.java | 294 +
.../server/blockmanagement/SlowPeerTracker.java | 2 +-
.../hdfs/server/datanode/BPServiceActor.java | 40 +-
.../server/datanode/BlockChecksumHelper.java | 6 +-
.../hdfs/server/datanode/BlockReceiver.java | 61 +-
.../server/datanode/BlockRecoveryWorker.java | 10 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 87 +-
.../hdfs/server/datanode/DataNodeMXBean.java | 7 +
.../hdfs/server/datanode/DataStorage.java | 4 +-
.../datanode/IncrementalBlockReportManager.java | 41 +-
.../server/datanode/checker/AbstractFuture.java | 1295 +
.../datanode/checker/DatasetVolumeChecker.java | 15 +-
.../checker/StorageLocationChecker.java | 1 +
.../datanode/checker/ThrottledAsyncChecker.java | 37 +-
.../server/datanode/checker/TimeoutFuture.java | 162 +
.../StripedBlockChecksumReconstructor.java | 48 +-
.../erasurecode/StripedBlockReconstructor.java | 1 +
.../erasurecode/StripedReconstructor.java | 6 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 2 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 14 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 2 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 13 +-
.../datanode/metrics/DataNodeDiskMetrics.java | 62 +-
.../datanode/metrics/DataNodeMetrics.java | 35 +
.../diskbalancer/DiskBalancerException.java | 1 +
.../server/diskbalancer/command/Command.java | 29 +-
.../diskbalancer/command/ReportCommand.java | 6 +-
.../hdfs/server/namenode/AclTransformation.java | 4 +-
.../namenode/ErasureCodingPolicyManager.java | 110 +-
.../server/namenode/FSDirErasureCodingOp.java | 54 +-
.../server/namenode/FSDirStatAndListingOp.java | 10 +-
.../server/namenode/FSImageFormatPBINode.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 15 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 12 +-
.../hadoop/hdfs/server/namenode/JournalSet.java | 2 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 6 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 6 +-
.../server/namenode/NameNodeStatusMXBean.java | 8 +
.../ha/ConfiguredFailoverProxyProvider.java | 15 +-
.../ha/RequestHedgingProxyProvider.java | 43 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 5 +
.../hdfs/server/protocol/DatanodeProtocol.java | 3 +-
.../apache/hadoop/hdfs/tools/AdminHelper.java | 6 +-
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 2 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 24 +-
.../offlineEditsViewer/OfflineEditsViewer.java | 19 +-
.../OfflineImageReconstructor.java | 10 +-
.../offlineImageViewer/PBImageXmlWriter.java | 21 +-
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 6 +
.../src/main/proto/DatanodeProtocol.proto | 15 +
.../src/main/resources/hdfs-default.xml | 10 +
.../src/site/markdown/HDFSCommands.md | 2 +-
.../src/site/markdown/HDFSDiskbalancer.md | 4 +-
.../src/site/markdown/HDFSErasureCoding.md | 53 +-
.../src/site/markdown/HdfsEditsViewer.md | 2 +
.../apache/hadoop/cli/TestErasureCodingCLI.java | 3 +
.../hadoop/cli/util/CacheAdminCmdExecutor.java | 4 +-
.../hadoop/cli/util/CryptoAdminCmdExecutor.java | 4 +-
.../cli/util/ErasureCodingCliCmdExecutor.java | 4 +-
.../apache/hadoop/hdfs/DFSNetworkTopology.java | 36 -
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 35 +-
.../apache/hadoop/hdfs/DFSTopologyNodeImpl.java | 253 -
.../hdfs/ErasureCodeBenchmarkThroughput.java | 5 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 12 +
.../hadoop/hdfs/TestDFSNetworkTopology.java | 260 -
.../TestDFSRSDefault10x4StripedInputStream.java | 2 +-
...TestDFSRSDefault10x4StripedOutputStream.java | 2 +-
...fault10x4StripedOutputStreamWithFailure.java | 2 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 5 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 4 +-
.../TestDFSStripedOutputStreamWithFailure.java | 4 +-
.../hdfs/TestDFSXORStripedInputStream.java | 2 +-
.../hdfs/TestDFSXORStripedOutputStream.java | 2 +-
...estDFSXORStripedOutputStreamWithFailure.java | 2 +-
.../hadoop/hdfs/TestDataTransferProtocol.java | 2 +-
.../apache/hadoop/hdfs/TestDecommission.java | 98 +
.../hdfs/TestDecommissionWithStriped.java | 7 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 57 +-
.../TestErasureCodeBenchmarkThroughput.java | 2 +
.../hadoop/hdfs/TestErasureCodingPolicies.java | 49 +-
.../TestErasureCodingPolicyWithSnapshot.java | 31 +-
.../apache/hadoop/hdfs/TestFileChecksum.java | 7 +-
.../org/apache/hadoop/hdfs/TestFileStatus.java | 18 +-
.../hadoop/hdfs/TestFileStatusWithECPolicy.java | 28 +-
.../hadoop/hdfs/TestLeaseRecoveryStriped.java | 5 +-
.../hadoop/hdfs/TestMaintenanceState.java | 51 +
.../hdfs/TestReadStripedFileWithDecoding.java | 7 +-
.../TestReadStripedFileWithMissingBlocks.java | 5 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 9 +-
.../org/apache/hadoop/hdfs/TestSafeMode.java | 24 +
.../hdfs/TestSafeModeWithStripedFile.java | 7 +-
.../hadoop/hdfs/TestSetrepIncreasing.java | 44 +
.../TestUnsetAndChangeDirectoryEcPolicy.java | 8 +-
.../hadoop/hdfs/TestWriteReadStripedFile.java | 7 +-
.../hdfs/TestWriteStripedFileWithFailure.java | 5 +-
.../hadoop/hdfs/net/TestDFSNetworkTopology.java | 568 +
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 39 +-
.../hdfs/qjournal/client/TestQJMWithFaults.java | 2 +-
.../client/TestQuorumJournalManager.java | 2 +-
.../hdfs/server/balancer/TestBalancer.java | 18 +-
.../blockmanagement/TestBlockInfoStriped.java | 4 +-
.../blockmanagement/TestBlockManager.java | 113 +-
.../TestBlockTokenWithDFSStriped.java | 10 +-
.../TestLowRedundancyBlockQueues.java | 4 +-
.../TestNameNodePrunesMissingStorages.java | 4 +-
...constructStripedBlocksWithRackAwareness.java | 16 +-
.../TestSequentialBlockGroupId.java | 8 +-
.../blockmanagement/TestSlowDiskTracker.java | 439 +
.../TestSortLocatedStripedBlock.java | 4 +-
.../datanode/InternalDataNodeTestUtils.java | 4 +-
.../server/datanode/TestBPOfferService.java | 4 +-
.../datanode/TestBlockCountersInPendingIBR.java | 146 +
.../hdfs/server/datanode/TestBlockRecovery.java | 7 +-
.../datanode/TestBpServiceActorScheduler.java | 20 +-
.../TestDataNodeErasureCodingMetrics.java | 7 +-
.../server/datanode/TestDataNodeLifeline.java | 8 +-
.../server/datanode/TestDataNodeMXBean.java | 33 +-
.../TestDataNodeVolumeFailureReporting.java | 116 +-
.../hdfs/server/datanode/TestDataStorage.java | 2 +-
.../TestDatanodeProtocolRetryPolicy.java | 4 +-
.../server/datanode/TestFsDatasetCache.java | 4 +-
.../hdfs/server/datanode/TestStorageReport.java | 4 +-
.../TestDatasetVolumeCheckerTimeout.java | 134 +
.../checker/TestThrottledAsyncChecker.java | 14 +-
.../TestThrottledAsyncCheckerTimeout.java | 223 +
.../command/TestDiskBalancerCommand.java | 71 +-
.../hadoop/hdfs/server/mover/TestMover.java | 7 +-
.../hdfs/server/namenode/FSAclBaseTest.java | 25 +-
.../server/namenode/NNThroughputBenchmark.java | 7 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 3 +-
.../TestAddOverReplicatedStripedBlocks.java | 8 +-
.../namenode/TestAddStripedBlockInFBR.java | 8 +-
.../server/namenode/TestAddStripedBlocks.java | 13 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 4 +-
.../server/namenode/TestEnabledECPolicies.java | 162 +
.../server/namenode/TestFSEditLogLoader.java | 7 +-
.../hdfs/server/namenode/TestFSImage.java | 10 +-
.../namenode/TestFavoredNodesEndToEnd.java | 23 +
.../hadoop/hdfs/server/namenode/TestFsck.java | 41 +-
.../server/namenode/TestNameNodeMXBean.java | 14 +-
.../namenode/TestNameNodeStatusMXBean.java | 148 +
.../namenode/TestQuotaWithStripedBlocks.java | 5 +-
.../namenode/TestReconstructStripedBlocks.java | 12 +-
.../server/namenode/TestStripedINodeFile.java | 9 +-
.../ha/TestRequestHedgingProxyProvider.java | 108 +-
.../namenode/ha/TestRetryCacheWithHA.java | 2 +-
.../snapshot/TestRenameWithSnapshots.java | 6 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 75 +
.../shortcircuit/TestShortCircuitLocalRead.java | 2 +-
.../TestOfflineEditsViewer.java | 18 +
.../TestOfflineImageViewer.java | 108 +-
...TestOfflineImageViewerWithStripedBlocks.java | 10 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 72 +
.../src/test/resources/testCryptoConf.xml | 31 +
.../test/resources/testErasureCodingConf.xml | 37 +-
.../Apache_Hadoop_MapReduce_Common_2.8.0.xml | 113 +
.../Apache_Hadoop_MapReduce_Core_2.8.0.xml | 27490 ++++++++++++
.../Apache_Hadoop_MapReduce_JobClient_2.8.0.xml | 16 +
.../v2/app/job/impl/TaskAttemptImpl.java | 285 +-
.../mapreduce/v2/app/webapp/ConfBlock.java | 2 +-
.../v2/app/rm/TestRMContainerAllocator.java | 453 +-
.../hadoop/filecache/DistributedCache.java | 8 +-
.../java/org/apache/hadoop/mapred/JobConf.java | 8 +-
.../mapred/lib/FieldSelectionMapReduce.java | 2 +-
.../mapred/lib/KeyFieldBasedComparator.java | 2 +-
.../hadoop/mapreduce/JobResourceUploader.java | 36 +-
.../apache/hadoop/mapreduce/JobSubmitter.java | 2 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 8 +-
.../mapreduce/filecache/DistributedCache.java | 8 +-
.../lib/fieldsel/FieldSelectionHelper.java | 7 +-
.../lib/fieldsel/FieldSelectionMapper.java | 2 +-
.../lib/fieldsel/FieldSelectionReducer.java | 2 +-
.../lib/input/KeyValueLineRecordReader.java | 13 +-
.../mapreduce/lib/jobcontrol/JobControl.java | 2 +-
.../mapreduce/lib/output/TextOutputFormat.java | 9 +-
.../lib/partition/KeyFieldBasedComparator.java | 4 +-
.../lib/partition/KeyFieldBasedPartitioner.java | 2 +-
.../hadoop/mapreduce/util/ConfigUtil.java | 8 +-
.../mapreduce/TestJobResourceUploader.java | 40 +-
.../org/apache/hadoop/mapred/YARNRunner.java | 4 +-
.../apache/hadoop/fs/slive/OperationOutput.java | 4 +-
.../hadoop/mapred/TestFieldSelection.java | 2 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 2 +-
.../mapred/lib/TestKeyFieldBasedComparator.java | 4 +-
.../lib/fieldsel/TestMRFieldSelection.java | 2 +-
.../TestMRKeyFieldBasedComparator.java | 4 +-
.../apache/hadoop/mapred/ShuffleHandler.java | 68 +-
.../hadoop/mapred/TestShuffleHandler.java | 31 +
hadoop-mapreduce-project/pom.xml | 1 +
hadoop-project-dist/pom.xml | 2 +-
hadoop-project/pom.xml | 22 +-
.../oss/TestAliyunOSSFileSystemContract.java | 35 +-
.../hadoop/fs/s3a/S3ABlockOutputStream.java | 5 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 10 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 34 +-
.../apache/hadoop/fs/s3a/S3ClientFactory.java | 7 +-
.../fs/s3a/SimpleAWSCredentialsProvider.java | 3 +-
.../fs/s3a/TemporaryAWSCredentialsProvider.java | 3 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 3 +
.../site/markdown/tools/hadoop-aws/testing.md | 1 +
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 44 +-
.../fs/s3a/ITestS3AFileSystemContract.java | 39 +-
.../fs/s3a/ITestS3ATemporaryCredentials.java | 2 +-
.../hadoop/fs/s3a/MockS3ClientFactory.java | 2 +-
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 16 +-
.../hadoop/fs/s3a/TestS3AGetFileStatus.java | 9 +
.../hadoop/fs/s3native/TestS3Credentials.java | 18 -
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 39 +-
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 4 +
.../META-INF/org.apache.hadoop.fs.FileSystem | 16 -
.../src/site/markdown/index.md | 87 +-
.../hadoop/fs/adl/TestAzureADTokenProvider.java | 5 +-
.../apache/hadoop/fs/adl/TestGetFileStatus.java | 13 +-
.../fs/adl/TestValidateConfiguration.java | 83 +-
.../adl/live/TestAdlFileSystemContractLive.java | 3 +-
.../fs/azure/AzureNativeFileSystemStore.java | 2 +-
.../hadoop/fs/azure/BlockBlobAppendStream.java | 2 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 69 +-
.../fs/azure/RemoteSASKeyGeneratorImpl.java | 191 +-
.../fs/azure/RemoteWasbAuthorizerImpl.java | 107 +-
.../fs/azure/SecureStorageInterfaceImpl.java | 11 +-
.../fs/azure/WasbAuthorizerInterface.java | 5 +-
.../hadoop/fs/azure/WasbRemoteCallHelper.java | 3 +-
.../hadoop/fs/azure/security/Constants.java | 54 +
.../hadoop/fs/azure/security/SecurityUtils.java | 86 +
.../security/WasbDelegationTokenIdentifier.java | 48 +
.../fs/azure/security/WasbTokenRenewer.java | 150 +
.../hadoop/fs/azure/security/package-info.java | 22 +
...apache.hadoop.security.token.TokenIdentifier | 16 +
...rg.apache.hadoop.security.token.TokenRenewer | 16 +
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 4 +-
.../org/apache/hadoop/tools/CopyListing.java | 73 +-
.../hadoop/tools/CopyListingFileStatus.java | 91 +-
.../java/org/apache/hadoop/tools/DistCp.java | 96 +-
.../org/apache/hadoop/tools/DistCpContext.java | 198 +
.../apache/hadoop/tools/DistCpOptionSwitch.java | 12 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 913 +-
.../org/apache/hadoop/tools/DistCpSync.java | 42 +-
.../hadoop/tools/FileBasedCopyListing.java | 12 +-
.../apache/hadoop/tools/GlobbedCopyListing.java | 17 +-
.../org/apache/hadoop/tools/OptionsParser.java | 271 +-
.../apache/hadoop/tools/SimpleCopyListing.java | 190 +-
.../hadoop/tools/mapred/CopyCommitter.java | 189 +-
.../apache/hadoop/tools/mapred/CopyMapper.java | 40 +-
.../tools/mapred/RetriableFileCopyCommand.java | 29 +-
.../tools/mapred/UniformSizeInputFormat.java | 5 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 126 +-
.../src/site/markdown/DistCp.md.vm | 3 +-
.../apache/hadoop/tools/TestCopyListing.java | 51 +-
.../hadoop/tools/TestCopyListingFileStatus.java | 1 +
.../apache/hadoop/tools/TestDistCpOptions.java | 500 +
.../org/apache/hadoop/tools/TestDistCpSync.java | 68 +-
.../hadoop/tools/TestDistCpSyncReverseBase.java | 44 +-
.../apache/hadoop/tools/TestDistCpSystem.java | 368 +-
.../apache/hadoop/tools/TestDistCpViewFs.java | 10 +-
.../hadoop/tools/TestFileBasedCopyListing.java | 9 +-
.../hadoop/tools/TestGlobbedCopyListing.java | 11 +-
.../apache/hadoop/tools/TestIntegration.java | 20 +-
.../apache/hadoop/tools/TestOptionsParser.java | 81 +-
.../contract/AbstractContractDistCpTest.java | 6 +-
.../hadoop/tools/mapred/TestCopyCommitter.java | 39 +-
.../mapred/TestUniformSizeInputFormat.java | 15 +-
.../mapred/lib/TestDynamicInputFormat.java | 17 +-
.../hadoop/tools/util/TestDistCpUtils.java | 123 +-
.../services/org.apache.hadoop.fs.FileSystem | 16 -
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 22 +-
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 29 +-
.../sls/scheduler/FairSchedulerMetrics.java | 308 +-
.../sls/scheduler/ResourceSchedulerWrapper.java | 972 -
.../sls/scheduler/SLSCapacityScheduler.java | 686 +-
.../yarn/sls/scheduler/SLSFairScheduler.java | 339 +
.../yarn/sls/scheduler/SchedulerMetrics.java | 602 +-
.../yarn/sls/scheduler/SchedulerWrapper.java | 23 +-
.../hadoop/yarn/sls/scheduler/Tracker.java | 46 +
.../apache/hadoop/yarn/sls/utils/SLSUtils.java | 1 +
.../apache/hadoop/yarn/sls/web/SLSWebApp.java | 12 +-
.../yarn/sls/appmaster/TestAMSimulator.java | 87 +-
.../yarn/sls/nodemanager/TestNMSimulator.java | 32 +-
.../hadoop/streaming/StreamKeyValUtil.java | 8 +-
.../jdiff/Apache_Hadoop_YARN_Client_2.8.0.xml | 2316 +
.../jdiff/Apache_Hadoop_YARN_Common_2.8.0.xml | 2665 ++
.../Apache_Hadoop_YARN_Server_Common_2.8.0.xml | 829 +
.../records/ApplicationSubmissionContext.java | 57 +-
.../org/apache/hadoop/yarn/conf/HAUtil.java | 30 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 28 +-
.../src/main/proto/yarn_protos.proto | 2 +-
.../distributedshell/TestDistributedShell.java | 30 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 178 +-
.../hadoop/yarn/client/TestRMFailover.java | 39 +-
.../yarn/client/api/impl/TestAMRMClient.java | 272 +-
.../api/impl/TestAMRMClientOnRMRestart.java | 59 +-
.../hadoop/yarn/client/cli/TestLogsCLI.java | 31 +
.../pb/ApplicationSubmissionContextPBImpl.java | 66 +-
.../yarn/api/records/impl/pb/ProtoUtils.java | 6 +-
.../yarn/client/api/impl/TimelineConnector.java | 4 +-
.../client/api/impl/TimelineV2ClientImpl.java | 9 +-
.../logaggregation/AggregatedLogFormat.java | 52 +-
.../yarn/logaggregation/LogCLIHelpers.java | 11 +-
.../nodelabels/CommonNodeLabelsManager.java | 22 +
.../hadoop/yarn/util/resource/Resources.java | 7 +
.../org/apache/hadoop/yarn/webapp/WebApp.java | 4 +-
.../src/main/resources/yarn-default.xml | 20 +
.../org/apache/hadoop/yarn/conf/TestHAUtil.java | 86 +-
.../yarn/util/resource/TestResources.java | 28 +-
.../client/impl/zk/RegistrySecurity.java | 12 +-
.../webapp/AHSWebServices.java | 32 +-
.../timeline/RollingLevelDBTimelineStore.java | 263 +-
.../webapp/TestAHSWebServices.java | 17 +
.../webapp/TestTimelineWebServices.java | 27 -
.../server/webapp/YarnWebServiceParams.java | 1 +
.../nodemanager/NodeStatusUpdaterImpl.java | 4 +-
.../containermanager/ContainerManagerImpl.java | 57 +-
.../application/ApplicationImpl.java | 4 +-
.../containermanager/container/Container.java | 2 +
.../container/ContainerImpl.java | 8 +
.../linux/runtime/docker/DockerPullCommand.java | 31 +
.../logaggregation/AppLogAggregatorImpl.java | 2 -
.../timelineservice/NMTimelinePublisher.java | 8 +
.../nodemanager/webapp/NMWebServices.java | 6 +-
.../nodemanager/TestNodeStatusUpdater.java | 2 +-
.../TestContainerManagerRecovery.java | 18 +-
.../launcher/TestContainerLaunch.java | 12 +-
.../runtime/docker/TestDockerPullCommand.java | 49 +
.../TestLocalCacheDirectoryManager.java | 6 +-
.../TestAppLogAggregatorImpl.java | 8 +-
.../TestNMTimelinePublisher.java | 10 +-
.../nodemanager/webapp/MockContainer.java | 5 +
.../nodemanager/webapp/TestNMWebServices.java | 6 +
.../server/resourcemanager/ClientRMService.java | 290 +-
.../server/resourcemanager/RMAppManager.java | 110 +-
.../server/resourcemanager/RMAuditLogger.java | 5 +
.../server/resourcemanager/RMServerUtils.java | 68 +-
.../server/resourcemanager/ResourceManager.java | 2 +-
.../capacity/IntraQueueCandidatesSelector.java | 5 +
.../recovery/LeveldbRMStateStore.java | 8 +
.../server/resourcemanager/rmapp/RMApp.java | 3 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 54 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 35 +-
.../scheduler/AbstractYarnScheduler.java | 5 +
.../scheduler/ClusterNodeTracker.java | 65 +-
.../resourcemanager/scheduler/QueueMetrics.java | 13 +
.../scheduler/ResourceScheduler.java | 9 +
.../scheduler/capacity/CapacityScheduler.java | 2 +-
.../capacity/CapacitySchedulerQueueManager.java | 7 +-
.../scheduler/capacity/ParentQueue.java | 6 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 10 +-
.../fair/AllocationFileLoaderService.java | 2 +-
.../scheduler/fair/FSAppAttempt.java | 54 +-
.../scheduler/fair/FSLeafQueue.java | 24 +-
.../scheduler/fair/FSParentQueue.java | 21 +
.../resourcemanager/scheduler/fair/FSQueue.java | 41 +-
.../scheduler/fair/FairScheduler.java | 28 +-
.../scheduler/fair/QueueManager.java | 2 +-
.../scheduler/policy/OrderingPolicy.java | 105 +-
.../resourcemanager/webapp/dao/AppInfo.java | 2 +-
.../server/resourcemanager/ACLsTestBase.java | 10 -
.../server/resourcemanager/Application.java | 2 +-
.../yarn/server/resourcemanager/MockRM.java | 45 +-
.../server/resourcemanager/RMHATestBase.java | 20 +-
.../ReservationACLsTestBase.java | 5 +-
.../server/resourcemanager/TestAppManager.java | 195 +-
.../resourcemanager/TestApplicationCleanup.java | 44 +-
.../TestApplicationMasterLauncher.java | 11 +-
.../TestApplicationMasterService.java | 19 +-
.../resourcemanager/TestClientRMService.java | 8 +-
.../TestNodeBlacklistingOnAMFailures.java | 213 +-
.../yarn/server/resourcemanager/TestRM.java | 10 +-
.../resourcemanager/TestRMAdminService.java | 95 +
.../resourcemanager/TestRMServerUtils.java | 297 +
.../TestReservationSystemWithRMHA.java | 5 +-
.../applicationsmanager/MockAsm.java | 6 +-
.../TestAMRMRPCNodeUpdates.java | 18 +-
.../TestRMAppLogAggregationStatus.java | 2 +
.../metrics/TestSystemMetricsPublisher.java | 3 +-
...ionalCapacityPreemptionPolicyIntraQueue.java | 55 +
.../recovery/TestLeveldbRMStateStore.java | 19 +-
.../resourcemanager/resource/TestResources.java | 4 +-
.../resourcetracker/TestNMReconnect.java | 14 +-
.../server/resourcemanager/rmapp/MockRMApp.java | 11 +-
.../rmapp/TestRMAppTransitions.java | 9 +-
.../attempt/TestRMAppAttemptTransitions.java | 19 +-
.../rmcontainer/TestRMContainerImpl.java | 1 -
.../capacity/TestApplicationLimits.java | 4 +-
.../TestApplicationLimitsByPartition.java | 4 +-
.../capacity/TestApplicationPriority.java | 29 +-
.../capacity/TestCapacityScheduler.java | 2 +-
.../scheduler/capacity/TestLeafQueue.java | 3 +-
.../scheduler/capacity/TestParentQueue.java | 18 +-
.../scheduler/fair/TestFairScheduler.java | 182 +-
.../fair/TestFairSchedulerPreemption.java | 12 +-
.../security/TestClientToAMTokens.java | 23 +-
.../webapp/TestRMWebServicesApps.java | 10 +-
.../pom.xml | 19 +
.../collector/TimelineCollector.java | 37 +-
.../collector/TimelineCollectorManager.java | 7 +-
.../collector/TimelineCollectorWebService.java | 12 +-
.../reader/TimelineReaderServer.java | 9 +
.../collector/TestTimelineCollector.java | 63 +
.../src/site/markdown/FairScheduler.md | 40 +-
.../src/site/markdown/TimelineServiceV2.md | 9 +
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 20 +-
.../app/components/base-chart-component.js | 4 +
.../main/webapp/app/components/nodes-heatmap.js | 106 +-
.../main/webapp/app/helpers/log-files-comma.js | 10 +-
.../app/serializers/yarn-node-container.js | 2 +-
.../main/webapp/app/serializers/yarn-node.js | 4 +-
.../main/webapp/app/serializers/yarn-rm-node.js | 4 +-
.../src/main/webapp/app/styles/app.css | 12 +
.../src/main/webapp/app/templates/yarn-node.hbs | 23 +-
.../main/webapp/app/templates/yarn-nodes.hbs | 2 +-
.../webapp/app/templates/yarn-nodes/table.hbs | 13 +-
.../hadoop-yarn-ui/src/main/webapp/package.json | 6 +-
516 files changed, 92672 insertions(+), 6565 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
----------------------------------------------------------------------
diff --cc hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 9984666,b4658ae..1289115
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@@ -322,26 -322,12 +322,36 @@@ log4j.appender.EWMA.cleanupInterval=${y
log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
+ # Fair scheduler requests log on state dump
+ log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER
+ log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=false
+ log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender
+ log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log
+ log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout
+ log4j.appender.FSLOGGER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+ log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize}
-log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex}
++log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex}
++
+#
+# Add a logger for ozone that is separate from the Datanode.
+#
+log4j.logger.org.apache.hadoop.ozone=DEBUG,OZONE,FILE
+
+# Do not log into datanode logs. Remove this line to have single log.
+log4j.additivity.org.apache.hadoop.ozone=false
+
+# For development purposes, log both to console and log file.
+log4j.appender.OZONE=org.apache.log4j.ConsoleAppender
+log4j.appender.OZONE.Threshold=info
+log4j.appender.OZONE.layout=org.apache.log4j.PatternLayout
+log4j.appender.OZONE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \
+ %X{component} %X{function} %X{resource} %X{user} %X{request} - %m%n
+
+# Real ozone logger that writes to ozone.log
+log4j.appender.FILE=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.FILE.File=${hadoop.log.dir}/ozone.log
+log4j.appender.FILE.Threshold=debug
+log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
+log4j.appender.FILE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \
+(%F:%L) %X{function} %X{resource} %X{user} %X{request} - \
+%m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
----------------------------------------------------------------------
diff --cc hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
index f7ab52e,bd48b15..4eda7f2
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
@@@ -91,7 -92,7 +92,7 @@@ public class SimpleTcpServer
LOG.info("Started listening to TCP requests at port " + boundPort + " for "
+ rpcProgram + " with workerCount " + workerCount);
}
--
++
// boundPort will be set only after server starts
public int getBoundPort() {
return this.boundPort;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-project/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
index 5f2265b,a2105c7..e22a3a2
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
@@@ -82,14 -103,24 +103,24 @@@ public class RemoteWasbAuthorizerImpl i
@Override
public void init(Configuration conf)
throws WasbAuthorizationException, IOException {
+ LOG.debug("Initializing RemoteWasbAuthorizerImpl instance");
+ Iterator<Token<? extends TokenIdentifier>> tokenIterator = null;
+ try {
- delegationToken = SecurityUtils.getDelegationTokenFromCredentials();
++ delegationToken = SecurityUtils.getDelegationTokenFromCredentials();
+ } catch (IOException e) {
+ final String msg = "Error in fetching the WASB delegation token";
+ LOG.error(msg, e);
+ throw new IOException(msg, e);
+ }
- remoteAuthorizerServiceUrl = conf.get(KEY_REMOTE_AUTH_SERVICE_URL);
+ remoteAuthorizerServiceUrl = SecurityUtils
+ .getRemoteAuthServiceUrls(conf);
if (remoteAuthorizerServiceUrl == null
-- || remoteAuthorizerServiceUrl.isEmpty()) {
++ || remoteAuthorizerServiceUrl.isEmpty()) {
throw new WasbAuthorizationException(
"fs.azure.authorization.remote.service.url config not set"
- + " in configuration.");
+ + " in configuration.");
}
this.remoteCallHelper = new WasbRemoteCallHelper();
@@@ -96,41 -130,86 +130,86 @@@
}
@Override
- public boolean authorize(String wasbAbsolutePath, String accessType,
- String delegationToken) throws WasbAuthorizationException, IOException {
-
+ public boolean authorize(String wasbAbsolutePath, String accessType)
+ throws WasbAuthorizationException, IOException {
+ try {
+ URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl);
+ uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP);
+ uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME,
+ wasbAbsolutePath);
+ uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME,
+ accessType);
- uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
- delegationToken);
++ if (isSecurityEnabled && StringUtils.isNotEmpty(delegationToken)) {
++ uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
++ delegationToken);
++ }
+
- String responseBody = remoteCallHelper.makeRemoteGetRequest(
- new HttpGet(uriBuilder.build()));
++ String responseBody = null;
++ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
++ UserGroupInformation connectUgi = ugi.getRealUser();
++ if (connectUgi == null) {
++ connectUgi = ugi;
++ } else {
++ uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName());
++ }
++ if (isSecurityEnabled && !connectUgi.hasKerberosCredentials()) {
++ connectUgi = UserGroupInformation.getLoginUser();
++ }
++ connectUgi.checkTGTAndReloginFromKeytab();
++
+ try {
- URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl);
- uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP);
- uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME,
- wasbAbsolutePath);
- uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME,
- accessType);
- if (isSecurityEnabled && StringUtils.isNotEmpty(delegationToken)) {
- uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
- delegationToken);
- }
-
- String responseBody = null;
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- UserGroupInformation connectUgi = ugi.getRealUser();
- if (connectUgi == null) {
- connectUgi = ugi;
- } else {
- uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName());
- }
- if (isSecurityEnabled && !connectUgi.hasKerberosCredentials()) {
- connectUgi = UserGroupInformation.getLoginUser();
- }
- connectUgi.checkTGTAndReloginFromKeytab();
-
- try {
- responseBody = connectUgi
- .doAs(new PrivilegedExceptionAction<String>() {
- @Override
- public String run() throws Exception {
- AuthenticatedURL.Token token = null;
- HttpGet httpGet = new HttpGet(uriBuilder.build());
- if (isKerberosSupportEnabled && UserGroupInformation
- .isSecurityEnabled() && (delegationToken == null
- || delegationToken.isEmpty())) {
- token = new AuthenticatedURL.Token();
- final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator();
- try {
- kerberosAuthenticator
- .authenticate(uriBuilder.build().toURL(), token);
- Validate.isTrue(token.isSet(),
- "Authenticated Token is NOT present. The request cannot proceed.");
- } catch (AuthenticationException e){
- throw new IOException("Authentication failed in check authorization", e);
- }
- if (token != null) {
- httpGet.setHeader("Cookie",
- AuthenticatedURL.AUTH_COOKIE + "=" + token);
- }
++ responseBody = connectUgi
++ .doAs(new PrivilegedExceptionAction<String>() {
++ @Override
++ public String run() throws Exception {
++ AuthenticatedURL.Token token = null;
++ HttpGet httpGet = new HttpGet(uriBuilder.build());
++ if (isKerberosSupportEnabled && UserGroupInformation
++ .isSecurityEnabled() && (delegationToken == null
++ || delegationToken.isEmpty())) {
++ token = new AuthenticatedURL.Token();
++ final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator();
++ try {
++ kerberosAuthenticator
++ .authenticate(uriBuilder.build().toURL(), token);
++ Validate.isTrue(token.isSet(),
++ "Authenticated Token is NOT present. The request cannot proceed.");
++ } catch (AuthenticationException e){
++ throw new IOException("Authentication failed in check authorization", e);
++ }
++ if (token != null) {
++ httpGet.setHeader("Cookie",
++ AuthenticatedURL.AUTH_COOKIE + "=" + token);
+ }
- return remoteCallHelper.makeRemoteGetRequest(httpGet);
+ }
- });
- } catch (InterruptedException e) {
- LOG.error("Error in check authorization", e);
- throw new WasbAuthorizationException("Error in check authorize", e);
- }
-
- ObjectMapper objectMapper = new ObjectMapper();
- RemoteAuthorizerResponse authorizerResponse =
- objectMapper
- .readValue(responseBody, RemoteAuthorizerResponse.class);
-
- if (authorizerResponse == null) {
- throw new WasbAuthorizationException(
- "RemoteAuthorizerResponse object null from remote call");
- } else if (authorizerResponse.getResponseCode()
- == REMOTE_CALL_SUCCESS_CODE) {
- return authorizerResponse.getAuthorizationResult();
- } else {
- throw new WasbAuthorizationException("Remote authorization"
- + " serivce encountered an error "
- + authorizerResponse.getResponseMessage());
- }
- } catch (URISyntaxException | WasbRemoteCallException
- | JsonParseException | JsonMappingException ex) {
- throw new WasbAuthorizationException(ex);
++ return remoteCallHelper.makeRemoteGetRequest(httpGet);
++ }
++ });
++ } catch (InterruptedException e) {
++ LOG.error("Error in check authorization", e);
++ throw new WasbAuthorizationException("Error in check authorize", e);
+ }
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ RemoteAuthorizerResponse authorizerResponse =
- objectMapper.readValue(responseBody, RemoteAuthorizerResponse.class);
++ objectMapper
++ .readValue(responseBody, RemoteAuthorizerResponse.class);
+
+ if (authorizerResponse == null) {
+ throw new WasbAuthorizationException(
+ "RemoteAuthorizerResponse object null from remote call");
+ } else if (authorizerResponse.getResponseCode()
+ == REMOTE_CALL_SUCCESS_CODE) {
+ return authorizerResponse.getAuthorizationResult();
+ } else {
+ throw new WasbAuthorizationException("Remote authorization"
- + " service encountered an error "
++ + " serivce encountered an error "
+ + authorizerResponse.getResponseMessage());
+ }
+ } catch (URISyntaxException | WasbRemoteCallException
+ | JsonParseException | JsonMappingException ex) {
+ throw new WasbAuthorizationException(ex);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --cc hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 43c0271,06ba137..6269f21
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@@ -99,7 -100,7 +100,7 @@@ public class TestAMRMClient
private List<NodeReport> nodeReports = null;
private ApplicationAttemptId attemptId = null;
private int nodeCount = 3;
--
++
static final int rolling_interval_sec = 13;
static final long am_expire_ms = 4000;
@@@ -126,12 -127,16 +127,16 @@@
@Before
public void setup() throws Exception {
- // start minicluster
conf = new YarnConfiguration();
+ createClusterAndStartApplication();
+ }
+
+ private void createClusterAndStartApplication() throws Exception {
+ // start minicluster
conf.set(YarnConfiguration.RM_SCHEDULER, schedulerName);
conf.setLong(
-- YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
-- rolling_interval_sec);
++ YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
++ rolling_interval_sec);
conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
// set the minimum allocation so that resource decrease can go under 1024
@@@ -148,7 -153,7 +153,7 @@@
// get node info
assertTrue("All node managers did not connect to the RM within the "
-- + "allotted 5-second timeout",
++ + "allotted 5-second timeout",
yarnCluster.waitForNodeManagersToConnect(5000L));
nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
assertEquals("Not all node managers were reported running",
@@@ -164,7 -169,7 +169,7 @@@
racks = new String[]{ rack };
// submit new app
-- ApplicationSubmissionContext appContext =
++ ApplicationSubmissionContext appContext =
yarnClient.createApplication().getApplicationSubmissionContext();
ApplicationId appId = appContext.getApplicationId();
// set the application name
@@@ -178,10 -183,10 +183,10 @@@
// Set up the container launch context for the application master
ContainerLaunchContext amContainer =
BuilderUtils.newContainerLaunchContext(
-- Collections.<String, LocalResource> emptyMap(),
-- new HashMap<String, String>(), Arrays.asList("sleep", "100"),
-- new HashMap<String, ByteBuffer>(), null,
-- new HashMap<ApplicationAccessType, String>());
++ Collections.<String, LocalResource> emptyMap(),
++ new HashMap<String, String>(), Arrays.asList("sleep", "100"),
++ new HashMap<String, ByteBuffer>(), null,
++ new HashMap<ApplicationAccessType, String>());
appContext.setAMContainerSpec(amContainer);
appContext.setResource(Resource.newInstance(1024, 1));
// Create the request to send to the applications manager
@@@ -199,7 -204,7 +204,7 @@@
attemptId = appReport.getCurrentApplicationAttemptId();
appAttempt =
yarnCluster.getResourceManager().getRMContext().getRMApps()
-- .get(attemptId.getApplicationId()).getCurrentAppAttempt();
++ .get(attemptId.getApplicationId()).getCurrentAppAttempt();
while (true) {
if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
break;
@@@ -211,14 -216,14 +216,14 @@@
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.setLoginUser(UserGroupInformation
-- .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
++ .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
// emulate RM setup of AMRM token in credentials by adding the token
// *before* setting the token service
UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
--
++
@After
public void teardown() throws YarnException, IOException {
yarnClient.killApplication(attemptId.getApplicationId());
@@@ -245,7 -250,7 +250,7 @@@
amClient.getMatchingRequests(priority, node, testCapability1);
assertEquals("Expected no matching requests.", matches.size(), 0);
}
--
++
@Test (timeout=60000)
public void testAMRMClientMatchingFit() throws YarnException, IOException {
AMRMClient<ContainerRequest> amClient = null;
@@@ -255,7 -260,7 +260,7 @@@
amClient.init(conf);
amClient.start();
amClient.registerApplicationMaster("Host", 10000, "");
--
++
Resource capability1 = Resource.newInstance(1024, 2);
Resource capability2 = Resource.newInstance(1024, 1);
Resource capability3 = Resource.newInstance(1000, 2);
@@@ -264,19 -269,19 +269,19 @@@
Resource capability6 = Resource.newInstance(2000, 1);
Resource capability7 = Resource.newInstance(2000, 1);
-- ContainerRequest storedContainer1 =
++ ContainerRequest storedContainer1 =
new ContainerRequest(capability1, nodes, racks, priority);
-- ContainerRequest storedContainer2 =
++ ContainerRequest storedContainer2 =
new ContainerRequest(capability2, nodes, racks, priority);
-- ContainerRequest storedContainer3 =
++ ContainerRequest storedContainer3 =
new ContainerRequest(capability3, nodes, racks, priority);
-- ContainerRequest storedContainer4 =
++ ContainerRequest storedContainer4 =
new ContainerRequest(capability4, nodes, racks, priority);
-- ContainerRequest storedContainer5 =
++ ContainerRequest storedContainer5 =
new ContainerRequest(capability5, nodes, racks, priority);
-- ContainerRequest storedContainer6 =
++ ContainerRequest storedContainer6 =
new ContainerRequest(capability6, nodes, racks, priority);
-- ContainerRequest storedContainer7 =
++ ContainerRequest storedContainer7 =
new ContainerRequest(capability7, nodes, racks, priority2, false);
amClient.addContainerRequest(storedContainer1);
amClient.addContainerRequest(storedContainer2);
@@@ -297,7 -302,7 +302,7 @@@
amClient.addContainerRequest(storedContainer11);
amClient.addContainerRequest(storedContainer33);
amClient.addContainerRequest(storedContainer43);
--
++
// test matching of containers
List<? extends Collection<ContainerRequest>> matches;
ContainerRequest storedRequest;
@@@ -327,7 -332,7 +332,7 @@@
storedRequest = iter.next();
assertEquals(storedContainer33, storedRequest);
amClient.removeContainerRequest(storedContainer33);
--
++
// exact matching with order maintained
Resource testCapability2 = Resource.newInstance(2000, 1);
matches = amClient.getMatchingRequests(priority, node, testCapability2);
@@@ -342,12 -347,12 +347,12 @@@
}
}
amClient.removeContainerRequest(storedContainer6);
--
++
// matching with larger container. all requests returned
Resource testCapability3 = Resource.newInstance(4000, 4);
matches = amClient.getMatchingRequests(priority, node, testCapability3);
assert(matches.size() == 4);
--
++
Resource testCapability4 = Resource.newInstance(1024, 2);
matches = amClient.getMatchingRequests(priority, node, testCapability4);
assert(matches.size() == 2);
@@@ -357,14 -362,14 +362,14 @@@
ContainerRequest testRequest = testSet.iterator().next();
assertTrue(testRequest != storedContainer4);
assertTrue(testRequest != storedContainer5);
-- assert(testRequest == storedContainer2 ||
-- testRequest == storedContainer3);
++ assert(testRequest == storedContainer2 ||
++ testRequest == storedContainer3);
}
--
++
Resource testCapability5 = Resource.newInstance(512, 4);
matches = amClient.getMatchingRequests(priority, node, testCapability5);
assert(matches.size() == 0);
--
++
// verify requests without relaxed locality are only returned at specific
// locations
Resource testCapability7 = Resource.newInstance(2000, 1);
@@@ -373,7 -378,7 +378,7 @@@
assert(matches.size() == 0);
matches = amClient.getMatchingRequests(priority2, node, testCapability7);
assert(matches.size() == 1);
--
++
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
null, null);
@@@ -516,14 -521,14 +521,14 @@@
}
}
}
--
++
private void verifyMatches(
-- List<? extends Collection<ContainerRequest>> matches,
-- int matchSize) {
++ List<? extends Collection<ContainerRequest>> matches,
++ int matchSize) {
assertEquals(1, matches.size());
assertEquals(matches.get(0).size(), matchSize);
}
--
++
@Test (timeout=60000)
public void testAMRMClientMatchingFitInferredRack() throws YarnException, IOException {
AMRMClientImpl<ContainerRequest> amClient = null;
@@@ -533,10 -538,10 +538,10 @@@
amClient.init(conf);
amClient.start();
amClient.registerApplicationMaster("Host", 10000, "");
--
++
Resource capability = Resource.newInstance(1024, 2);
-- ContainerRequest storedContainer1 =
++ ContainerRequest storedContainer1 =
new ContainerRequest(capability, nodes, null, priority);
amClient.addContainerRequest(storedContainer1);
@@@ -553,12 -558,12 +558,12 @@@
verifyMatches(matches, 1);
storedRequest = matches.get(0).iterator().next();
assertEquals(storedContainer1, storedRequest);
--
++
// inferred rack match no longer valid after request is removed
amClient.removeContainerRequest(storedContainer1);
matches = amClient.getMatchingRequests(priority, rack, capability);
assertTrue(matches.isEmpty());
--
++
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
null, null);
@@@ -576,24 -581,24 +581,24 @@@
// start am rm client
amClient =
(AMRMClientImpl<ContainerRequest>) AMRMClient
-- .<ContainerRequest> createAMRMClient();
++ .<ContainerRequest> createAMRMClient();
amClient.init(conf);
amClient.start();
amClient.registerApplicationMaster("Host", 10000, "");
--
++
Priority priority1 = Records.newRecord(Priority.class);
priority1.setPriority(2);
--
-- ContainerRequest storedContainer1 =
++
++ ContainerRequest storedContainer1 =
new ContainerRequest(capability, nodes, racks, priority);
-- ContainerRequest storedContainer2 =
++ ContainerRequest storedContainer2 =
new ContainerRequest(capability, nodes, racks, priority);
-- ContainerRequest storedContainer3 =
++ ContainerRequest storedContainer3 =
new ContainerRequest(capability, null, null, priority1);
amClient.addContainerRequest(storedContainer1);
amClient.addContainerRequest(storedContainer2);
amClient.addContainerRequest(storedContainer3);
--
++
// test addition and storage
RemoteRequestsTable<ContainerRequest> remoteRequestsTable =
amClient.getTable(0);
@@@ -604,21 -609,21 +609,21 @@@
containersRequestedAny = remoteRequestsTable.get(priority1,
ResourceRequest.ANY, ExecutionType.GUARANTEED, capability)
.remoteRequest.getNumContainers();
-- assertEquals(1, containersRequestedAny);
-- List<? extends Collection<ContainerRequest>> matches =
++ assertEquals(1, containersRequestedAny);
++ List<? extends Collection<ContainerRequest>> matches =
amClient.getMatchingRequests(priority, node, capability);
verifyMatches(matches, 2);
matches = amClient.getMatchingRequests(priority, rack, capability);
verifyMatches(matches, 2);
-- matches =
++ matches =
amClient.getMatchingRequests(priority, ResourceRequest.ANY, capability);
verifyMatches(matches, 2);
matches = amClient.getMatchingRequests(priority1, rack, capability);
assertTrue(matches.isEmpty());
-- matches =
++ matches =
amClient.getMatchingRequests(priority1, ResourceRequest.ANY, capability);
verifyMatches(matches, 1);
--
++
// test removal
amClient.removeContainerRequest(storedContainer3);
matches = amClient.getMatchingRequests(priority, node, capability);
@@@ -628,20 -633,20 +633,20 @@@
verifyMatches(matches, 1);
matches = amClient.getMatchingRequests(priority, rack, capability);
verifyMatches(matches, 1);
--
++
// test matching of containers
ContainerRequest storedRequest = matches.get(0).iterator().next();
assertEquals(storedContainer1, storedRequest);
amClient.removeContainerRequest(storedContainer1);
-- matches =
++ matches =
amClient.getMatchingRequests(priority, ResourceRequest.ANY, capability);
assertTrue(matches.isEmpty());
-- matches =
++ matches =
amClient.getMatchingRequests(priority1, ResourceRequest.ANY, capability);
assertTrue(matches.isEmpty());
// 0 requests left. everything got cleaned up
assertTrue(amClient.getTable(0).isEmpty());
--
++
// go through an exemplary allocation, matching and release cycle
amClient.addContainerRequest(storedContainer1);
amClient.addContainerRequest(storedContainer3);
@@@ -655,16 -660,16 +660,16 @@@
AllocateResponse allocResponse = amClient.allocate(0.1f);
assertEquals(0, amClient.ask.size());
assertEquals(0, amClient.release.size());
--
++
assertEquals(nodeCount, amClient.getClusterNodeCount());
allocatedContainerCount += allocResponse.getAllocatedContainers().size();
for(Container container : allocResponse.getAllocatedContainers()) {
-- ContainerRequest expectedRequest =
++ ContainerRequest expectedRequest =
container.getPriority().equals(storedContainer1.getPriority()) ?
storedContainer1 : storedContainer3;
-- matches = amClient.getMatchingRequests(container.getPriority(),
-- ResourceRequest.ANY,
-- container.getResource());
++ matches = amClient.getMatchingRequests(container.getPriority(),
++ ResourceRequest.ANY,
++ container.getResource());
// test correct matched container is returned
verifyMatches(matches, 1);
ContainerRequest matchedRequest = matches.get(0).iterator().next();
@@@ -678,7 -683,7 +683,7 @@@
triggerSchedulingWithNMHeartBeat();
}
}
--
++
assertEquals(2, allocatedContainerCount);
AllocateResponse allocResponse = amClient.allocate(0.1f);
assertEquals(0, amClient.release.size());
@@@ -686,7 -691,7 +691,7 @@@
assertEquals(0, allocResponse.getAllocatedContainers().size());
// 0 requests left. everything got cleaned up
assertTrue(remoteRequestsTable.isEmpty());
--
++
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
null, null);
@@@ -724,46 -729,46 +729,46 @@@
// start am rm client
amClient =
(AMRMClientImpl<ContainerRequest>) AMRMClient
-- .<ContainerRequest> createAMRMClient();
++ .<ContainerRequest> createAMRMClient();
amClient.init(conf);
amClient.start();
amClient.registerApplicationMaster("Host", 10000, "");
--
++
assertEquals(0, amClient.ask.size());
assertEquals(0, amClient.release.size());
--
-- ContainerRequest storedContainer1 =
++
++ ContainerRequest storedContainer1 =
new ContainerRequest(capability, nodes, racks, priority);
amClient.addContainerRequest(storedContainer1);
assertEquals(3, amClient.ask.size());
assertEquals(0, amClient.release.size());
--
++
List<String> localNodeBlacklist = new ArrayList<String>();
localNodeBlacklist.add(node);
--
++
// put node in black list, so no container assignment
amClient.updateBlacklist(localNodeBlacklist, null);
int allocatedContainerCount = getAllocatedContainersNumber(amClient,
-- DEFAULT_ITERATION);
++ DEFAULT_ITERATION);
// the only node is in blacklist, so no allocation
assertEquals(0, allocatedContainerCount);
// Remove node from blacklist, so get assigned with 2
amClient.updateBlacklist(null, localNodeBlacklist);
-- ContainerRequest storedContainer2 =
-- new ContainerRequest(capability, nodes, racks, priority);
++ ContainerRequest storedContainer2 =
++ new ContainerRequest(capability, nodes, racks, priority);
amClient.addContainerRequest(storedContainer2);
allocatedContainerCount = getAllocatedContainersNumber(amClient,
DEFAULT_ITERATION);
assertEquals(2, allocatedContainerCount);
--
++
// Test in case exception in allocate(), blacklist is kept
assertTrue(amClient.blacklistAdditions.isEmpty());
assertTrue(amClient.blacklistRemovals.isEmpty());
--
++
// create a invalid ContainerRequest - memory value is minus
-- ContainerRequest invalidContainerRequest =
++ ContainerRequest invalidContainerRequest =
new ContainerRequest(Resource.newInstance(-1024, 1),
nodes, racks, priority);
amClient.addContainerRequest(invalidContainerRequest);
@@@ -781,7 -786,7 +786,7 @@@
}
}
}
--
++
@Test (timeout=60000)
public void testAMRMClientWithBlacklist() throws YarnException, IOException {
AMRMClientImpl<ContainerRequest> amClient = null;
@@@ -789,12 -794,12 +794,12 @@@
// start am rm client
amClient =
(AMRMClientImpl<ContainerRequest>) AMRMClient
-- .<ContainerRequest> createAMRMClient();
++ .<ContainerRequest> createAMRMClient();
amClient.init(conf);
amClient.start();
amClient.registerApplicationMaster("Host", 10000, "");
String[] nodes = {"node1", "node2", "node3"};
--
++
// Add nodes[0] and nodes[1]
List<String> nodeList01 = new ArrayList<String>();
nodeList01.add(nodes[0]);
@@@ -802,7 -807,7 +807,7 @@@
amClient.updateBlacklist(nodeList01, null);
assertEquals(2, amClient.blacklistAdditions.size());
assertEquals(0, amClient.blacklistRemovals.size());
--
++
// Add nodes[0] again, verify it is not added duplicated.
List<String> nodeList02 = new ArrayList<String>();
nodeList02.add(nodes[0]);
@@@ -810,7 -815,7 +815,7 @@@
amClient.updateBlacklist(nodeList02, null);
assertEquals(3, amClient.blacklistAdditions.size());
assertEquals(0, amClient.blacklistRemovals.size());
--
++
// Add nodes[1] and nodes[2] to removal list,
// Verify addition list remove these two nodes.
List<String> nodeList12 = new ArrayList<String>();
@@@ -819,7 -824,7 +824,7 @@@
amClient.updateBlacklist(null, nodeList12);
assertEquals(1, amClient.blacklistAdditions.size());
assertEquals(2, amClient.blacklistRemovals.size());
--
++
// Add nodes[1] again to addition list,
// Verify removal list will remove this node.
List<String> nodeList1 = new ArrayList<String>();
@@@ -844,10 -849,10 +849,10 @@@
AllocateResponse allocResponse = amClient.allocate(0.1f);
assertEquals(0, amClient.ask.size());
assertEquals(0, amClient.release.size());
--
++
assertEquals(nodeCount, amClient.getClusterNodeCount());
allocatedContainerCount += allocResponse.getAllocatedContainers().size();
--
++
if(allocatedContainerCount == 0) {
// let NM heartbeat to RM and trigger allocations
triggerSchedulingWithNMHeartBeat();
@@@ -876,7 -892,7 +892,7 @@@
//setting an instance NMTokenCache
amClient.setNMTokenCache(new NMTokenCache());
//asserting we are not using the singleton instance cache
-- Assert.assertNotSame(NMTokenCache.getSingleton(),
++ Assert.assertNotSame(NMTokenCache.getSingleton(),
amClient.getNMTokenCache());
amClient.init(conf);
@@@ -899,7 -915,7 +915,7 @@@
}
}
}
--
++
@Test(timeout=30000)
public void testAskWithNodeLabels() {
AMRMClientImpl<ContainerRequest> client =
@@@ -920,7 -936,7 +936,7 @@@
Assert.assertEquals(1, client.ask.size());
Assert.assertEquals("a", client.ask.iterator().next()
.getNodeLabelExpression());
--
++
// add exp=x to ANY, rack and node, only resource request has ANY resource
// name will be assigned the label expression
// add exp=x then add exp=a to ANY in same priority, only exp=a should kept
@@@ -947,7 -963,7 +963,7 @@@
}
}
}
--
++
private void verifyAddRequestFailed(AMRMClient<ContainerRequest> client,
ContainerRequest request) {
try {
@@@ -957,7 -973,7 +973,7 @@@
}
Assert.fail();
}
--
++
@Test(timeout=30000)
public void testAskWithInvalidNodeLabels() {
AMRMClientImpl<ContainerRequest> client =
@@@ -1130,10 -1146,10 +1146,10 @@@
private void testAllocation(final AMRMClientImpl<ContainerRequest> amClient)
throws YarnException, IOException {
// setup container request
--
++
assertEquals(0, amClient.ask.size());
assertEquals(0, amClient.release.size());
--
++
amClient.addContainerRequest(
new ContainerRequest(capability, nodes, racks, priority));
amClient.addContainerRequest(
@@@ -1154,17 -1170,17 +1170,17 @@@
int allocatedContainerCount = 0;
int iterationsLeft = 3;
Set<ContainerId> releases = new TreeSet<ContainerId>();
--
++
amClient.getNMTokenCache().clearCache();
Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
HashMap<String, Token> receivedNMTokens = new HashMap<String, Token>();
--
++
while (allocatedContainerCount < containersRequestedAny
&& iterationsLeft-- > 0) {
AllocateResponse allocResponse = amClient.allocate(0.1f);
assertEquals(0, amClient.ask.size());
assertEquals(0, amClient.release.size());
--
++
assertEquals(nodeCount, amClient.getClusterNodeCount());
allocatedContainerCount += allocResponse.getAllocatedContainers().size();
for(Container container : allocResponse.getAllocatedContainers()) {
@@@ -1172,29 -1188,29 +1188,29 @@@
releases.add(rejectContainerId);
amClient.releaseAssignedContainer(rejectContainerId);
}
--
++
for (NMToken token : allocResponse.getNMTokens()) {
String nodeID = token.getNodeId().toString();
if (receivedNMTokens.containsKey(nodeID)) {
-- Assert.fail("Received token again for : " + nodeID);
++ Assert.fail("Received token again for : " + nodeID);
}
receivedNMTokens.put(nodeID, token.getToken());
}
--
++
if(allocatedContainerCount < containersRequestedAny) {
// let NM heartbeat to RM and trigger allocations
triggerSchedulingWithNMHeartBeat();
}
}
--
++
// Should receive atleast 1 token
Assert.assertTrue(receivedNMTokens.size() > 0
&& receivedNMTokens.size() <= nodeCount);
--
++
assertEquals(allocatedContainerCount, containersRequestedAny);
assertEquals(2, amClient.release.size());
assertEquals(0, amClient.ask.size());
--
++
// need to tell the AMRMClient that we dont need these resources anymore
amClient.removeContainerRequest(
new ContainerRequest(capability, nodes, racks, priority));
@@@ -1204,7 -1220,7 +1220,7 @@@
// send 0 container count request for resources that are no longer needed
ResourceRequest snoopRequest = amClient.ask.iterator().next();
assertEquals(0, snoopRequest.getNumContainers());
--
++
// test RPC exception handling
amClient.addContainerRequest(new ContainerRequest(capability, nodes,
racks, priority));
@@@ -1212,7 -1228,7 +1228,7 @@@
racks, priority));
snoopRequest = amClient.ask.iterator().next();
assertEquals(2, snoopRequest.getNumContainers());
--
++
ApplicationMasterProtocol realRM = amClient.rmClient;
try {
ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class);
@@@ -1221,8 -1237,8 +1237,8 @@@
public AllocateResponse answer(InvocationOnMock invocation)
throws Exception {
amClient.removeContainerRequest(
-- new ContainerRequest(capability, nodes,
-- racks, priority));
++ new ContainerRequest(capability, nodes,
++ racks, priority));
amClient.removeContainerRequest(
new ContainerRequest(capability, nodes, racks, priority));
throw new Exception();
@@@ -1398,7 -1414,7 +1414,7 @@@
}
}
}
--
++
private void sleep(int sleepTime) {
try {
Thread.sleep(sleepTime);
@@@ -1414,7 -1430,7 +1430,7 @@@
try {
AMRMTokenSecretManager amrmTokenSecretManager =
yarnCluster.getResourceManager().getRMContext()
-- .getAMRMTokenSecretManager();
++ .getAMRMTokenSecretManager();
// start am rm client
amClient = AMRMClient.<ContainerRequest> createAMRMClient();
@@@ -1429,7 -1445,7 +1445,7 @@@
getAMRMToken();
Assert.assertNotNull(amrmToken_1);
Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(),
-- amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
++ amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
// Wait for enough time and make sure the roll_over happens
// At mean time, the old AMRMToken should continue to work
@@@ -1444,41 -1460,41 +1460,41 @@@
getAMRMToken();
Assert.assertNotNull(amrmToken_2);
Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(),
-- amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
++ amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
Assert.assertNotEquals(amrmToken_1, amrmToken_2);
// can do the allocate call with latest AMRMToken
AllocateResponse response = amClient.allocate(0.1f);
--
++
// Verify latest AMRMToken can be used to send allocation request.
UserGroupInformation testUser1 =
UserGroupInformation.createRemoteUser("testUser1");
--
-- AMRMTokenIdentifierForTest newVersionTokenIdentifier =
++
++ AMRMTokenIdentifierForTest newVersionTokenIdentifier =
new AMRMTokenIdentifierForTest(amrmToken_2.decodeIdentifier(), "message");
--
++
Assert.assertEquals("Message is changed after set to newVersionTokenIdentifier",
"message", newVersionTokenIdentifier.getMessage());
-- org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newVersionToken =
++ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newVersionToken =
new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> (
-- newVersionTokenIdentifier.getBytes(),
++ newVersionTokenIdentifier.getBytes(),
amrmTokenSecretManager.retrievePassword(newVersionTokenIdentifier),
newVersionTokenIdentifier.getKind(), new Text());
--
++
SecurityUtil.setTokenService(newVersionToken, yarnCluster
-- .getResourceManager().getApplicationMasterService().getBindAddress());
++ .getResourceManager().getApplicationMasterService().getBindAddress());
testUser1.addToken(newVersionToken);
--
++
AllocateRequest request = Records.newRecord(AllocateRequest.class);
request.setResponseId(response.getResponseId());
testUser1.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
@Override
public ApplicationMasterProtocol run() {
return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(
-- ApplicationMasterProtocol.class,
-- yarnCluster.getResourceManager().getApplicationMasterService()
-- .getBindAddress(), conf);
++ ApplicationMasterProtocol.class,
++ yarnCluster.getResourceManager().getApplicationMasterService()
++ .getBindAddress(), conf);
}
}).allocate(request);
@@@ -1486,12 -1502,12 +1502,12 @@@
// and can not use this rolled-over token to make a allocate all.
while (true) {
if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager
-- .getCurrnetMasterKeyData().getMasterKey().getKeyId()) {
++ .getCurrnetMasterKeyData().getMasterKey().getKeyId()) {
if (amrmTokenSecretManager.getNextMasterKeyData() == null) {
break;
} else if (amrmToken_2.decodeIdentifier().getKeyId() !=
amrmTokenSecretManager.getNextMasterKeyData().getMasterKey()
-- .getKeyId()) {
++ .getKeyId()) {
break;
}
}
@@@ -1503,27 -1519,27 +1519,27 @@@
UserGroupInformation testUser2 =
UserGroupInformation.createRemoteUser("testUser2");
SecurityUtil.setTokenService(amrmToken_2, yarnCluster
-- .getResourceManager().getApplicationMasterService().getBindAddress());
++ .getResourceManager().getApplicationMasterService().getBindAddress());
testUser2.addToken(amrmToken_2);
testUser2.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
@Override
public ApplicationMasterProtocol run() {
return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(
-- ApplicationMasterProtocol.class,
-- yarnCluster.getResourceManager().getApplicationMasterService()
-- .getBindAddress(), conf);
++ ApplicationMasterProtocol.class,
++ yarnCluster.getResourceManager().getApplicationMasterService()
++ .getBindAddress(), conf);
}
}).allocate(Records.newRecord(AllocateRequest.class));
Assert.fail("The old Token should not work");
} catch (Exception ex) {
Assert.assertTrue(ex instanceof InvalidToken);
Assert.assertTrue(ex.getMessage().contains(
-- "Invalid AMRMToken from "
-- + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
++ "Invalid AMRMToken from "
++ + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
}
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
-- null, null);
++ null, null);
} finally {
if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
@@@ -1534,7 -1550,7 +1550,7 @@@
@SuppressWarnings("unchecked")
private org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>
-- getAMRMToken() throws IOException {
++ getAMRMToken() throws IOException {
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
Iterator<org.apache.hadoop.security.token.Token<?>> iter =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
----------------------------------------------------------------------
diff --cc hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
index ae7ab0d,40eb54d..d42acaa
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
@@@ -16,7 -16,7 +16,7 @@@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
++ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: HADOOP-14256. [S3A DOC] Correct
the format for "Seoul" example. Contributed by Brahma Reddy Battula.
Posted by ae...@apache.org.
HADOOP-14256. [S3A DOC] Correct the format for "Seoul" example. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d053eeb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d053eeb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d053eeb
Branch: refs/heads/HDFS-7240
Commit: 0d053eeb3002f348d34cf3aac5b43333af8ff10a
Parents: de7efd2
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Mar 30 18:11:50 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Mar 30 18:11:50 2017 +0900
----------------------------------------------------------------------
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d053eeb/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 3e99656..82c3588 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -1024,6 +1024,7 @@ Frankfurt
```
Seoul
+
```xml
<property>
<name>fs.s3a.endpoint</name>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-6376. Exceptions caused by
synchronous putEntities requests can be swallowed (Haibo Chen via Varun
Saxena)
Posted by ae...@apache.org.
YARN-6376. Exceptions caused by synchronous putEntities requests can be swallowed (Haibo Chen via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b58777a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b58777a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b58777a9
Branch: refs/heads/HDFS-7240
Commit: b58777a9c9a5b6f2e4bcfd2b3bede33f25f80dec
Parents: 7c2bc44
Author: Varun Saxena <va...@apache.org>
Authored: Fri Mar 31 02:17:20 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Fri Mar 31 02:17:20 2017 +0530
----------------------------------------------------------------------
.../timelineservice/collector/TimelineCollector.java | 10 ++++++++--
.../collector/TimelineCollectorManager.java | 7 ++++++-
2 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b58777a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
index 353066b..4c9e9f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -137,8 +137,14 @@ public abstract class TimelineCollector extends CompositeService {
+ callerUgi + ")");
}
- TimelineWriteResponse response = writeTimelineEntities(entities);
- flushBufferedTimelineEntities();
+ TimelineWriteResponse response;
+ // synchronize on the writer object so that no other threads can
+ // flush the writer buffer concurrently and swallow any exception
+ // caused by the timeline enitites that are being put here.
+ synchronized (writer) {
+ response = writeTimelineEntities(entities);
+ flushBufferedTimelineEntities();
+ }
return response;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b58777a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 19896e8..8ef9b43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -259,7 +259,12 @@ public class TimelineCollectorManager extends AbstractService {
public void run() {
try {
- writer.flush();
+ // synchronize on the writer object to avoid flushing timeline
+ // entities placed on the buffer by synchronous putEntities
+ // requests.
+ synchronized (writer) {
+ writer.flush();
+ }
} catch (Throwable th) {
// we need to handle all exceptions or subsequent execution may be
// suppressed
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HADOOP-11794. Enable distcp to
copy blocks in parallel. Contributed by Yongjun Zhang, Wei-Chiu Chuang,
Xiao Chen.
Posted by ae...@apache.org.
HADOOP-11794. Enable distcp to copy blocks in parallel. Contributed by Yongjun Zhang, Wei-Chiu Chuang, Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/064c8b25
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/064c8b25
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/064c8b25
Branch: refs/heads/HDFS-7240
Commit: 064c8b25eca9bc825dc07a54d9147d65c9290a03
Parents: e5f0622
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Thu Mar 30 17:01:15 2017 -0700
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Thu Mar 30 17:01:15 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 22 +-
.../org/apache/hadoop/tools/CopyListing.java | 37 +-
.../hadoop/tools/CopyListingFileStatus.java | 87 ++++-
.../java/org/apache/hadoop/tools/DistCp.java | 52 +++
.../apache/hadoop/tools/DistCpOptionSwitch.java | 10 +
.../org/apache/hadoop/tools/DistCpOptions.java | 22 +-
.../org/apache/hadoop/tools/OptionsParser.java | 36 +-
.../apache/hadoop/tools/SimpleCopyListing.java | 83 +++--
.../hadoop/tools/mapred/CopyCommitter.java | 174 ++++++++-
.../apache/hadoop/tools/mapred/CopyMapper.java | 40 +-
.../tools/mapred/RetriableFileCopyCommand.java | 26 +-
.../tools/mapred/UniformSizeInputFormat.java | 5 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 111 +++++-
.../src/site/markdown/DistCp.md.vm | 1 +
.../apache/hadoop/tools/TestDistCpSystem.java | 368 +++++++++++++++++--
.../apache/hadoop/tools/TestOptionsParser.java | 2 +-
.../hadoop/tools/mapred/TestCopyCommitter.java | 5 +-
17 files changed, 971 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 1329195..9b782f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -862,7 +862,27 @@ public class DFSTestUtil {
out.write(toAppend);
}
}
-
+
+ /**
+ * Append specified length of bytes to a given file, starting with new block.
+ * @param fs The file system
+ * @param p Path of the file to append
+ * @param length Length of bytes to append to the file
+ * @throws IOException
+ */
+ public static void appendFileNewBlock(DistributedFileSystem fs,
+ Path p, int length) throws IOException {
+ assert fs.exists(p);
+ assert length >= 0;
+ byte[] toAppend = new byte[length];
+ Random random = new Random();
+ random.nextBytes(toAppend);
+ try (FSDataOutputStream out = fs.append(p,
+ EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null)) {
+ out.write(toAppend);
+ }
+ }
+
/**
* @return url content as string (UTF-8 encoding assumed)
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
index 481aa61..9ebf9d2 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
@@ -145,12 +145,22 @@ public abstract class CopyListing extends Configured {
Configuration config = getConf();
FileSystem fs = pathToListFile.getFileSystem(config);
- Path sortedList = DistCpUtils.sortListing(fs, config, pathToListFile);
+ final boolean splitLargeFile = options.splitLargeFile();
+
+ // When splitLargeFile is enabled, we don't randomize the copylist
+ // earlier, so we don't do the sorting here. For a file that has
+ // multiple entries due to split, we check here that their
+ // <chunkOffset, chunkLength> is continuous.
+ //
+ Path checkPath = splitLargeFile?
+ pathToListFile : DistCpUtils.sortListing(fs, config, pathToListFile);
SequenceFile.Reader reader = new SequenceFile.Reader(
- config, SequenceFile.Reader.file(sortedList));
+ config, SequenceFile.Reader.file(checkPath));
try {
Text lastKey = new Text("*"); //source relative path can never hold *
+ long lastChunkOffset = -1;
+ long lastChunkLength = -1;
CopyListingFileStatus lastFileStatus = new CopyListingFileStatus();
Text currentKey = new Text();
@@ -161,8 +171,21 @@ public abstract class CopyListing extends Configured {
if (currentKey.equals(lastKey)) {
CopyListingFileStatus currentFileStatus = new CopyListingFileStatus();
reader.getCurrentValue(currentFileStatus);
- throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " +
- currentFileStatus.getPath() + " would cause duplicates. Aborting");
+ if (!splitLargeFile) {
+ throw new DuplicateFileException("File " + lastFileStatus.getPath()
+ + " and " + currentFileStatus.getPath()
+ + " would cause duplicates. Aborting");
+ } else {
+ if (lastChunkOffset + lastChunkLength !=
+ currentFileStatus.getChunkOffset()) {
+ throw new InvalidInputException("File " + lastFileStatus.getPath()
+ + " " + lastChunkOffset + "," + lastChunkLength
+ + " and " + currentFileStatus.getPath()
+ + " " + currentFileStatus.getChunkOffset() + ","
+ + currentFileStatus.getChunkLength()
+ + " are not continuous. Aborting");
+ }
+ }
}
reader.getCurrentValue(lastFileStatus);
if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
@@ -181,8 +204,12 @@ public abstract class CopyListing extends Configured {
xAttrSupportCheckFsSet.add(lastFsUri);
}
}
- lastKey.set(currentKey);
+ lastKey.set(currentKey);
+ if (splitLargeFile) {
+ lastChunkOffset = lastFileStatus.getChunkOffset();
+ lastChunkLength = lastFileStatus.getChunkLength();
+ }
if (options.shouldUseDiff() && LOG.isDebugEnabled()) {
LOG.debug("Copy list entry " + idx + ": " +
lastFileStatus.getPath().toUri().getPath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
index 00d4b32..29c59ac 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
@@ -74,6 +74,14 @@ public final class CopyListingFileStatus implements Writable {
private List<AclEntry> aclEntries;
private Map<String, byte[]> xAttrs;
+ // <chunkOffset, chunkLength> represents the offset and length of a file
+ // chunk in number of bytes.
+ // used when splitting a large file to chunks to copy in parallel.
+ // If a file is not large enough to split, chunkOffset would be 0 and
+ // chunkLength would be the length of the file.
+ private long chunkOffset = 0;
+ private long chunkLength = Long.MAX_VALUE;
+
/**
* Default constructor.
*/
@@ -96,11 +104,32 @@ public final class CopyListingFileStatus implements Writable {
fileStatus.getPath());
}
+ public CopyListingFileStatus(FileStatus fileStatus,
+ long chunkOffset, long chunkLength) {
+ this(fileStatus.getLen(), fileStatus.isDirectory(),
+ fileStatus.getReplication(), fileStatus.getBlockSize(),
+ fileStatus.getModificationTime(), fileStatus.getAccessTime(),
+ fileStatus.getPermission(), fileStatus.getOwner(),
+ fileStatus.getGroup(),
+ fileStatus.getPath());
+ this.chunkOffset = chunkOffset;
+ this.chunkLength = chunkLength;
+ }
+
@SuppressWarnings("checkstyle:parameternumber")
public CopyListingFileStatus(long length, boolean isdir,
int blockReplication, long blocksize, long modificationTime,
long accessTime, FsPermission permission, String owner, String group,
Path path) {
+ this(length, isdir, blockReplication, blocksize, modificationTime,
+ accessTime, permission, owner, group, path, 0, Long.MAX_VALUE);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ public CopyListingFileStatus(long length, boolean isdir,
+ int blockReplication, long blocksize, long modificationTime,
+ long accessTime, FsPermission permission, String owner, String group,
+ Path path, long chunkOffset, long chunkLength) {
this.length = length;
this.isdir = isdir;
this.blockReplication = (short)blockReplication;
@@ -117,6 +146,23 @@ public final class CopyListingFileStatus implements Writable {
this.owner = (owner == null) ? "" : owner;
this.group = (group == null) ? "" : group;
this.path = path;
+ this.chunkOffset = chunkOffset;
+ this.chunkLength = chunkLength;
+ }
+
+ public CopyListingFileStatus(CopyListingFileStatus other) {
+ this.length = other.length;
+ this.isdir = other.isdir;
+ this.blockReplication = other.blockReplication;
+ this.blocksize = other.blocksize;
+ this.modificationTime = other.modificationTime;
+ this.accessTime = other.accessTime;
+ this.permission = other.permission;
+ this.owner = other.owner;
+ this.group = other.group;
+ this.path = new Path(other.path.toUri());
+ this.chunkOffset = other.chunkOffset;
+ this.chunkLength = other.chunkLength;
}
public Path getPath() {
@@ -200,6 +246,31 @@ public final class CopyListingFileStatus implements Writable {
this.xAttrs = xAttrs;
}
+ public long getChunkOffset() {
+ return chunkOffset;
+ }
+
+ public void setChunkOffset(long offset) {
+ this.chunkOffset = offset;
+ }
+
+ public long getChunkLength() {
+ return chunkLength;
+ }
+
+ public void setChunkLength(long chunkLength) {
+ this.chunkLength = chunkLength;
+ }
+
+ public boolean isSplit() {
+ return getChunkLength() != Long.MAX_VALUE &&
+ getChunkLength() != getLen();
+ }
+
+ public long getSizeToCopy() {
+ return isSplit()? getChunkLength() : getLen();
+ }
+
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN);
@@ -244,6 +315,9 @@ public final class CopyListingFileStatus implements Writable {
} else {
out.writeInt(NO_XATTRS);
}
+
+ out.writeLong(chunkOffset);
+ out.writeLong(chunkLength);
}
@Override
@@ -292,6 +366,9 @@ public final class CopyListingFileStatus implements Writable {
} else {
xAttrs = null;
}
+
+ chunkOffset = in.readLong();
+ chunkLength = in.readLong();
}
@Override
@@ -317,8 +394,14 @@ public final class CopyListingFileStatus implements Writable {
public String toString() {
StringBuilder sb = new StringBuilder(super.toString());
sb.append('{');
- sb.append("aclEntries = " + aclEntries);
- sb.append(", xAttrs = " + xAttrs);
+ sb.append(this.getPath().toString());
+ sb.append(" length = ").append(this.getLen());
+ sb.append(" aclEntries = ").append(aclEntries);
+ sb.append(", xAttrs = ").append(xAttrs);
+ if (isSplit()) {
+ sb.append(", chunkOffset = ").append(this.getChunkOffset());
+ sb.append(", chunkLength = ").append(this.getChunkLength());
+ }
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index ab58e9c..8c2fa24 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.Cluster;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
+import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.CopyListing.*;
import org.apache.hadoop.tools.mapred.CopyMapper;
import org.apache.hadoop.tools.mapred.CopyOutputFormat;
@@ -134,6 +135,7 @@ public class DistCp extends Configured implements Tool {
try {
inputOptions = (OptionsParser.parse(argv));
+ setOptionsForSplitLargeFile();
setTargetPathExists();
LOG.info("Input Options: " + inputOptions);
} catch (Throwable e) {
@@ -235,6 +237,56 @@ public class DistCp extends Configured implements Tool {
getConf().setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS,
targetExists);
}
+
+ /**
+ * Check if concat is supported by fs.
+ * Throws UnsupportedOperationException if not.
+ */
+ private void checkConcatSupport(FileSystem fs) {
+ try {
+ Path[] src = null;
+ Path tgt = null;
+ fs.concat(tgt, src);
+ } catch (UnsupportedOperationException use) {
+ throw new UnsupportedOperationException(
+ DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() +
+ " is not supported since the target file system doesn't" +
+ " support concat.", use);
+ } catch (Exception e) {
+ // Ignore other exception
+ }
+ }
+
+ /**
+ * Set up needed options for splitting large files.
+ */
+ private void setOptionsForSplitLargeFile() throws IOException {
+ if (!inputOptions.splitLargeFile()) {
+ return;
+ }
+ Path target = inputOptions.getTargetPath();
+ FileSystem targetFS = target.getFileSystem(getConf());
+ checkConcatSupport(targetFS);
+
+ LOG.info("Enabling preserving blocksize since "
+ + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " is passed.");
+ inputOptions.preserve(FileAttribute.BLOCKSIZE);
+
+ LOG.info("Set " +
+ DistCpOptionSwitch.APPEND.getSwitch()
+ + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()
+ + " is passed.");
+ inputOptions.setAppend(false);
+
+ LOG.info("Set " +
+ DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES
+ + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()
+ + " is passed.");
+ getConf().setBoolean(
+ DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false);
+ }
+
+
/**
* Create Job object for submitting it, with all the configuration
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index fb47d76..ced9b54 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -169,6 +169,16 @@ public enum DistCpOptionSwitch {
new Option("sizelimit", true, "(Deprecated!) Limit number of files " +
"copied to <= n bytes")),
+ BLOCKS_PER_CHUNK("",
+ new Option("blocksperchunk", true, "If set to a positive value, files"
+ + "with more blocks than this value will be split into chunks of "
+ + "<blocksperchunk> blocks to be transferred in parallel, and "
+ + "reassembled on the destination. By default, <blocksperchunk> is "
+ + "0 and the files will be transmitted in their entirety without "
+ + "splitting. This switch is only applicable when the source file "
+ + "system implements getBlockLocations method and the target file "
+ + "system implements concat method")),
+
/**
* Specify bandwidth per map in MB, accepts bandwidth as a fraction
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index 8c37ff3..9822d83 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -97,7 +97,11 @@ public class DistCpOptions {
// targetPathExist is a derived field, it's initialized in the
// beginning of distcp.
private boolean targetPathExists = true;
-
+
+ // Size of chunk in number of blocks when splitting large file into chunks
+ // to copy in parallel. Default is 0 and file are not splitted.
+ private int blocksPerChunk = 0;
+
public static enum FileAttribute{
REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES;
@@ -166,6 +170,7 @@ public class DistCpOptions {
this.targetPath = that.getTargetPath();
this.targetPathExists = that.getTargetPathExists();
this.filtersFile = that.getFiltersFile();
+ this.blocksPerChunk = that.blocksPerChunk;
}
}
@@ -578,6 +583,18 @@ public class DistCpOptions {
this.filtersFile = filtersFilename;
}
+ public final void setBlocksPerChunk(int csize) {
+ this.blocksPerChunk = csize;
+ }
+
+ public final int getBlocksPerChunk() {
+ return blocksPerChunk;
+ }
+
+ public final boolean splitLargeFile() {
+ return blocksPerChunk > 0;
+ }
+
void validate() {
if ((useDiff || useRdiff) && deleteMissing) {
// -delete and -diff/-rdiff are mutually exclusive. For backward
@@ -669,6 +686,8 @@ public class DistCpOptions {
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.FILTERS,
filtersFile);
}
+ DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BLOCKS_PER_CHUNK,
+ String.valueOf(blocksPerChunk));
}
/**
@@ -704,6 +723,7 @@ public class DistCpOptions {
", targetPath=" + targetPath +
", targetPathExists=" + targetPathExists +
", filtersFile='" + filtersFile + '\'' +
+ ", blocksPerChunk=" + blocksPerChunk +
'}';
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
index d0f82ca..8881264 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
@@ -172,11 +172,44 @@ public class OptionsParser {
DistCpOptionSwitch.FILTERS.getSwitch()));
}
+ parseBlocksPerChunk(command, option);
+
option.validate();
return option;
}
+
+ /**
+ * A helper method to parse chunk size in number of blocks.
+ * Used when breaking large file into chunks to copy in parallel.
+ *
+ * @param command command line arguments
+ */
+ private static void parseBlocksPerChunk(CommandLine command,
+ DistCpOptions option) {
+ boolean hasOption =
+ command.hasOption(DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch());
+ LOG.info("parseChunkSize: " +
+ DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " " + hasOption);
+ if (hasOption) {
+ String chunkSizeString = getVal(command,
+ DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch().trim());
+ try {
+ int csize = Integer.parseInt(chunkSizeString);
+ if (csize < 0) {
+ csize = 0;
+ }
+ LOG.info("Set distcp blocksPerChunk to " + csize);
+ option.setBlocksPerChunk(csize);
+ }
+ catch (NumberFormatException e) {
+ throw new IllegalArgumentException("blocksPerChunk is invalid: "
+ + chunkSizeString, e);
+ }
+ }
+ }
+
/**
* parseSizeLimit is a helper method for parsing the deprecated
* argument SIZE_LIMIT.
@@ -211,8 +244,7 @@ public class OptionsParser {
DistCpOptionSwitch.FILE_LIMIT.getSwitch().trim());
try {
Integer.parseInt(fileLimitString);
- }
- catch (NumberFormatException e) {
+ } catch (NumberFormatException e) {
throw new IllegalArgumentException("File-limit is invalid: "
+ fileLimitString, e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
index 105e4f2..af91347 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.tools;
import com.google.common.collect.Lists;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
@@ -47,6 +48,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
+import java.util.LinkedList;
import static org.apache.hadoop.tools.DistCpConstants
.HDFS_RESERVED_RAW_DIRECTORY_NAME;
@@ -240,10 +242,10 @@ public class SimpleCopyListing extends CopyListing {
final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs();
- CopyListingFileStatus fileCopyListingStatus =
+ LinkedList<CopyListingFileStatus> fileCopyListingStatus =
DistCpUtils.toCopyListingFileStatus(sourceFS, fileStatus,
- preserveAcls, preserveXAttrs, preserveRawXAttrs);
-
+ preserveAcls, preserveXAttrs, preserveRawXAttrs,
+ options.getBlocksPerChunk());
writeToFileListingRoot(fileListWriter, fileCopyListingStatus,
sourceRoot, options);
}
@@ -348,9 +350,10 @@ public class SimpleCopyListing extends CopyListing {
FileStatus[] sourceFiles = sourceFS.listStatus(path);
boolean explore = (sourceFiles != null && sourceFiles.length > 0);
if (!explore || rootStatus.isDirectory()) {
- CopyListingFileStatus rootCopyListingStatus =
- DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus,
- preserveAcls, preserveXAttrs, preserveRawXAttrs);
+ LinkedList<CopyListingFileStatus> rootCopyListingStatus =
+ DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus,
+ preserveAcls, preserveXAttrs, preserveRawXAttrs,
+ options.getBlocksPerChunk());
writeToFileListingRoot(fileListWriter, rootCopyListingStatus,
sourcePathRoot, options);
}
@@ -360,20 +363,20 @@ public class SimpleCopyListing extends CopyListing {
if (LOG.isDebugEnabled()) {
LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy.");
}
- CopyListingFileStatus sourceCopyListingStatus =
- DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus,
- preserveAcls && sourceStatus.isDirectory(),
- preserveXAttrs && sourceStatus.isDirectory(),
- preserveRawXAttrs && sourceStatus.isDirectory());
- if (randomizeFileListing) {
- addToFileListing(statusList,
- new FileStatusInfo(sourceCopyListingStatus, sourcePathRoot),
- fileListWriter);
- } else {
- writeToFileListing(fileListWriter, sourceCopyListingStatus,
- sourcePathRoot);
+ LinkedList<CopyListingFileStatus> sourceCopyListingStatus =
+ DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus,
+ preserveAcls && sourceStatus.isDirectory(),
+ preserveXAttrs && sourceStatus.isDirectory(),
+ preserveRawXAttrs && sourceStatus.isDirectory(),
+ options.getBlocksPerChunk());
+ for (CopyListingFileStatus fs : sourceCopyListingStatus) {
+ if (randomizeFileListing) {
+ addToFileListing(statusList,
+ new FileStatusInfo(fs, sourcePathRoot), fileListWriter);
+ } else {
+ writeToFileListing(fileListWriter, fs, sourcePathRoot);
+ }
}
-
if (sourceStatus.isDirectory()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding source dir for traverse: " + sourceStatus.getPath());
@@ -641,18 +644,20 @@ public class SimpleCopyListing extends CopyListing {
LOG.debug("Recording source-path: " + child.getPath() + " for copy.");
}
if (workResult.getSuccess()) {
- CopyListingFileStatus childCopyListingStatus =
+ LinkedList<CopyListingFileStatus> childCopyListingStatus =
DistCpUtils.toCopyListingFileStatus(sourceFS, child,
preserveAcls && child.isDirectory(),
preserveXAttrs && child.isDirectory(),
- preserveRawXattrs && child.isDirectory());
- if (randomizeFileListing) {
- addToFileListing(fileStatuses,
- new FileStatusInfo(childCopyListingStatus, sourcePathRoot),
- fileListWriter);
- } else {
- writeToFileListing(fileListWriter, childCopyListingStatus,
- sourcePathRoot);
+ preserveRawXattrs && child.isDirectory(),
+ options.getBlocksPerChunk());
+
+ for (CopyListingFileStatus fs : childCopyListingStatus) {
+ if (randomizeFileListing) {
+ addToFileListing(fileStatuses,
+ new FileStatusInfo(fs, sourcePathRoot), fileListWriter);
+ } else {
+ writeToFileListing(fileListWriter, fs, sourcePathRoot);
+ }
}
}
if (retry < maxRetries) {
@@ -675,19 +680,21 @@ public class SimpleCopyListing extends CopyListing {
}
private void writeToFileListingRoot(SequenceFile.Writer fileListWriter,
- CopyListingFileStatus fileStatus, Path sourcePathRoot,
+ LinkedList<CopyListingFileStatus> fileStatus, Path sourcePathRoot,
DistCpOptions options) throws IOException {
boolean syncOrOverwrite = options.shouldSyncFolder() ||
options.shouldOverwrite();
- if (fileStatus.getPath().equals(sourcePathRoot) &&
- fileStatus.isDirectory() && syncOrOverwrite) {
- // Skip the root-paths when syncOrOverwrite
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skip " + fileStatus.getPath());
- }
- return;
+ for (CopyListingFileStatus fs : fileStatus) {
+ if (fs.getPath().equals(sourcePathRoot) &&
+ fs.isDirectory() && syncOrOverwrite) {
+ // Skip the root-paths when syncOrOverwrite
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skip " + fs.getPath());
+ }
+ return;
+ }
+ writeToFileListing(fileListWriter, fs, sourcePathRoot);
}
- writeToFileListing(fileListWriter, fileStatus, sourcePathRoot);
}
private void writeToFileListing(SequenceFile.Writer fileListWriter,
@@ -707,7 +714,7 @@ public class SimpleCopyListing extends CopyListing {
fileListWriter.sync();
if (!fileStatus.isDirectory()) {
- totalBytesToCopy += fileStatus.getLen();
+ totalBytesToCopy += fileStatus.getSizeToCopy();
} else {
totalDirs++;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
index 75cefb4..6ddaab9 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -34,14 +35,17 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.tools.CopyListing;
import org.apache.hadoop.tools.CopyListingFileStatus;
import org.apache.hadoop.tools.DistCpConstants;
+import org.apache.hadoop.tools.DistCpOptionSwitch;
import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.GlobbedCopyListing;
import org.apache.hadoop.tools.util.DistCpUtils;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.LinkedList;
import java.util.List;
/**
@@ -63,7 +67,8 @@ public class CopyCommitter extends FileOutputCommitter {
private boolean syncFolder = false;
private boolean overwrite = false;
private boolean targetPathExists = true;
-
+ private boolean ignoreFailures = false;
+
/**
* Create a output committer
*
@@ -82,8 +87,13 @@ public class CopyCommitter extends FileOutputCommitter {
Configuration conf = jobContext.getConfiguration();
syncFolder = conf.getBoolean(DistCpConstants.CONF_LABEL_SYNC_FOLDERS, false);
overwrite = conf.getBoolean(DistCpConstants.CONF_LABEL_OVERWRITE, false);
- targetPathExists = conf.getBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true);
-
+ targetPathExists = conf.getBoolean(
+ DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true);
+ ignoreFailures = conf.getBoolean(
+ DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
+
+ concatFileChunks(conf);
+
super.commitJob(jobContext);
cleanupTempFiles(jobContext);
@@ -169,9 +179,112 @@ public class CopyCommitter extends FileOutputCommitter {
}
}
+ private boolean isFileNotFoundException(IOException e) {
+ if (e instanceof FileNotFoundException) {
+ return true;
+ }
+
+ if (e instanceof RemoteException) {
+ return ((RemoteException)e).unwrapRemoteException()
+ instanceof FileNotFoundException;
+ }
+
+ return false;
+ }
+
+ /**
+ * Concat chunk files for the same file into one.
+ * Iterate through copy listing, identify chunk files for the same file,
+ * concat them into one.
+ */
+ private void concatFileChunks(Configuration conf) throws IOException {
+
+ LOG.info("concat file chunks ...");
+
+ String spath = conf.get(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH);
+ if (spath == null || spath.isEmpty()) {
+ return;
+ }
+ Path sourceListing = new Path(spath);
+ SequenceFile.Reader sourceReader = new SequenceFile.Reader(conf,
+ SequenceFile.Reader.file(sourceListing));
+ Path targetRoot =
+ new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
+
+ try {
+ CopyListingFileStatus srcFileStatus = new CopyListingFileStatus();
+ Text srcRelPath = new Text();
+ CopyListingFileStatus lastFileStatus = null;
+ LinkedList<Path> allChunkPaths = new LinkedList<Path>();
+
+ // Iterate over every source path that was copied.
+ while (sourceReader.next(srcRelPath, srcFileStatus)) {
+ if (srcFileStatus.isDirectory()) {
+ continue;
+ }
+ Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath);
+ Path targetFileChunkPath =
+ DistCpUtils.getSplitChunkPath(targetFile, srcFileStatus);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" add " + targetFileChunkPath + " to concat.");
+ }
+ allChunkPaths.add(targetFileChunkPath);
+ if (srcFileStatus.getChunkOffset() + srcFileStatus.getChunkLength()
+ == srcFileStatus.getLen()) {
+ // This is the last chunk of the splits, consolidate allChunkPaths
+ try {
+ concatFileChunks(conf, targetFile, allChunkPaths);
+ } catch (IOException e) {
+ // If the concat failed because a chunk file doesn't exist,
+ // then we assume that the CopyMapper has skipped copying this
+ // file, and we ignore the exception here.
+ // If a chunk file should have been created but it was not, then
+ // the CopyMapper would have failed.
+ if (!isFileNotFoundException(e)) {
+ String emsg = "Failed to concat chunk files for " + targetFile;
+ if (!ignoreFailures) {
+ throw new IOException(emsg, e);
+ } else {
+ LOG.warn(emsg, e);
+ }
+ }
+ }
+ allChunkPaths.clear();
+ lastFileStatus = null;
+ } else {
+ if (lastFileStatus == null) {
+ lastFileStatus = new CopyListingFileStatus(srcFileStatus);
+ } else {
+ // Two neighboring chunks have to be consecutive ones for the same
+ // file, for them to be merged
+ if (!srcFileStatus.getPath().equals(lastFileStatus.getPath()) ||
+ srcFileStatus.getChunkOffset() !=
+ (lastFileStatus.getChunkOffset() +
+ lastFileStatus.getChunkLength())) {
+ String emsg = "Inconsistent sequence file: current " +
+ "chunk file " + srcFileStatus + " doesnt match prior " +
+ "entry " + lastFileStatus;
+ if (!ignoreFailures) {
+ throw new IOException(emsg);
+ } else {
+ LOG.warn(emsg + ", skipping concat this set.");
+ }
+ } else {
+ lastFileStatus.setChunkOffset(srcFileStatus.getChunkOffset());
+ lastFileStatus.setChunkLength(srcFileStatus.getChunkLength());
+ }
+ }
+ }
+ }
+ } finally {
+ IOUtils.closeStream(sourceReader);
+ }
+ }
+
// This method changes the target-directories' file-attributes (owner,
// user/group permissions, etc.) based on the corresponding source directories.
- private void preserveFileAttributesForDirectories(Configuration conf) throws IOException {
+ private void preserveFileAttributesForDirectories(Configuration conf)
+ throws IOException {
String attrSymbols = conf.get(DistCpConstants.CONF_LABEL_PRESERVE_STATUS);
final boolean syncOrOverwrite = syncFolder || overwrite;
@@ -325,4 +438,57 @@ public class CopyCommitter extends FileOutputCommitter {
", Unable to move to " + finalDir);
}
}
+
+ /**
+ * Concat the passed chunk files into one and rename it the targetFile.
+ */
+ private void concatFileChunks(Configuration conf, Path targetFile,
+ LinkedList<Path> allChunkPaths) throws IOException {
+ if (allChunkPaths.size() == 1) {
+ return;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("concat " + targetFile + " allChunkSize+ "
+ + allChunkPaths.size());
+ }
+ FileSystem dstfs = targetFile.getFileSystem(conf);
+
+ Path firstChunkFile = allChunkPaths.removeFirst();
+ Path[] restChunkFiles = new Path[allChunkPaths.size()];
+ allChunkPaths.toArray(restChunkFiles);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("concat: firstchunk: " + dstfs.getFileStatus(firstChunkFile));
+ int i = 0;
+ for (Path f : restChunkFiles) {
+ LOG.debug("concat: other chunk: " + i + ": " + dstfs.getFileStatus(f));
+ ++i;
+ }
+ }
+ dstfs.concat(firstChunkFile, restChunkFiles);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("concat: result: " + dstfs.getFileStatus(firstChunkFile));
+ }
+ rename(dstfs, firstChunkFile, targetFile);
+ }
+
+ /**
+ * Rename tmp to dst on destFileSys.
+ * @param destFileSys the file ssystem
+ * @param tmp the source path
+ * @param dst the destination path
+ * @throws IOException if renaming failed
+ */
+ private static void rename(FileSystem destFileSys, Path tmp, Path dst)
+ throws IOException {
+ try {
+ if (destFileSys.exists(dst)) {
+ destFileSys.delete(dst, true);
+ }
+ destFileSys.rename(tmp, dst);
+ } catch (IOException ioe) {
+ throw new IOException("Fail to rename tmp file (=" + tmp
+ + ") to destination file (=" + dst + ")", ioe);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
index e1873f1..d6b3ba8 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
@@ -156,10 +156,12 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
sourceFS = sourcePath.getFileSystem(conf);
final boolean preserveXAttrs =
fileAttributes.contains(FileAttribute.XATTR);
- sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS,
- sourceFS.getFileStatus(sourcePath),
- fileAttributes.contains(FileAttribute.ACL),
- preserveXAttrs, preserveRawXattrs);
+ sourceCurrStatus = DistCpUtils.toCopyListingFileStatusHelper(sourceFS,
+ sourceFS.getFileStatus(sourcePath),
+ fileAttributes.contains(FileAttribute.ACL),
+ preserveXAttrs, preserveRawXattrs,
+ sourceFileStatus.getChunkOffset(),
+ sourceFileStatus.getChunkLength());
} catch (FileNotFoundException e) {
throw new IOException(new RetriableFileCopyCommand.CopyReadException(e));
}
@@ -173,7 +175,8 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
LOG.debug("Path could not be found: " + target, ignore);
}
- if (targetStatus != null && (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) {
+ if (targetStatus != null &&
+ (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) {
throw new IOException("Can't replace " + target + ". Target is " +
getFileType(targetStatus) + ", Source is " + getFileType(sourceCurrStatus));
}
@@ -183,19 +186,28 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
return;
}
- FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, targetStatus);
+ FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target,
+ targetStatus);
+
+ Path tmpTarget = target;
if (action == FileAction.SKIP) {
LOG.info("Skipping copy of " + sourceCurrStatus.getPath()
+ " to " + target);
updateSkipCounters(context, sourceCurrStatus);
context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath()));
+
} else {
- copyFileWithRetry(description, sourceCurrStatus, target, context,
+ if (sourceCurrStatus.isSplit()) {
+ tmpTarget = DistCpUtils.getSplitChunkPath(target, sourceCurrStatus);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("copying " + sourceCurrStatus + " " + tmpTarget);
+ }
+ copyFileWithRetry(description, sourceCurrStatus, tmpTarget, context,
action, fileAttributes);
}
-
- DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus,
- fileAttributes, preserveRawXattrs);
+ DistCpUtils.preserve(target.getFileSystem(conf), tmpTarget,
+ sourceCurrStatus, fileAttributes, preserveRawXattrs);
} catch (IOException exception) {
handleFailures(exception, sourceFileStatus, target, context);
}
@@ -261,8 +273,12 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
private void handleFailures(IOException exception,
CopyListingFileStatus sourceFileStatus, Path target, Context context)
throws IOException, InterruptedException {
- LOG.error("Failure in copying " + sourceFileStatus.getPath() + " to " +
- target, exception);
+ LOG.error("Failure in copying " + sourceFileStatus.getPath() +
+ (sourceFileStatus.isSplit()? ","
+ + " offset=" + sourceFileStatus.getChunkOffset()
+ + " chunkLength=" + sourceFileStatus.getChunkLength()
+ : "") +
+ " to " + target, exception);
if (ignoreFailures &&
ExceptionUtils.indexOfType(exception, CopyReadException.class) != -1) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
index 06acd78..2c17fef 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
@@ -118,17 +118,21 @@ public class RetriableFileCopyCommand extends RetriableCommand {
.contains(FileAttribute.CHECKSUMTYPE) ? sourceFS
.getFileChecksum(sourcePath) : null;
- final long offset = action == FileAction.APPEND ? targetFS.getFileStatus(
- target).getLen() : 0;
+ long offset = (action == FileAction.APPEND) ?
+ targetFS.getFileStatus(target).getLen() : source.getChunkOffset();
long bytesRead = copyToFile(targetPath, targetFS, source,
offset, context, fileAttributes, sourceChecksum);
- compareFileLengths(source, targetPath, configuration, bytesRead
- + offset);
+ if (!source.isSplit()) {
+ compareFileLengths(source, targetPath, configuration, bytesRead
+ + offset);
+ }
//At this point, src&dest lengths are same. if length==0, we skip checksum
if ((bytesRead != 0) && (!skipCrc)) {
- compareCheckSums(sourceFS, source.getPath(), sourceChecksum,
- targetFS, targetPath);
+ if (!source.isSplit()) {
+ compareCheckSums(sourceFS, source.getPath(), sourceChecksum,
+ targetFS, targetPath);
+ }
}
// it's not append case, thus we first write to a temporary file, rename
// it to the target path.
@@ -249,16 +253,26 @@ public class RetriableFileCopyCommand extends RetriableCommand {
ThrottledInputStream inStream = null;
long totalBytesRead = 0;
+ long chunkLength = source2.getChunkLength();
+ boolean finished = false;
try {
inStream = getInputStream(source, context.getConfiguration());
int bytesRead = readBytes(inStream, buf, sourceOffset);
while (bytesRead >= 0) {
+ if (chunkLength > 0 &&
+ (totalBytesRead + bytesRead) >= chunkLength) {
+ bytesRead = (int)(chunkLength - totalBytesRead);
+ finished = true;
+ }
totalBytesRead += bytesRead;
if (action == FileAction.APPEND) {
sourceOffset += bytesRead;
}
outStream.write(buf, 0, bytesRead);
updateContextStatus(totalBytesRead, context, source2);
+ if (finished) {
+ break;
+ }
bytesRead = readBytes(inStream, buf, sourceOffset);
}
outStream.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
index 3e86d09..d1c18ea 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
@@ -99,7 +99,8 @@ public class UniformSizeInputFormat
while (reader.next(srcRelPath, srcFileStatus)) {
// If adding the current file would cause the bytes per map to exceed
// limit. Add the current file to new split
- if (currentSplitSize + srcFileStatus.getLen() > nBytesPerSplit && lastPosition != 0) {
+ if (currentSplitSize + srcFileStatus.getChunkLength() > nBytesPerSplit
+ && lastPosition != 0) {
FileSplit split = new FileSplit(listingFilePath, lastSplitStart,
lastPosition - lastSplitStart, null);
if (LOG.isDebugEnabled()) {
@@ -109,7 +110,7 @@ public class UniformSizeInputFormat
lastSplitStart = lastPosition;
currentSplitSize = 0;
}
- currentSplitSize += srcFileStatus.getLen();
+ currentSplitSize += srcFileStatus.getChunkLength();
lastPosition = reader.getPosition();
}
if (lastPosition > lastSplitStart) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
index 76bc4c5..e315b84 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
@@ -19,9 +19,11 @@
package org.apache.hadoop.tools.util;
import com.google.common.collect.Maps;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -30,6 +32,7 @@ import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputFormat;
@@ -44,6 +47,7 @@ import org.apache.hadoop.util.StringUtils;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.EnumSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -116,7 +120,7 @@ public class DistCpUtils {
* @return Class implementing the strategy specified in options.
*/
public static Class<? extends InputFormat> getStrategy(Configuration conf,
- DistCpOptions options) {
+ DistCpOptions options) {
String confLabel = "distcp."
+ StringUtils.toLowerCase(options.getCopyStrategy())
+ ".strategy" + ".impl";
@@ -298,6 +302,86 @@ public class DistCpUtils {
}
/**
+ * Converts FileStatus to a list of CopyListingFileStatus.
+ * The resulted list contains either one CopyListingFileStatus per chunk of
+ * file-blocks (if file-size exceeds blockSize * blocksPerChunk, and there
+ * are more blocks in the file than blocksperChunk), or a single
+ * CopyListingFileStatus for the entire file (if file-size is too small to
+ * split).
+ * If preserving ACLs, populates the CopyListingFileStatus with the ACLs.
+ * If preserving XAttrs, populates the CopyListingFileStatus with the XAttrs.
+ *
+ * @param fileSystem FileSystem containing the file
+ * @param fileStatus FileStatus of file
+ * @param preserveAcls boolean true if preserving ACLs
+ * @param preserveXAttrs boolean true if preserving XAttrs
+ * @param preserveRawXAttrs boolean true if preserving raw.* XAttrs
+ * @param blocksPerChunk size of chunks when copying chunks in parallel
+ * @return list of CopyListingFileStatus
+ * @throws IOException if there is an I/O error
+ */
+ public static LinkedList<CopyListingFileStatus> toCopyListingFileStatus(
+ FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls,
+ boolean preserveXAttrs, boolean preserveRawXAttrs, int blocksPerChunk)
+ throws IOException {
+ LinkedList<CopyListingFileStatus> copyListingFileStatus =
+ new LinkedList<CopyListingFileStatus>();
+
+ final CopyListingFileStatus clfs = toCopyListingFileStatusHelper(
+ fileSystem, fileStatus, preserveAcls,
+ preserveXAttrs, preserveRawXAttrs,
+ 0, fileStatus.getLen());
+ final long blockSize = fileStatus.getBlockSize();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("toCopyListing: " + fileStatus + " chunkSize: "
+ + blocksPerChunk + " isDFS: " +
+ (fileSystem instanceof DistributedFileSystem));
+ }
+ if ((blocksPerChunk > 0) &&
+ !fileStatus.isDirectory() &&
+ (fileStatus.getLen() > blockSize * blocksPerChunk)) {
+ // split only when the file size is larger than the intended chunk size
+ final BlockLocation[] blockLocations;
+ blockLocations = fileSystem.getFileBlockLocations(fileStatus, 0,
+ fileStatus.getLen());
+
+ int numBlocks = blockLocations.length;
+ long curPos = 0;
+ if (numBlocks <= blocksPerChunk) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" add file " + clfs);
+ }
+ copyListingFileStatus.add(clfs);
+ } else {
+ int i = 0;
+ while (i < numBlocks) {
+ long curLength = 0;
+ for (int j = 0; j < blocksPerChunk && i < numBlocks; ++j, ++i) {
+ curLength += blockLocations[i].getLength();
+ }
+ if (curLength > 0) {
+ CopyListingFileStatus clfs1 = new CopyListingFileStatus(clfs);
+ clfs1.setChunkOffset(curPos);
+ clfs1.setChunkLength(curLength);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" add file chunk " + clfs1);
+ }
+ copyListingFileStatus.add(clfs1);
+ curPos += curLength;
+ }
+ }
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" add file/dir " + clfs);
+ }
+ copyListingFileStatus.add(clfs);
+ }
+
+ return copyListingFileStatus;
+ }
+
+ /**
* Converts a FileStatus to a CopyListingFileStatus. If preserving ACLs,
* populates the CopyListingFileStatus with the ACLs. If preserving XAttrs,
* populates the CopyListingFileStatus with the XAttrs.
@@ -307,13 +391,17 @@ public class DistCpUtils {
* @param preserveAcls boolean true if preserving ACLs
* @param preserveXAttrs boolean true if preserving XAttrs
* @param preserveRawXAttrs boolean true if preserving raw.* XAttrs
+ * @param chunkOffset chunk offset in bytes
+ * @param chunkLength chunk length in bytes
+ * @return CopyListingFileStatus
* @throws IOException if there is an I/O error
*/
- public static CopyListingFileStatus toCopyListingFileStatus(
+ public static CopyListingFileStatus toCopyListingFileStatusHelper(
FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls,
- boolean preserveXAttrs, boolean preserveRawXAttrs) throws IOException {
+ boolean preserveXAttrs, boolean preserveRawXAttrs,
+ long chunkOffset, long chunkLength) throws IOException {
CopyListingFileStatus copyListingFileStatus =
- new CopyListingFileStatus(fileStatus);
+ new CopyListingFileStatus(fileStatus, chunkOffset, chunkLength);
if (preserveAcls) {
FsPermission perm = fileStatus.getPermission();
if (perm.getAclBit()) {
@@ -470,4 +558,19 @@ public class DistCpUtils {
return (sourceChecksum == null || targetChecksum == null ||
sourceChecksum.equals(targetChecksum));
}
+
+ /*
+ * Return the Path for a given chunk.
+ * Used when splitting large file into chunks to copy in parallel.
+ * @param targetFile path to target file
+ * @param srcFileStatus source file status in copy listing
+ * @return path to the chunk specified by the parameters to store
+ * in target cluster temporarily
+ */
+ public static Path getSplitChunkPath(Path targetFile,
+ CopyListingFileStatus srcFileStatus) {
+ return new Path(targetFile.toString()
+ + ".____distcpSplit____" + srcFileStatus.getChunkOffset()
+ + "." + srcFileStatus.getChunkLength());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
index 41a6e94..a77deb2 100644
--- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
+++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
@@ -237,6 +237,7 @@ Flag | Description | Notes
`-rdiff <newSnapshot> <oldSnapshot>` | Use snapshot diff report between given two snapshots to identify what has been changed on the target since the snapshot `<oldSnapshot>` was created on the target, and apply the diff reversely to the target, and copy modified files from the source's `<oldSnapshot>`, to make the target the same as `<oldSnapshot>`. | This option is valid only with `-update` option and the following conditions should be satisfied. <ol><li>Both the source and the target FileSystem must be DistributedFileSystem. The source and the target can be two different clusters/paths, or they can be exactly the same cluster/path. In the latter case, modified files are copied from target's `<oldSnapshot>` to target's current state).</li> <li> Two snapshots `<newSnapshot>` and `<oldSnapshot>` have been created on the target FS, and `<oldSnapshot>` is older than `<newSnapshot>`. No change has been made on target since `<newSnapshot>` was created on the target. </li> <li> The sour
ce has the same snapshot `<oldSnapshot>`, which has the same content as the `<oldSnapshot>` on the target. All the files/directories in the target's `<oldSnapshot>` are the same with source's `<oldSnapshot>`.</li> </ol> |
`-numListstatusThreads` | Number of threads to use for building file listing | At most 40 threads.
`-skipcrccheck` | Whether to skip CRC checks between source and target paths. |
+`-blocksperchunk <blocksperchunk>` | Number of blocks per chunk. When specified, split files into chunks to copy in parallel | If set to a positive value, files with more blocks than this value will be split into chunks of `<blocksperchunk>` blocks to be transferred in parallel, and reassembled on the destination. By default, `<blocksperchunk>` is 0 and the files will be transmitted in their entirety without splitting. This switch is only applicable when the source file system implements getBlockLocations method and the target file system implements concat method. |
Architecture of DistCp
----------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
index e3018a0..b2266b3 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
@@ -23,17 +23,27 @@ import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.OutputStream;
+import java.io.PrintStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
+import java.util.Random;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.util.ToolRunner;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -47,11 +57,15 @@ import org.junit.rules.Timeout;
*/
public class TestDistCpSystem {
+ private static final Log LOG =
+ LogFactory.getLog(TestDistCpSystem.class);
+
@Rule
public Timeout globalTimeout = new Timeout(30000);
private static final String SRCDAT = "srcdat";
private static final String DSTDAT = "dstdat";
+ private static final long BLOCK_SIZE = 1024;
private static MiniDFSCluster cluster;
private static Configuration conf;
@@ -63,27 +77,76 @@ public class TestDistCpSystem {
this.path = path;
this.isDir = isDir;
}
- String getPath() { return path; }
- boolean isDirectory() { return isDir; }
+
+ String getPath() {
+ return path;
+ }
+
+ boolean isDirectory() {
+ return isDir;
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws IOException {
+ conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+ cluster.waitActive();
+ }
+
+ @AfterClass
+ public static void afterClass() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ static String execCmd(FsShell shell, String... args) throws Exception {
+ ByteArrayOutputStream baout = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(baout, true);
+ PrintStream old = System.out;
+ System.setOut(out);
+ shell.run(args);
+ out.close();
+ System.setOut(old);
+ return baout.toString();
}
- private void createFiles(FileSystem fs, String topdir,
- FileEntry[] entries) throws IOException {
+ private void createFiles(DistributedFileSystem fs, String topdir,
+ FileEntry[] entries, long chunkSize) throws IOException {
+ long seed = System.currentTimeMillis();
+ Random rand = new Random(seed);
+ short replicationFactor = 2;
for (FileEntry entry : entries) {
- Path newpath = new Path(topdir + "/" + entry.getPath());
+ Path newPath = new Path(topdir + "/" + entry.getPath());
if (entry.isDirectory()) {
- fs.mkdirs(newpath);
+ fs.mkdirs(newPath);
} else {
- OutputStream out = fs.create(newpath);
- try {
- out.write((topdir + "/" + entry).getBytes());
- out.write("\n".getBytes());
- } finally {
- out.close();
+ long fileSize = BLOCK_SIZE *100;
+ int bufSize = 128;
+ if (chunkSize == -1) {
+ DFSTestUtil.createFile(fs, newPath, bufSize,
+ fileSize, BLOCK_SIZE, replicationFactor, seed);
+ } else {
+ // Create a variable length block file, by creating
+ // one block of half block size at the chunk boundary
+ long seg1 = chunkSize * BLOCK_SIZE - BLOCK_SIZE / 2;
+ long seg2 = fileSize - seg1;
+ DFSTestUtil.createFile(fs, newPath, bufSize,
+ seg1, BLOCK_SIZE, replicationFactor, seed);
+ DFSTestUtil.appendFileNewBlock(fs, newPath, (int)seg2);
}
}
+ seed = System.currentTimeMillis() + rand.nextLong();
}
}
+
+ private void createFiles(DistributedFileSystem fs, String topdir,
+ FileEntry[] entries) throws IOException {
+ createFiles(fs, topdir, entries, -1);
+ }
private static FileStatus[] getFileStatus(FileSystem fs,
String topdir, FileEntry[] files) throws IOException {
@@ -104,18 +167,19 @@ public class TestDistCpSystem {
}
private void testPreserveUserHelper(String testRoot,
- FileEntry[] srcEntries,
- FileEntry[] dstEntries,
- boolean createSrcDir,
- boolean createTgtDir,
- boolean update) throws Exception {
+ FileEntry[] srcEntries,
+ FileEntry[] dstEntries,
+ boolean createSrcDir,
+ boolean createTgtDir,
+ boolean update) throws Exception {
final String testSrcRel = SRCDAT;
final String testSrc = testRoot + "/" + testSrcRel;
final String testDstRel = DSTDAT;
final String testDst = testRoot + "/" + testDstRel;
String nnUri = FileSystem.getDefaultUri(conf).toString();
- FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
+ DistributedFileSystem fs = (DistributedFileSystem)
+ FileSystem.get(URI.create(nnUri), conf);
fs.mkdirs(new Path(testRoot));
if (createSrcDir) {
fs.mkdirs(new Path(testSrc));
@@ -129,8 +193,8 @@ public class TestDistCpSystem {
for(int i = 0; i < srcEntries.length; i++) {
fs.setOwner(srcstats[i].getPath(), "u" + i, null);
}
- String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc,
- nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst};
+ String[] args = update? new String[]{"-pub", "-update", nnUri+testSrc,
+ nnUri+testDst} : new String[]{"-pub", nnUri+testSrc, nnUri+testDst};
ToolRunner.run(conf, new DistCp(), args);
@@ -145,20 +209,263 @@ public class TestDistCpSystem {
deldir(fs, testRoot);
}
- @BeforeClass
- public static void beforeClass() throws IOException {
- conf = new Configuration();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- cluster.waitActive();
+ private void compareFiles(FileSystem fs, FileStatus srcStat,
+ FileStatus dstStat) throws Exception {
+ LOG.info("Comparing " + srcStat + " and " + dstStat);
+ assertEquals(srcStat.isDirectory(), dstStat.isDirectory());
+ assertEquals(srcStat.getReplication(), dstStat.getReplication());
+ assertEquals("File POSIX permission should match",
+ srcStat.getPermission(), dstStat.getPermission());
+ assertEquals("File user ownership should match",
+ srcStat.getOwner(), dstStat.getOwner());
+ assertEquals("File group ownership should match",
+ srcStat.getGroup(), dstStat.getGroup());
+ // TODO; check ACL attributes
+
+ if (srcStat.isDirectory()) {
+ return;
+ }
+
+ assertEquals("File length should match (" + srcStat.getPath() + ")",
+ srcStat.getLen(), dstStat.getLen());
+
+ FSDataInputStream srcIn = fs.open(srcStat.getPath());
+ FSDataInputStream dstIn = fs.open(dstStat.getPath());
+ try {
+ byte[] readSrc = new byte[(int)
+ HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT];
+ byte[] readDst = new byte[(int)
+ HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT];
+
+ int srcBytesRead = 0, tgtBytesRead = 0;
+ int srcIdx = 0, tgtIdx = 0;
+ long totalComparedBytes = 0;
+ while (true) {
+ if (srcBytesRead == 0) {
+ srcBytesRead = srcIn.read(readSrc);
+ srcIdx = 0;
+ }
+ if (tgtBytesRead == 0) {
+ tgtBytesRead = dstIn.read(readDst);
+ tgtIdx = 0;
+ }
+ if (srcBytesRead == 0 || tgtBytesRead == 0) {
+ LOG.info("______ compared src and dst files for "
+ + totalComparedBytes + " bytes, content match.");
+ if (srcBytesRead != tgtBytesRead) {
+ Assert.fail("Read mismatching size, compared "
+ + totalComparedBytes + " bytes between src and dst file "
+ + srcStat + " and " + dstStat);
+ }
+ if (totalComparedBytes != srcStat.getLen()) {
+ Assert.fail("Only read/compared " + totalComparedBytes +
+ " bytes between src and dst file " + srcStat +
+ " and " + dstStat);
+ } else {
+ // success
+ break;
+ }
+ }
+ for (; srcIdx < srcBytesRead && tgtIdx < tgtBytesRead;
+ ++srcIdx, ++tgtIdx) {
+ if (readSrc[srcIdx] != readDst[tgtIdx]) {
+ Assert.fail("src and dst file does not match at "
+ + totalComparedBytes + " between "
+ + srcStat + " and " + dstStat);
+ }
+ ++totalComparedBytes;
+ }
+ LOG.info("______ compared src and dst files for "
+ + totalComparedBytes + " bytes, content match. FileLength: "
+ + srcStat.getLen());
+ if (totalComparedBytes == srcStat.getLen()) {
+ LOG.info("______ Final:" + srcIdx + " "
+ + srcBytesRead + " " + tgtIdx + " " + tgtBytesRead);
+ break;
+ }
+ if (srcIdx == srcBytesRead) {
+ srcBytesRead = 0;
+ }
+ if (tgtIdx == tgtBytesRead) {
+ tgtBytesRead = 0;
+ }
+ }
+ } finally {
+ if (srcIn != null) {
+ srcIn.close();
+ }
+ if (dstIn != null) {
+ dstIn.close();
+ }
+ }
}
- @AfterClass
- public static void afterClass() throws IOException {
- if (cluster != null) {
- cluster.shutdown();
+ // WC: needed because the current distcp does not create target dirs
+ private void createDestDir(FileSystem fs, String testDst,
+ FileStatus[] srcStats, FileEntry[] srcFiles) throws IOException {
+ fs.mkdirs(new Path(testDst));
+
+ for (int i=0; i<srcStats.length; i++) {
+ FileStatus srcStat = srcStats[i];
+ if (srcStat.isDirectory()) {
+ Path dstPath = new Path(testDst, srcFiles[i].getPath());
+ fs.mkdirs(dstPath);
+ fs.setOwner(dstPath, srcStat.getOwner(), srcStat.getGroup());
+ }
+ }
+ }
+
+ private void copyAndVerify(final DistributedFileSystem fs,
+ final FileEntry[] srcFiles, final FileStatus[] srcStats,
+ final String testDst,
+ final String[] args) throws Exception {
+ final String testRoot = "/testdir";
+ FsShell shell = new FsShell(fs.getConf());
+
+ LOG.info("ls before distcp");
+ LOG.info(execCmd(shell, "-lsr", testRoot));
+
+ LOG.info("_____ running distcp: " + args[0] + " " + args[1]);
+ ToolRunner.run(conf, new DistCp(), args);
+
+ LOG.info("ls after distcp");
+ LOG.info(execCmd(shell, "-lsr", testRoot));
+
+ FileStatus[] dstStat = getFileStatus(fs, testDst, srcFiles);
+ for (int i=0; i< dstStat.length; i++) {
+ compareFiles(fs, srcStats[i], dstStat[i]);
}
}
+ private void chunkCopy(FileEntry[] srcFiles) throws Exception {
+ final String testRoot = "/testdir";
+ final String testSrcRel = SRCDAT;
+ final String testSrc = testRoot + "/" + testSrcRel;
+ final String testDstRel = DSTDAT;
+ final String testDst = testRoot + "/" + testDstRel;
+ long chunkSize = 8;
+
+ String nnUri = FileSystem.getDefaultUri(conf).toString();
+ DistributedFileSystem fs = (DistributedFileSystem)
+ FileSystem.get(URI.create(nnUri), conf);
+
+ createFiles(fs, testRoot, srcFiles, chunkSize);
+
+ FileStatus[] srcStats = getFileStatus(fs, testRoot, srcFiles);
+ for (int i = 0; i < srcFiles.length; i++) {
+ fs.setOwner(srcStats[i].getPath(), "u" + i, "g" + i);
+ }
+ // get file status after updating owners
+ srcStats = getFileStatus(fs, testRoot, srcFiles);
+
+ createDestDir(fs, testDst, srcStats, srcFiles);
+
+ String[] args = new String[] {"-pugp", "-blocksperchunk",
+ String.valueOf(chunkSize),
+ nnUri + testSrc, nnUri + testDst};
+
+ copyAndVerify(fs, srcFiles, srcStats, testDst, args);
+ // Do it again
+ copyAndVerify(fs, srcFiles, srcStats, testDst, args);
+
+ // modify last file and rerun distcp with -update option
+ LOG.info("Modify a file and copy again");
+ for(int i=srcFiles.length-1; i >=0; --i) {
+ if (!srcFiles[i].isDirectory()) {
+ LOG.info("Modifying " + srcStats[i].getPath());
+ DFSTestUtil.appendFileNewBlock(fs, srcStats[i].getPath(),
+ (int)BLOCK_SIZE * 3);
+ break;
+ }
+ }
+ // get file status after modifying file
+ srcStats = getFileStatus(fs, testRoot, srcFiles);
+
+ args = new String[] {"-pugp", "-update", "-blocksperchunk",
+ String.valueOf(chunkSize),
+ nnUri + testSrc, nnUri + testDst + "/" + testSrcRel};
+
+ copyAndVerify(fs, srcFiles, srcStats, testDst, args);
+
+ deldir(fs, testRoot);
+ }
+
+ @Test
+ public void testRecursiveChunkCopy() throws Exception {
+ FileEntry[] srcFiles = {
+ new FileEntry(SRCDAT, true),
+ new FileEntry(SRCDAT + "/file0", false),
+ new FileEntry(SRCDAT + "/dir1", true),
+ new FileEntry(SRCDAT + "/dir2", true),
+ new FileEntry(SRCDAT + "/dir1/file1", false)
+ };
+ chunkCopy(srcFiles);
+ }
+
+ @Test
+ public void testChunkCopyOneFile() throws Exception {
+ FileEntry[] srcFiles = {
+ new FileEntry(SRCDAT, true),
+ new FileEntry(SRCDAT + "/file0", false)
+ };
+ chunkCopy(srcFiles);
+ }
+
+ @Test
+ public void testDistcpLargeFile() throws Exception {
+ FileEntry[] srcfiles = {
+ new FileEntry(SRCDAT, true),
+ new FileEntry(SRCDAT + "/file", false)
+ };
+
+ final String testRoot = "/testdir";
+ final String testSrcRel = SRCDAT;
+ final String testSrc = testRoot + "/" + testSrcRel;
+ final String testDstRel = DSTDAT;
+ final String testDst = testRoot + "/" + testDstRel;
+
+ String nnUri = FileSystem.getDefaultUri(conf).toString();
+ DistributedFileSystem fs =
+ (DistributedFileSystem) FileSystem.get(URI.create(nnUri), conf);
+ fs.mkdirs(new Path(testRoot));
+ fs.mkdirs(new Path(testSrc));
+ fs.mkdirs(new Path(testDst));
+ long chunkSize = 6;
+ createFiles(fs, testRoot, srcfiles, chunkSize);
+
+ String srcFileName = testRoot + Path.SEPARATOR + srcfiles[1].getPath();
+ Path srcfile = new Path(srcFileName);
+
+ if(!cluster.getFileSystem().exists(srcfile)){
+ throw new Exception("src not exist");
+ }
+
+ final long srcLen = fs.getFileStatus(srcfile).getLen();
+
+ FileStatus[] srcstats = getFileStatus(fs, testRoot, srcfiles);
+ for (int i = 0; i < srcfiles.length; i++) {
+ fs.setOwner(srcstats[i].getPath(), "u" + i, null);
+ }
+ String[] args = new String[] {
+ "-blocksperchunk",
+ String.valueOf(chunkSize),
+ nnUri + testSrc,
+ nnUri + testDst
+ };
+
+ LOG.info("_____ running distcp: " + args[0] + " " + args[1]);
+ ToolRunner.run(conf, new DistCp(), args);
+
+ String realTgtPath = testDst;
+ FileStatus[] dststat = getFileStatus(fs, realTgtPath, srcfiles);
+ assertEquals("File length should match", srcLen,
+ dststat[dststat.length - 1].getLen());
+
+ this.compareFiles(fs, srcstats[srcstats.length-1],
+ dststat[dststat.length-1]);
+ deldir(fs, testRoot);
+ }
+
@Test
public void testPreserveUseNonEmptyDir() throws Exception {
String testRoot = "/testdir." + getMethodName();
@@ -180,7 +487,6 @@ public class TestDistCpSystem {
testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false);
}
-
@Test
public void testPreserveUserEmptyDir() throws Exception {
String testRoot = "/testdir." + getMethodName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index efe4627..f94ba97 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -394,7 +394,7 @@ public class TestOptionsParser {
+ "copyStrategy='uniformsize', preserveStatus=[], "
+ "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, "
+ "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, "
- + "targetPathExists=true, filtersFile='null'}";
+ + "targetPathExists=true, filtersFile='null', blocksPerChunk=0}";
String optionString = option.toString();
Assert.assertEquals(val, optionString);
Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/064c8b25/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
index 2e9a350..2452d6f 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
@@ -81,6 +81,10 @@ public class TestCopyCommitter {
@Before
public void createMetaFolder() {
config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta");
+ // Unset listing file path since the config is shared by
+ // multiple tests, and some test doesn't set it, such as
+ // testNoCommitAction, but the distcp code will check it.
+ config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, "");
Path meta = new Path("/meta");
try {
cluster.getFileSystem().mkdirs(meta);
@@ -326,7 +330,6 @@ public class TestCopyCommitter {
committer.commitJob(jobContext);
Assert.assertFalse(fs.exists(new Path(workPath)));
Assert.assertTrue(fs.exists(new Path(finalPath)));
-
} catch (IOException e) {
LOG.error("Exception encountered while testing for preserve status", e);
Assert.fail("Atomic commit failure");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-5654. Not be able to run SLS
with FairScheduler (yufeigu via rkanter)
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
index 8645a69..a8792e8 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
@@ -18,40 +18,170 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.Locale;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.Lock;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.CsvReporter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SlidingWindowReservoir;
+import com.codahale.metrics.Timer;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.MetricRegistry;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
+import org.apache.hadoop.yarn.sls.web.SLSWebApp;
+import org.apache.log4j.Logger;
@Private
@Unstable
public abstract class SchedulerMetrics {
+ private static final String EOL = System.getProperty("line.separator");
+ private static final int SAMPLING_SIZE = 60;
+ private static final Logger LOG = Logger.getLogger(SchedulerMetrics.class);
+
protected ResourceScheduler scheduler;
protected Set<String> trackedQueues;
protected MetricRegistry metrics;
protected Set<String> appTrackedMetrics;
protected Set<String> queueTrackedMetrics;
-
+
+ private Configuration conf;
+ private ScheduledExecutorService pool;
+ private SLSWebApp web;
+
+ // metrics
+ private String metricsOutputDir;
+ private BufferedWriter metricsLogBW;
+ private BufferedWriter jobRuntimeLogBW;
+ private boolean running = false;
+
+ // counters for scheduler allocate/handle operations
+ private Counter schedulerAllocateCounter;
+ private Counter schedulerHandleCounter;
+ private Map<SchedulerEventType, Counter> schedulerHandleCounterMap;
+
+ // Timers for scheduler allocate/handle operations
+ private Timer schedulerAllocateTimer;
+ private Timer schedulerHandleTimer;
+ private Map<SchedulerEventType, Timer> schedulerHandleTimerMap;
+ private List<Histogram> schedulerHistogramList;
+ private Map<Histogram, Timer> histogramTimerMap;
+ private Lock samplerLock;
+ private Lock queueLock;
+
+ static Class getSchedulerMetricsClass(Configuration conf,
+ Class schedulerClass) throws ClassNotFoundException {
+ Class metricClass = null;
+ String schedulerMetricsType = conf.get(schedulerClass.getName());
+ if (schedulerMetricsType != null) {
+ metricClass = Class.forName(schedulerMetricsType);
+ }
+
+ if (schedulerClass.equals(FairScheduler.class)) {
+ metricClass = FairSchedulerMetrics.class;
+ } else if (schedulerClass.equals(CapacityScheduler.class)) {
+ metricClass = CapacitySchedulerMetrics.class;
+ } else if (schedulerClass.equals(FifoScheduler.class)) {
+ metricClass = FifoSchedulerMetrics.class;
+ }
+
+ return metricClass;
+ }
+
+ static SchedulerMetrics getInstance(Configuration conf, Class schedulerClass)
+ throws ClassNotFoundException {
+ Class schedulerMetricClass = getSchedulerMetricsClass(conf, schedulerClass);
+ return (SchedulerMetrics) ReflectionUtils
+ .newInstance(schedulerMetricClass, new Configuration());
+ }
+
public SchedulerMetrics() {
+ metrics = new MetricRegistry();
+
appTrackedMetrics = new HashSet<>();
appTrackedMetrics.add("live.containers");
appTrackedMetrics.add("reserved.containers");
+
queueTrackedMetrics = new HashSet<>();
+ trackedQueues = new HashSet<>();
+
+ samplerLock = new ReentrantLock();
+ queueLock = new ReentrantLock();
}
-
- public void init(ResourceScheduler scheduler, MetricRegistry metrics) {
- this.scheduler = scheduler;
- this.trackedQueues = new HashSet<>();
- this.metrics = metrics;
+
+ void init(ResourceScheduler resourceScheduler, Configuration config)
+ throws Exception {
+ this.scheduler = resourceScheduler;
+ this.conf = config;
+
+ metricsOutputDir = conf.get(SLSConfiguration.METRICS_OUTPUT_DIR);
+
+ // register various metrics
+ registerJvmMetrics();
+ registerClusterResourceMetrics();
+ registerContainerAppNumMetrics();
+ registerSchedulerMetrics();
+
+ // .csv output
+ initMetricsCSVOutput();
+
+ // start web app to provide real-time tracking
+ int metricsWebAddressPort = conf.getInt(
+ SLSConfiguration.METRICS_WEB_ADDRESS_PORT,
+ SLSConfiguration.METRICS_WEB_ADDRESS_PORT_DEFAULT);
+ web = new SLSWebApp((SchedulerWrapper)scheduler, metricsWebAddressPort);
+ web.start();
+
+ // a thread to update histogram timer
+ pool = new ScheduledThreadPoolExecutor(2);
+ pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000,
+ TimeUnit.MILLISECONDS);
+
+ // a thread to output metrics for real-tiem tracking
+ pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000,
+ TimeUnit.MILLISECONDS);
+
+ // application running information
+ jobRuntimeLogBW =
+ new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
+ metricsOutputDir + "/jobruntime.csv"), "UTF-8"));
+ jobRuntimeLogBW.write("JobID,real_start_time,real_end_time," +
+ "simulate_start_time,simulate_end_time" + EOL);
+ jobRuntimeLogBW.flush();
+ }
+
+ public MetricRegistry getMetrics() {
+ return metrics;
}
protected SchedulerApplicationAttempt getSchedulerAppAttempt(
@@ -117,7 +247,392 @@ public abstract class SchedulerMetrics {
public Set<String> getAppTrackedMetrics() {
return appTrackedMetrics;
}
+
public Set<String> getQueueTrackedMetrics() {
return queueTrackedMetrics;
}
+
+ private void registerJvmMetrics() {
+ // add JVM gauges
+ metrics.register("variable.jvm.free.memory",
+ new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return Runtime.getRuntime().freeMemory();
+ }
+ }
+ );
+ metrics.register("variable.jvm.max.memory",
+ new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return Runtime.getRuntime().maxMemory();
+ }
+ }
+ );
+ metrics.register("variable.jvm.total.memory",
+ new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return Runtime.getRuntime().totalMemory();
+ }
+ }
+ );
+ }
+
+ private void registerClusterResourceMetrics() {
+ metrics.register("variable.cluster.allocated.memory",
+ new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ if (scheduler.getRootQueueMetrics() == null) {
+ return 0L;
+ } else {
+ return scheduler.getRootQueueMetrics().getAllocatedMB();
+ }
+ }
+ }
+ );
+ metrics.register("variable.cluster.allocated.vcores",
+ new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ if (scheduler.getRootQueueMetrics() == null) {
+ return 0;
+ } else {
+ return scheduler.getRootQueueMetrics().getAllocatedVirtualCores();
+ }
+ }
+ }
+ );
+ metrics.register("variable.cluster.available.memory",
+ new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ if (scheduler.getRootQueueMetrics() == null) {
+ return 0L;
+ } else {
+ return scheduler.getRootQueueMetrics().getAvailableMB();
+ }
+ }
+ }
+ );
+ metrics.register("variable.cluster.available.vcores",
+ new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ if (scheduler.getRootQueueMetrics() == null) {
+ return 0;
+ } else {
+ return scheduler.getRootQueueMetrics().getAvailableVirtualCores();
+ }
+ }
+ }
+ );
+ }
+
+ private void registerContainerAppNumMetrics() {
+ metrics.register("variable.running.application",
+ new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ if (scheduler.getRootQueueMetrics() == null) {
+ return 0;
+ } else {
+ return scheduler.getRootQueueMetrics().getAppsRunning();
+ }
+ }
+ }
+ );
+ metrics.register("variable.running.container",
+ new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ if (scheduler.getRootQueueMetrics() == null) {
+ return 0;
+ } else {
+ return scheduler.getRootQueueMetrics().getAllocatedContainers();
+ }
+ }
+ }
+ );
+ }
+
+ private void registerSchedulerMetrics() {
+ samplerLock.lock();
+ try {
+ // counters for scheduler operations
+ schedulerAllocateCounter = metrics.counter(
+ "counter.scheduler.operation.allocate");
+ schedulerHandleCounter = metrics.counter(
+ "counter.scheduler.operation.handle");
+ schedulerHandleCounterMap = new HashMap<>();
+ for (SchedulerEventType e : SchedulerEventType.values()) {
+ Counter counter = metrics.counter(
+ "counter.scheduler.operation.handle." + e);
+ schedulerHandleCounterMap.put(e, counter);
+ }
+ // timers for scheduler operations
+ int timeWindowSize = conf.getInt(
+ SLSConfiguration.METRICS_TIMER_WINDOW_SIZE,
+ SLSConfiguration.METRICS_TIMER_WINDOW_SIZE_DEFAULT);
+ schedulerAllocateTimer = new Timer(
+ new SlidingWindowReservoir(timeWindowSize));
+ schedulerHandleTimer = new Timer(
+ new SlidingWindowReservoir(timeWindowSize));
+ schedulerHandleTimerMap = new HashMap<>();
+ for (SchedulerEventType e : SchedulerEventType.values()) {
+ Timer timer = new Timer(new SlidingWindowReservoir(timeWindowSize));
+ schedulerHandleTimerMap.put(e, timer);
+ }
+ // histogram for scheduler operations (Samplers)
+ schedulerHistogramList = new ArrayList<>();
+ histogramTimerMap = new HashMap<>();
+ Histogram schedulerAllocateHistogram = new Histogram(
+ new SlidingWindowReservoir(SAMPLING_SIZE));
+ metrics.register("sampler.scheduler.operation.allocate.timecost",
+ schedulerAllocateHistogram);
+ schedulerHistogramList.add(schedulerAllocateHistogram);
+ histogramTimerMap.put(schedulerAllocateHistogram, schedulerAllocateTimer);
+ Histogram schedulerHandleHistogram = new Histogram(
+ new SlidingWindowReservoir(SAMPLING_SIZE));
+ metrics.register("sampler.scheduler.operation.handle.timecost",
+ schedulerHandleHistogram);
+ schedulerHistogramList.add(schedulerHandleHistogram);
+ histogramTimerMap.put(schedulerHandleHistogram, schedulerHandleTimer);
+ for (SchedulerEventType e : SchedulerEventType.values()) {
+ Histogram histogram = new Histogram(
+ new SlidingWindowReservoir(SAMPLING_SIZE));
+ metrics.register(
+ "sampler.scheduler.operation.handle." + e + ".timecost",
+ histogram);
+ schedulerHistogramList.add(histogram);
+ histogramTimerMap.put(histogram, schedulerHandleTimerMap.get(e));
+ }
+ } finally {
+ samplerLock.unlock();
+ }
+ }
+
+ private void initMetricsCSVOutput() {
+ int timeIntervalMS = conf.getInt(
+ SLSConfiguration.METRICS_RECORD_INTERVAL_MS,
+ SLSConfiguration.METRICS_RECORD_INTERVAL_MS_DEFAULT);
+ File dir = new File(metricsOutputDir + "/metrics");
+ if(!dir.exists() && !dir.mkdirs()) {
+ LOG.error("Cannot create directory " + dir.getAbsoluteFile());
+ }
+ final CsvReporter reporter = CsvReporter.forRegistry(metrics)
+ .formatFor(Locale.US)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build(new File(metricsOutputDir + "/metrics"));
+ reporter.start(timeIntervalMS, TimeUnit.MILLISECONDS);
+ }
+
+ boolean isRunning() {
+ return running;
+ }
+
+ void setRunning(boolean running) {
+ this.running = running;
+ }
+
+ class HistogramsRunnable implements Runnable {
+ @Override
+ public void run() {
+ samplerLock.lock();
+ try {
+ for (Histogram histogram : schedulerHistogramList) {
+ Timer timer = histogramTimerMap.get(histogram);
+ histogram.update((int) timer.getSnapshot().getMean());
+ }
+ } finally {
+ samplerLock.unlock();
+ }
+ }
+ }
+
+ class MetricsLogRunnable implements Runnable {
+ private boolean firstLine = true;
+
+ MetricsLogRunnable() {
+ try {
+ metricsLogBW =
+ new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
+ metricsOutputDir + "/realtimetrack.json"), "UTF-8"));
+ metricsLogBW.write("[");
+ } catch (IOException e) {
+ LOG.info(e.getMessage());
+ }
+ }
+
+ @Override
+ public void run() {
+ if(running) {
+ // all WebApp to get real tracking json
+ String trackingMetrics = web.generateRealTimeTrackingMetrics();
+ // output
+ try {
+ if(firstLine) {
+ metricsLogBW.write(trackingMetrics + EOL);
+ firstLine = false;
+ } else {
+ metricsLogBW.write("," + trackingMetrics + EOL);
+ }
+ metricsLogBW.flush();
+ } catch (IOException e) {
+ LOG.info(e.getMessage());
+ }
+ }
+ }
+ }
+
+ void tearDown() throws Exception {
+ if (metricsLogBW != null) {
+ metricsLogBW.write("]");
+ metricsLogBW.close();
+ }
+
+ if (web != null) {
+ web.stop();
+ }
+
+ if (jobRuntimeLogBW != null) {
+ jobRuntimeLogBW.close();
+ }
+
+ if (pool != null) {
+ pool.shutdown();
+ }
+ }
+
+ void increaseSchedulerAllocationCounter() {
+ schedulerAllocateCounter.inc();
+ }
+
+ void increaseSchedulerHandleCounter(SchedulerEventType schedulerEventType) {
+ schedulerHandleCounter.inc();
+ schedulerHandleCounterMap.get(schedulerEventType).inc();
+ }
+
+ Timer getSchedulerAllocateTimer() {
+ return schedulerAllocateTimer;
+ }
+
+ Timer getSchedulerHandleTimer() {
+ return schedulerHandleTimer;
+ }
+
+ Timer getSchedulerHandleTimer(SchedulerEventType schedulerEventType) {
+ return schedulerHandleTimerMap.get(schedulerEventType);
+ }
+
+ private enum QueueMetric {
+ PENDING_MEMORY("pending.memory"),
+ PENDING_VCORES("pending.cores"),
+ ALLOCATED_MEMORY("allocated.memory"),
+ ALLOCATED_VCORES("allocated.cores");
+
+ private String value;
+
+ QueueMetric(String value) {
+ this.value = value;
+ }
+ }
+
+ private String getQueueMetricName(String queue, QueueMetric metric) {
+ return "counter.queue." + queue + "." + metric.value;
+ }
+
+ private void traceQueueIfNotTraced(String queue) {
+ queueLock.lock();
+ try {
+ if (!isTracked(queue)) {
+ trackQueue(queue);
+ }
+ } finally {
+ queueLock.unlock();
+ }
+ }
+
+ void initQueueMetric(String queueName){
+ SortedMap<String, Counter> counterMap = metrics.getCounters();
+
+ for (QueueMetric queueMetric : QueueMetric.values()) {
+ String metricName = getQueueMetricName(queueName, queueMetric);
+ if (!counterMap.containsKey(metricName)) {
+ metrics.counter(metricName);
+ counterMap = metrics.getCounters();
+ }
+ }
+
+ traceQueueIfNotTraced(queueName);
+ }
+
+ void updateQueueMetrics(Resource pendingResource, Resource allocatedResource,
+ String queueName) {
+ SortedMap<String, Counter> counterMap = metrics.getCounters();
+ for(QueueMetric metric : QueueMetric.values()) {
+ String metricName = getQueueMetricName(queueName, metric);
+ if (!counterMap.containsKey(metricName)) {
+ metrics.counter(metricName);
+ counterMap = metrics.getCounters();
+ }
+
+ if (metric == QueueMetric.PENDING_MEMORY) {
+ counterMap.get(metricName).inc(pendingResource.getMemorySize());
+ } else if (metric == QueueMetric.PENDING_VCORES) {
+ counterMap.get(metricName).inc(pendingResource.getVirtualCores());
+ } else if (metric == QueueMetric.ALLOCATED_MEMORY) {
+ counterMap.get(metricName).inc(allocatedResource.getMemorySize());
+ } else if (metric == QueueMetric.ALLOCATED_VCORES){
+ counterMap.get(metricName).inc(allocatedResource.getVirtualCores());
+ }
+ }
+
+ traceQueueIfNotTraced(queueName);
+ }
+
+ void updateQueueMetricsByRelease(Resource releaseResource, String queue) {
+ SortedMap<String, Counter> counterMap = metrics.getCounters();
+ String name = getQueueMetricName(queue, QueueMetric.ALLOCATED_MEMORY);
+ if (!counterMap.containsKey(name)) {
+ metrics.counter(name);
+ counterMap = metrics.getCounters();
+ }
+ counterMap.get(name).inc(-releaseResource.getMemorySize());
+
+ String vcoreMetric =
+ getQueueMetricName(queue, QueueMetric.ALLOCATED_VCORES);
+ if (!counterMap.containsKey(vcoreMetric)) {
+ metrics.counter(vcoreMetric);
+ counterMap = metrics.getCounters();
+ }
+ counterMap.get(vcoreMetric).inc(-releaseResource.getVirtualCores());
+ }
+
+ public void addTrackedApp(ApplicationId appId,
+ String oldAppId) {
+ trackApp(appId, oldAppId);
+ }
+
+ public void removeTrackedApp(String oldAppId) {
+ untrackApp(oldAppId);
+ }
+
+ public void addAMRuntime(ApplicationId appId, long traceStartTimeMS,
+ long traceEndTimeMS, long simulateStartTimeMS, long simulateEndTimeMS) {
+ try {
+ // write job runtime information
+ StringBuilder sb = new StringBuilder();
+ sb.append(appId).append(",").append(traceStartTimeMS).append(",")
+ .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
+ .append(",").append(simulateEndTimeMS);
+ jobRuntimeLogBW.write(sb.toString() + EOL);
+ jobRuntimeLogBW.flush();
+ } catch (IOException e) {
+ LOG.info(e.getMessage());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
index 962b137..406f050 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
@@ -17,28 +17,12 @@
*/
package org.apache.hadoop.yarn.sls.scheduler;
-import java.util.Set;
-
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-
-import com.codahale.metrics.MetricRegistry;
@Private
@Unstable
public interface SchedulerWrapper {
-
- MetricRegistry getMetrics();
SchedulerMetrics getSchedulerMetrics();
- Set<String> getQueueSet();
- void setQueueSet(Set<String> queues);
- Set<String> getTrackedAppSet();
- void setTrackedAppSet(Set<String> apps);
- void addTrackedApp(ApplicationId appId, String oldAppId);
- void removeTrackedApp(String oldAppId);
- void addAMRuntime(ApplicationId appId,
- long traceStartTimeMS, long traceEndTimeMS,
- long simulateStartTimeMS, long simulateEndTimeMS);
-
+ Tracker getTracker();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/Tracker.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/Tracker.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/Tracker.java
new file mode 100644
index 0000000..42a5c3c
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/Tracker.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.sls.scheduler;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+import java.util.Set;
+
+@Private
+@Unstable
+public class Tracker {
+ private Set<String> queueSet;
+ private Set<String> trackedAppSet;
+
+ public void setQueueSet(Set<String> queues) {
+ queueSet = queues;
+ }
+
+ public Set<String> getQueueSet() {
+ return queueSet;
+ }
+
+ public void setTrackedAppSet(Set<String> apps) {
+ trackedAppSet = apps;
+ }
+
+ public Set<String> getTrackedAppSet() {
+ return trackedAppSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
index e5f7cd0..085edc0 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.tools.rumen.LoggedTaskAttempt;
@Private
@Unstable
public class SLSUtils {
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
// hostname includes the network path and the host name. for example
// "/default-rack/hostFoo" or "/coreSwitchA/TORSwitchB/hostBar".
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
index 33d4846..2d2ffc5 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
@@ -107,12 +107,12 @@ public class SLSWebApp extends HttpServlet {
public SLSWebApp(SchedulerWrapper wrapper, int metricsAddressPort) {
this.wrapper = wrapper;
- metrics = wrapper.getMetrics();
handleOperTimecostHistogramMap =
new HashMap<SchedulerEventType, Histogram>();
queueAllocatedMemoryCounterMap = new HashMap<String, Counter>();
queueAllocatedVCoresCounterMap = new HashMap<String, Counter>();
schedulerMetrics = wrapper.getSchedulerMetrics();
+ metrics = schedulerMetrics.getMetrics();
port = metricsAddressPort;
}
@@ -226,7 +226,7 @@ public class SLSWebApp extends HttpServlet {
response.setStatus(HttpServletResponse.SC_OK);
// queues {0}
- Set<String> queues = wrapper.getQueueSet();
+ Set<String> queues = wrapper.getTracker().getQueueSet();
StringBuilder queueInfo = new StringBuilder();
int i = 0;
@@ -265,7 +265,7 @@ public class SLSWebApp extends HttpServlet {
// tracked queues {0}
StringBuilder trackedQueueInfo = new StringBuilder();
- Set<String> trackedQueues = wrapper.getQueueSet();
+ Set<String> trackedQueues = wrapper.getTracker().getQueueSet();
for(String queue : trackedQueues) {
trackedQueueInfo.append("<option value='Queue ").append(queue)
.append("'>").append(queue).append("</option>");
@@ -273,7 +273,7 @@ public class SLSWebApp extends HttpServlet {
// tracked apps {1}
StringBuilder trackedAppInfo = new StringBuilder();
- Set<String> trackedApps = wrapper.getTrackedAppSet();
+ Set<String> trackedApps = wrapper.getTracker().getTrackedAppSet();
for(String job : trackedApps) {
trackedAppInfo.append("<option value='Job ").append(job)
.append("'>").append(job).append("</option>");
@@ -422,7 +422,7 @@ public class SLSWebApp extends HttpServlet {
// allocated resource for each queue
Map<String, Double> queueAllocatedMemoryMap = new HashMap<String, Double>();
Map<String, Long> queueAllocatedVCoresMap = new HashMap<String, Long>();
- for (String queue : wrapper.getQueueSet()) {
+ for (String queue : wrapper.getTracker().getQueueSet()) {
// memory
String key = "counter.queue." + queue + ".allocated.memory";
if (! queueAllocatedMemoryCounterMap.containsKey(queue) &&
@@ -462,7 +462,7 @@ public class SLSWebApp extends HttpServlet {
.append(",\"cluster.available.memory\":").append(availableMemoryGB)
.append(",\"cluster.available.vcores\":").append(availableVCoresGB);
- for (String queue : wrapper.getQueueSet()) {
+ for (String queue : wrapper.getTracker().getQueueSet()) {
sb.append(",\"queue.").append(queue).append(".allocated.memory\":")
.append(queueAllocatedMemoryMap.get(queue));
sb.append(",\"queue.").append(queue).append(".allocated.vcores\":")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
index f0d8e6f..ca3d195 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
@@ -22,37 +22,56 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
-import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
-import org.apache.hadoop.yarn.sls.scheduler.FairSchedulerMetrics;
-import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper;
+import org.apache.hadoop.yarn.sls.scheduler.*;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
+@RunWith(Parameterized.class)
public class TestAMSimulator {
private ResourceManager rm;
private YarnConfiguration conf;
private Path metricOutputDir;
+ private Class slsScheduler;
+ private Class scheduler;
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[][] {
+ {SLSFairScheduler.class, FairScheduler.class},
+ {SLSCapacityScheduler.class, CapacityScheduler.class}
+ });
+ }
+
+ public TestAMSimulator(Class slsScheduler, Class scheduler) {
+ this.slsScheduler = slsScheduler;
+ this.scheduler = scheduler;
+ }
+
@Before
public void setup() {
createMetricOutputDir();
conf = new YarnConfiguration();
conf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricOutputDir.toString());
- conf.set(YarnConfiguration.RM_SCHEDULER,
- "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper");
- conf.set(SLSConfiguration.RM_SCHEDULER,
- "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
+ conf.set(YarnConfiguration.RM_SCHEDULER, slsScheduler.getName());
+ conf.set(SLSConfiguration.RM_SCHEDULER, scheduler.getName());
conf.setBoolean(SLSConfiguration.METRICS_SWITCH, true);
rm = new ResourceManager();
rm.init(conf);
@@ -76,15 +95,17 @@ public class TestAMSimulator {
}
private void verifySchedulerMetrics(String appId) {
- SchedulerWrapper schedulerWrapper = (SchedulerWrapper)
- rm.getResourceScheduler();
- MetricRegistry metricRegistry = schedulerWrapper.getMetrics();
- for (FairSchedulerMetrics.Metric metric :
- FairSchedulerMetrics.Metric.values()) {
- String key = "variable.app." + appId + "." + metric.getValue()
- + ".memory";
- Assert.assertTrue(metricRegistry.getGauges().containsKey(key));
- Assert.assertNotNull(metricRegistry.getGauges().get(key).getValue());
+ if (scheduler.equals(FairScheduler.class)) {
+ SchedulerMetrics schedulerMetrics = ((SchedulerWrapper)
+ rm.getResourceScheduler()).getSchedulerMetrics();
+ MetricRegistry metricRegistry = schedulerMetrics.getMetrics();
+ for (FairSchedulerMetrics.Metric metric :
+ FairSchedulerMetrics.Metric.values()) {
+ String key = "variable.app." + appId + "." + metric.getValue() +
+ ".memory";
+ Assert.assertTrue(metricRegistry.getGauges().containsKey(key));
+ Assert.assertNotNull(metricRegistry.getGauges().get(key).getValue());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a5516c2/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
index f9a3932..2f10f7d 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
@@ -21,26 +21,50 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
+import org.apache.hadoop.yarn.sls.scheduler.SLSCapacityScheduler;
+import org.apache.hadoop.yarn.sls.scheduler.SLSFairScheduler;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import java.util.Arrays;
+import java.util.Collection;
+
+@RunWith(Parameterized.class)
public class TestNMSimulator {
private final int GB = 1024;
private ResourceManager rm;
private YarnConfiguration conf;
+ private Class slsScheduler;
+ private Class scheduler;
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[][] {
+ {SLSFairScheduler.class, FairScheduler.class},
+ {SLSCapacityScheduler.class, CapacityScheduler.class}
+ });
+ }
+
+ public TestNMSimulator(Class slsScheduler, Class scheduler) {
+ this.slsScheduler = slsScheduler;
+ this.scheduler = scheduler;
+ }
+
@Before
public void setup() {
conf = new YarnConfiguration();
- conf.set(YarnConfiguration.RM_SCHEDULER,
- "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper");
- conf.set(SLSConfiguration.RM_SCHEDULER,
- "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
+ conf.set(YarnConfiguration.RM_SCHEDULER, slsScheduler.getName());
+ conf.set(SLSConfiguration.RM_SCHEDULER, scheduler.getName());
conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false);
rm = new ResourceManager();
rm.init(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-6329. Remove unnecessary TODO
comment from AppLogAggregatorImpl.java. Contributed by victor bertschinger.
Posted by ae...@apache.org.
YARN-6329. Remove unnecessary TODO comment from AppLogAggregatorImpl.java. Contributed by victor bertschinger.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82fb9ce8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82fb9ce8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82fb9ce8
Branch: refs/heads/HDFS-7240
Commit: 82fb9ce8df7a534a4cbcec624d7b6c0b33e79123
Parents: 063b513
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Mar 29 10:35:30 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Mar 29 10:36:24 2017 +0900
----------------------------------------------------------------------
.../containermanager/logaggregation/AppLogAggregatorImpl.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fb9ce8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 065854e..d70acc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -573,8 +573,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
(remoteNodeLogFileForApp.getName() + LogAggregationUtils.TMP_FILE_SUFFIX));
}
- // TODO: The condition: containerId.getId() == 1 to determine an AM container
- // is not always true.
private boolean shouldUploadLogs(ContainerLogContext logContext) {
return logAggPolicy.shouldDoLogAggregation(logContext);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDFS-11603. Improve slow
mirror/disk warnings in BlockReceiver.
Posted by ae...@apache.org.
HDFS-11603. Improve slow mirror/disk warnings in BlockReceiver.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5485d93b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5485d93b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5485d93b
Branch: refs/heads/HDFS-7240
Commit: 5485d93bda3329a7c80767c3723cc6e1a9233dbc
Parents: 318bfb0
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Mar 31 12:10:20 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Mar 31 12:10:27 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/DatanodeInfo.java | 1 +
.../hdfs/server/datanode/BlockReceiver.java | 61 +++++++++++++++-----
2 files changed, 47 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5485d93b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index e1698c9..0a8c915 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -55,6 +55,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
private String softwareVersion;
private List<String> dependentHostNames = new LinkedList<>();
private String upgradeDomain;
+ public static final DatanodeInfo[] EMPTY_ARRAY = {};
// Datanode administrative states
public enum AdminStates {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5485d93b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index bb6bd55..00109e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -100,6 +100,7 @@ class BlockReceiver implements Closeable {
private DataTransferThrottler throttler;
private ReplicaOutputStreams streams;
private DatanodeInfo srcDataNode = null;
+ private DatanodeInfo[] downstreamDNs = DatanodeInfo.EMPTY_ARRAY;
private final DataNode datanode;
volatile private boolean mirrorError;
@@ -424,10 +425,10 @@ class BlockReceiver implements Closeable {
}
}
long duration = Time.monotonicNow() - begin;
- if (duration > datanodeSlowLogThresholdMs) {
+ if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
LOG.warn("Slow flushOrSync took " + duration + "ms (threshold="
+ datanodeSlowLogThresholdMs + "ms), isSync:" + isSync + ", flushTotalNanos="
- + flushTotalNanos + "ns");
+ + flushTotalNanos + "ns, volume=" + getVolumeBaseUri());
}
}
@@ -578,9 +579,10 @@ class BlockReceiver implements Closeable {
mirrorAddr,
duration);
trackSendPacketToLastNodeInPipeline(duration);
- if (duration > datanodeSlowLogThresholdMs) {
+ if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
LOG.warn("Slow BlockReceiver write packet to mirror took " + duration
- + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)");
+ + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), "
+ + "downstream DNs=" + Arrays.toString(downstreamDNs));
}
} catch (IOException e) {
handleMirrorOutError(e);
@@ -711,9 +713,10 @@ class BlockReceiver implements Closeable {
streams.writeDataToDisk(dataBuf.array(),
startByteToDisk, numBytesToDisk);
long duration = Time.monotonicNow() - begin;
- if (duration > datanodeSlowLogThresholdMs) {
+ if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
LOG.warn("Slow BlockReceiver write data to disk cost:" + duration
- + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)");
+ + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), "
+ + "volume=" + getVolumeBaseUri());
}
if (duration > maxWriteToDiskMs) {
@@ -902,9 +905,10 @@ class BlockReceiver implements Closeable {
}
lastCacheManagementOffset = offsetInBlock;
long duration = Time.monotonicNow() - begin;
- if (duration > datanodeSlowLogThresholdMs) {
+ if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
LOG.warn("Slow manageWriterOsCache took " + duration
- + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)");
+ + "ms (threshold=" + datanodeSlowLogThresholdMs
+ + "ms), volume=" + getVolumeBaseUri());
}
}
} catch (Throwable t) {
@@ -932,13 +936,7 @@ class BlockReceiver implements Closeable {
boolean responderClosed = false;
mirrorOut = mirrOut;
mirrorAddr = mirrAddr;
- isPenultimateNode = ((downstreams != null) && (downstreams.length == 1));
- if (isPenultimateNode) {
- mirrorNameForMetrics = (downstreams[0].getInfoSecurePort() != 0 ?
- downstreams[0].getInfoSecureAddr() : downstreams[0].getInfoAddr());
- LOG.debug("Will collect peer metrics for downstream node {}",
- mirrorNameForMetrics);
- }
+ initPerfMonitoring(downstreams);
throttler = throttlerArg;
this.replyOut = replyOut;
@@ -1058,6 +1056,39 @@ class BlockReceiver implements Closeable {
}
}
+ /**
+ * If we have downstream DNs and peerMetrics are enabled, then initialize
+ * some state for monitoring the performance of downstream DNs.
+ *
+ * @param downstreams downstream DNs, or null if there are none.
+ */
+ private void initPerfMonitoring(DatanodeInfo[] downstreams) {
+ if (downstreams != null && downstreams.length > 0) {
+ downstreamDNs = downstreams;
+ isPenultimateNode = (downstreams.length == 1);
+ if (isPenultimateNode && datanode.getPeerMetrics() != null) {
+ mirrorNameForMetrics = (downstreams[0].getInfoSecurePort() != 0 ?
+ downstreams[0].getInfoSecureAddr() : downstreams[0].getInfoAddr());
+ LOG.debug("Will collect peer metrics for downstream node {}",
+ mirrorNameForMetrics);
+ }
+ }
+ }
+
+ /**
+ * Fetch the base URI of the volume on which this replica resides.
+ *
+ * @returns Volume base URI as string if available. Else returns the
+ * the string "unavailable".
+ */
+ private String getVolumeBaseUri() {
+ final ReplicaInfo ri = replicaInfo.getReplicaInfo();
+ if (ri != null && ri.getVolume() != null) {
+ return ri.getVolume().getBaseURI().toString();
+ }
+ return "unavailable";
+ }
+
/** Cleanup a partial block
* if this write is for a replication request (and not from a client)
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HADOOP-14267. Make DistCpOptions
immutable. Contributed by Mingliang Liu
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
index af91347..8111b04 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
@@ -123,10 +123,10 @@ public class SimpleCopyListing extends CopyListing {
}
@Override
- protected void validatePaths(DistCpOptions options)
+ protected void validatePaths(DistCpContext context)
throws IOException, InvalidInputException {
- Path targetPath = options.getTargetPath();
+ Path targetPath = context.getTargetPath();
FileSystem targetFS = targetPath.getFileSystem(getConf());
boolean targetExists = false;
boolean targetIsFile = false;
@@ -142,12 +142,12 @@ public class SimpleCopyListing extends CopyListing {
//If target is a file, then source has to be single file
if (targetIsFile) {
- if (options.getSourcePaths().size() > 1) {
+ if (context.getSourcePaths().size() > 1) {
throw new InvalidInputException("Multiple source being copied to a file: " +
targetPath);
}
- Path srcPath = options.getSourcePaths().get(0);
+ Path srcPath = context.getSourcePaths().get(0);
FileSystem sourceFS = srcPath.getFileSystem(getConf());
if (!sourceFS.isFile(srcPath)) {
throw new InvalidInputException("Cannot copy " + srcPath +
@@ -155,12 +155,12 @@ public class SimpleCopyListing extends CopyListing {
}
}
- if (options.shouldAtomicCommit() && targetExists) {
+ if (context.shouldAtomicCommit() && targetExists) {
throw new InvalidInputException("Target path for atomic-commit already exists: " +
targetPath + ". Cannot atomic-commit to pre-existing target-path.");
}
- for (Path path: options.getSourcePaths()) {
+ for (Path path: context.getSourcePaths()) {
FileSystem fs = path.getFileSystem(getConf());
if (!fs.exists(path)) {
throw new InvalidInputException(path + " doesn't exist");
@@ -184,7 +184,7 @@ public class SimpleCopyListing extends CopyListing {
}
if (targetIsReservedRaw) {
- options.preserveRawXattrs();
+ context.setPreserveRawXattrs(true);
getConf().setBoolean(DistCpConstants.CONF_LABEL_PRESERVE_RAWXATTRS, true);
}
@@ -194,18 +194,19 @@ public class SimpleCopyListing extends CopyListing {
*/
Credentials credentials = getCredentials();
if (credentials != null) {
- Path[] inputPaths = options.getSourcePaths().toArray(new Path[1]);
+ Path[] inputPaths = context.getSourcePaths()
+ .toArray(new Path[1]);
TokenCache.obtainTokensForNamenodes(credentials, inputPaths, getConf());
}
}
@Override
protected void doBuildListing(Path pathToListingFile,
- DistCpOptions options) throws IOException {
- if(options.shouldUseSnapshotDiff()) {
- doBuildListingWithSnapshotDiff(getWriter(pathToListingFile), options);
- }else {
- doBuildListing(getWriter(pathToListingFile), options);
+ DistCpContext context) throws IOException {
+ if (context.shouldUseSnapshotDiff()) {
+ doBuildListingWithSnapshotDiff(getWriter(pathToListingFile), context);
+ } else {
+ doBuildListing(getWriter(pathToListingFile), context);
}
}
@@ -232,22 +233,22 @@ public class SimpleCopyListing extends CopyListing {
* @throws IOException
*/
private void addToFileListing(SequenceFile.Writer fileListWriter,
- Path sourceRoot, Path path, DistCpOptions options) throws IOException {
+ Path sourceRoot, Path path, DistCpContext context) throws IOException {
sourceRoot = getPathWithSchemeAndAuthority(sourceRoot);
path = getPathWithSchemeAndAuthority(path);
path = makeQualified(path);
FileSystem sourceFS = sourceRoot.getFileSystem(getConf());
FileStatus fileStatus = sourceFS.getFileStatus(path);
- final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
- final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
- final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs();
+ final boolean preserveAcls = context.shouldPreserve(FileAttribute.ACL);
+ final boolean preserveXAttrs = context.shouldPreserve(FileAttribute.XATTR);
+ final boolean preserveRawXAttrs = context.shouldPreserveRawXattrs();
LinkedList<CopyListingFileStatus> fileCopyListingStatus =
DistCpUtils.toCopyListingFileStatus(sourceFS, fileStatus,
preserveAcls, preserveXAttrs, preserveRawXAttrs,
- options.getBlocksPerChunk());
+ context.getBlocksPerChunk());
writeToFileListingRoot(fileListWriter, fileCopyListingStatus,
- sourceRoot, options);
+ sourceRoot, context);
}
/**
@@ -258,14 +259,16 @@ public class SimpleCopyListing extends CopyListing {
* {@link org.apache.hadoop.tools.DistCpSync#sync}. An item can be
* created/modified and renamed, in which case, the target path is put
* into the list.
+ * @param fileListWriter the list for holding processed results
+ * @param context The DistCp context with associated input options
* @throws IOException
*/
@VisibleForTesting
protected void doBuildListingWithSnapshotDiff(
- SequenceFile.Writer fileListWriter, DistCpOptions options)
+ SequenceFile.Writer fileListWriter, DistCpContext context)
throws IOException {
ArrayList<DiffInfo> diffList = distCpSync.prepareDiffListForCopyListing();
- Path sourceRoot = options.getSourcePaths().get(0);
+ Path sourceRoot = context.getSourcePaths().get(0);
FileSystem sourceFS = sourceRoot.getFileSystem(getConf());
try {
@@ -273,13 +276,13 @@ public class SimpleCopyListing extends CopyListing {
for (DiffInfo diff : diffList) {
// add snapshot paths prefix
diff.setTarget(
- new Path(options.getSourcePaths().get(0), diff.getTarget()));
+ new Path(context.getSourcePaths().get(0), diff.getTarget()));
if (diff.getType() == SnapshotDiffReport.DiffType.MODIFY) {
addToFileListing(fileListWriter,
- sourceRoot, diff.getTarget(), options);
+ sourceRoot, diff.getTarget(), context);
} else if (diff.getType() == SnapshotDiffReport.DiffType.CREATE) {
addToFileListing(fileListWriter,
- sourceRoot, diff.getTarget(), options);
+ sourceRoot, diff.getTarget(), context);
FileStatus sourceStatus = sourceFS.getFileStatus(diff.getTarget());
if (sourceStatus.isDirectory()) {
@@ -290,13 +293,13 @@ public class SimpleCopyListing extends CopyListing {
HashSet<String> excludeList =
distCpSync.getTraverseExcludeList(diff.getSource(),
- options.getSourcePaths().get(0));
+ context.getSourcePaths().get(0));
ArrayList<FileStatus> sourceDirs = new ArrayList<>();
sourceDirs.add(sourceStatus);
traverseDirectory(fileListWriter, sourceFS, sourceDirs,
- sourceRoot, options, excludeList, fileStatuses);
+ sourceRoot, context, excludeList, fileStatuses);
}
}
}
@@ -325,27 +328,30 @@ public class SimpleCopyListing extends CopyListing {
* See computeSourceRootPath method for how the root path of the source is
* computed.
* @param fileListWriter
- * @param options
+ * @param context The distcp context with associated input options
* @throws IOException
*/
@VisibleForTesting
protected void doBuildListing(SequenceFile.Writer fileListWriter,
- DistCpOptions options) throws IOException {
- if (options.getNumListstatusThreads() > 0) {
- numListstatusThreads = options.getNumListstatusThreads();
+ DistCpContext context) throws IOException {
+ if (context.getNumListstatusThreads() > 0) {
+ numListstatusThreads = context.getNumListstatusThreads();
}
try {
List<FileStatusInfo> statusList = Lists.newArrayList();
- for (Path path: options.getSourcePaths()) {
+ for (Path path: context.getSourcePaths()) {
FileSystem sourceFS = path.getFileSystem(getConf());
- final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
- final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
- final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs();
+ final boolean preserveAcls =
+ context.shouldPreserve(FileAttribute.ACL);
+ final boolean preserveXAttrs =
+ context.shouldPreserve(FileAttribute.XATTR);
+ final boolean preserveRawXAttrs =
+ context.shouldPreserveRawXattrs();
path = makeQualified(path);
FileStatus rootStatus = sourceFS.getFileStatus(path);
- Path sourcePathRoot = computeSourceRootPath(rootStatus, options);
+ Path sourcePathRoot = computeSourceRootPath(rootStatus, context);
FileStatus[] sourceFiles = sourceFS.listStatus(path);
boolean explore = (sourceFiles != null && sourceFiles.length > 0);
@@ -353,9 +359,9 @@ public class SimpleCopyListing extends CopyListing {
LinkedList<CopyListingFileStatus> rootCopyListingStatus =
DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus,
preserveAcls, preserveXAttrs, preserveRawXAttrs,
- options.getBlocksPerChunk());
+ context.getBlocksPerChunk());
writeToFileListingRoot(fileListWriter, rootCopyListingStatus,
- sourcePathRoot, options);
+ sourcePathRoot, context);
}
if (explore) {
ArrayList<FileStatus> sourceDirs = new ArrayList<FileStatus>();
@@ -368,7 +374,7 @@ public class SimpleCopyListing extends CopyListing {
preserveAcls && sourceStatus.isDirectory(),
preserveXAttrs && sourceStatus.isDirectory(),
preserveRawXAttrs && sourceStatus.isDirectory(),
- options.getBlocksPerChunk());
+ context.getBlocksPerChunk());
for (CopyListingFileStatus fs : sourceCopyListingStatus) {
if (randomizeFileListing) {
addToFileListing(statusList,
@@ -385,7 +391,7 @@ public class SimpleCopyListing extends CopyListing {
}
}
traverseDirectory(fileListWriter, sourceFS, sourceDirs,
- sourcePathRoot, options, null, statusList);
+ sourcePathRoot, context, null, statusList);
}
}
if (randomizeFileListing) {
@@ -447,13 +453,13 @@ public class SimpleCopyListing extends CopyListing {
}
private Path computeSourceRootPath(FileStatus sourceStatus,
- DistCpOptions options) throws IOException {
+ DistCpContext context) throws IOException {
- Path target = options.getTargetPath();
+ Path target = context.getTargetPath();
FileSystem targetFS = target.getFileSystem(getConf());
- final boolean targetPathExists = options.getTargetPathExists();
+ final boolean targetPathExists = context.isTargetPathExists();
- boolean solitaryFile = options.getSourcePaths().size() == 1
+ boolean solitaryFile = context.getSourcePaths().size() == 1
&& !sourceStatus.isDirectory();
if (solitaryFile) {
@@ -463,8 +469,11 @@ public class SimpleCopyListing extends CopyListing {
return sourceStatus.getPath().getParent();
}
} else {
- boolean specialHandling = (options.getSourcePaths().size() == 1 && !targetPathExists) ||
- options.shouldSyncFolder() || options.shouldOverwrite();
+ boolean specialHandling =
+ (context.getSourcePaths().size() == 1 &&
+ !targetPathExists) ||
+ context.shouldSyncFolder() ||
+ context.shouldOverwrite();
if ((specialHandling && sourceStatus.isDirectory()) ||
sourceStatus.getPath().isRoot()) {
@@ -610,13 +619,13 @@ public class SimpleCopyListing extends CopyListing {
FileSystem sourceFS,
ArrayList<FileStatus> sourceDirs,
Path sourcePathRoot,
- DistCpOptions options,
+ DistCpContext context,
HashSet<String> excludeList,
List<FileStatusInfo> fileStatuses)
throws IOException {
- final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
- final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
- final boolean preserveRawXattrs = options.shouldPreserveRawXattrs();
+ final boolean preserveAcls = context.shouldPreserve(FileAttribute.ACL);
+ final boolean preserveXAttrs = context.shouldPreserve(FileAttribute.XATTR);
+ final boolean preserveRawXattrs = context.shouldPreserveRawXattrs();
assert numListstatusThreads > 0;
if (LOG.isDebugEnabled()) {
@@ -649,7 +658,7 @@ public class SimpleCopyListing extends CopyListing {
preserveAcls && child.isDirectory(),
preserveXAttrs && child.isDirectory(),
preserveRawXattrs && child.isDirectory(),
- options.getBlocksPerChunk());
+ context.getBlocksPerChunk());
for (CopyListingFileStatus fs : childCopyListingStatus) {
if (randomizeFileListing) {
@@ -681,9 +690,9 @@ public class SimpleCopyListing extends CopyListing {
private void writeToFileListingRoot(SequenceFile.Writer fileListWriter,
LinkedList<CopyListingFileStatus> fileStatus, Path sourcePathRoot,
- DistCpOptions options) throws IOException {
- boolean syncOrOverwrite = options.shouldSyncFolder() ||
- options.shouldOverwrite();
+ DistCpContext context) throws IOException {
+ boolean syncOrOverwrite = context.shouldSyncFolder() ||
+ context.shouldOverwrite();
for (CopyListingFileStatus fs : fileStatus) {
if (fs.getPath().equals(sourcePathRoot) &&
fs.isDirectory() && syncOrOverwrite) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
index 6ddaab9..81c2be7 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.tools.CopyListing;
import org.apache.hadoop.tools.CopyListingFileStatus;
import org.apache.hadoop.tools.DistCpConstants;
import org.apache.hadoop.tools.DistCpOptionSwitch;
+import org.apache.hadoop.tools.DistCpContext;
import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.GlobbedCopyListing;
@@ -354,16 +355,18 @@ public class CopyCommitter extends FileOutputCommitter {
Path resultNonePath = Path.getPathWithoutSchemeAndAuthority(targetFinalPath)
.toString().startsWith(DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME)
? DistCpConstants.RAW_NONE_PATH : DistCpConstants.NONE_PATH;
- DistCpOptions options = new DistCpOptions(targets, resultNonePath);
//
// Set up options to be the same from the CopyListing.buildListing's perspective,
// so to collect similar listings as when doing the copy
//
- options.setOverwrite(overwrite);
- options.setSyncFolder(syncFolder);
- options.setTargetPathExists(targetPathExists);
-
- target.buildListing(targetListing, options);
+ DistCpOptions options = new DistCpOptions.Builder(targets, resultNonePath)
+ .withOverwrite(overwrite)
+ .withSyncFolder(syncFolder)
+ .build();
+ DistCpContext distCpContext = new DistCpContext(options);
+ distCpContext.setTargetPathExists(targetPathExists);
+
+ target.buildListing(targetListing, distCpContext);
Path sortedTargetListing = DistCpUtils.sortListing(clusterFS, conf, targetListing);
long totalLen = clusterFS.getFileStatus(sortedTargetListing).getLen();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
index e315b84..dbe750a 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.tools.CopyListing.AclsNotSupportedException;
import org.apache.hadoop.tools.CopyListing.XAttrsNotSupportedException;
import org.apache.hadoop.tools.CopyListingFileStatus;
-import org.apache.hadoop.tools.DistCpOptions;
+import org.apache.hadoop.tools.DistCpContext;
import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.mapred.UniformSizeInputFormat;
import org.apache.hadoop.util.StringUtils;
@@ -116,13 +116,13 @@ public class DistCpUtils {
* a particular strategy from distcp-default.xml
*
* @param conf - Configuration object
- * @param options - Handle to input options
+ * @param context - Distcp context with associated input options
* @return Class implementing the strategy specified in options.
*/
public static Class<? extends InputFormat> getStrategy(Configuration conf,
- DistCpOptions options) {
+ DistCpContext context) {
String confLabel = "distcp."
- + StringUtils.toLowerCase(options.getCopyStrategy())
+ + StringUtils.toLowerCase(context.getCopyStrategy())
+ ".strategy" + ".impl";
return conf.getClass(confLabel, UniformSizeInputFormat.class, InputFormat.class);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
index ea63e23..97a6f62 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
@@ -103,20 +103,19 @@ public class TestCopyListing extends SimpleCopyListing {
List<Path> srcPaths = new ArrayList<Path>();
srcPaths.add(new Path("/tmp/in/1"));
srcPaths.add(new Path("/tmp/in/2"));
- Path target = new Path("/tmp/out/1");
+ final Path target = new Path("/tmp/out/1");
TestDistCpUtils.createFile(fs, "/tmp/in/1");
TestDistCpUtils.createFile(fs, "/tmp/in/2");
fs.mkdirs(target);
- DistCpOptions options = new DistCpOptions(srcPaths, target);
- validatePaths(options);
+ final DistCpOptions options = new DistCpOptions.Builder(srcPaths, target)
+ .build();
+ validatePaths(new DistCpContext(options));
TestDistCpUtils.delete(fs, "/tmp");
//No errors
- target = new Path("/tmp/out/1");
fs.create(target).close();
- options = new DistCpOptions(srcPaths, target);
try {
- validatePaths(options);
+ validatePaths(new DistCpContext(options));
Assert.fail("Invalid inputs accepted");
} catch (InvalidInputException ignore) { }
TestDistCpUtils.delete(fs, "/tmp");
@@ -124,11 +123,9 @@ public class TestCopyListing extends SimpleCopyListing {
srcPaths.clear();
srcPaths.add(new Path("/tmp/in/1"));
fs.mkdirs(new Path("/tmp/in/1"));
- target = new Path("/tmp/out/1");
fs.create(target).close();
- options = new DistCpOptions(srcPaths, target);
try {
- validatePaths(options);
+ validatePaths(new DistCpContext(options));
Assert.fail("Invalid inputs accepted");
} catch (InvalidInputException ignore) { }
TestDistCpUtils.delete(fs, "/tmp");
@@ -151,10 +148,13 @@ public class TestCopyListing extends SimpleCopyListing {
TestDistCpUtils.createFile(fs, "/tmp/in/src2/1.txt");
Path target = new Path("/tmp/out");
Path listingFile = new Path("/tmp/list");
- DistCpOptions options = new DistCpOptions(srcPaths, target);
- CopyListing listing = CopyListing.getCopyListing(getConf(), CREDENTIALS, options);
+ final DistCpOptions options = new DistCpOptions.Builder(srcPaths, target)
+ .build();
+ final DistCpContext context = new DistCpContext(options);
+ CopyListing listing = CopyListing.getCopyListing(getConf(), CREDENTIALS,
+ context);
try {
- listing.buildListing(listingFile, options);
+ listing.buildListing(listingFile, context);
Assert.fail("Duplicates not detected");
} catch (DuplicateFileException ignore) {
}
@@ -196,11 +196,12 @@ public class TestCopyListing extends SimpleCopyListing {
Path listingFile = new Path("/tmp/file");
- DistCpOptions options = new DistCpOptions(srcPaths, target);
- options.setSyncFolder(true);
+ final DistCpOptions options = new DistCpOptions.Builder(srcPaths, target)
+ .withSyncFolder(true)
+ .build();
CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);
try {
- listing.buildListing(listingFile, options);
+ listing.buildListing(listingFile, new DistCpContext(options));
Assert.fail("Duplicates not detected");
} catch (DuplicateFileException ignore) {
}
@@ -209,7 +210,7 @@ public class TestCopyListing extends SimpleCopyListing {
TestDistCpUtils.delete(fs, "/tmp");
try {
- listing.buildListing(listingFile, options);
+ listing.buildListing(listingFile, new DistCpContext(options));
Assert.fail("Invalid input not detected");
} catch (InvalidInputException ignore) {
}
@@ -244,14 +245,14 @@ public class TestCopyListing extends SimpleCopyListing {
}
Path listingFile = new Path("/tmp/file");
- DistCpOptions options = new DistCpOptions(srcPaths, target);
- options.setSyncFolder(true);
+ final DistCpOptions options = new DistCpOptions.Builder(srcPaths, target)
+ .withSyncFolder(true).build();
// Check without randomizing files
getConf().setBoolean(
DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false);
SimpleCopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);
- listing.buildListing(listingFile, options);
+ listing.buildListing(listingFile, new DistCpContext(options));
Assert.assertEquals(listing.getNumberOfPaths(), pathCount);
validateFinalListing(listingFile, srcFiles);
@@ -265,7 +266,7 @@ public class TestCopyListing extends SimpleCopyListing {
// Set the seed for randomness, so that it can be verified later
long seed = System.nanoTime();
listing.setSeedForRandomListing(seed);
- listing.buildListing(listingFile, options);
+ listing.buildListing(listingFile, new DistCpContext(options));
Assert.assertEquals(listing.getNumberOfPaths(), pathCount);
// validate randomness
@@ -322,11 +323,12 @@ public class TestCopyListing extends SimpleCopyListing {
List<Path> srcPaths = new ArrayList<Path>();
srcPaths.add(sourceFile);
- DistCpOptions options = new DistCpOptions(srcPaths, targetFile);
+ DistCpOptions options = new DistCpOptions.Builder(srcPaths, targetFile)
+ .build();
CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);
final Path listFile = new Path(testRoot, "/tmp/fileList.seq");
- listing.buildListing(listFile, options);
+ listing.buildListing(listFile, new DistCpContext(options));
reader = new SequenceFile.Reader(getConf(), SequenceFile.Reader.file(listFile));
@@ -359,10 +361,11 @@ public class TestCopyListing extends SimpleCopyListing {
doThrow(expectedEx).when(writer).close();
SimpleCopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);
- DistCpOptions options = new DistCpOptions(srcs, new Path(outFile.toURI()));
+ final DistCpOptions options = new DistCpOptions.Builder(srcs,
+ new Path(outFile.toURI())).build();
Exception actualEx = null;
try {
- listing.doBuildListing(writer, options);
+ listing.doBuildListing(writer, new DistCpContext(options));
} catch (Exception e) {
actualEx = e;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
new file mode 100644
index 0000000..3525194
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
@@ -0,0 +1,500 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.tools;
+
+import java.util.Collections;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
+
+import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
+import static org.apache.hadoop.tools.DistCpOptions.MAX_NUM_LISTSTATUS_THREADS;
+import static org.junit.Assert.fail;
+
+/**
+ * This is to test constructing {@link DistCpOptions} manually with setters.
+ *
+ * The test cases in this class is very similar to the parser test, see
+ * {@link TestOptionsParser}.
+ */
+public class TestDistCpOptions {
+
+ private static final float DELTA = 0.001f;
+
+ @Test
+ public void testSetIgnoreFailure() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertFalse(builder.build().shouldIgnoreFailures());
+
+ builder.withIgnoreFailures(true);
+ Assert.assertTrue(builder.build().shouldIgnoreFailures());
+ }
+
+ @Test
+ public void testSetOverwrite() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertFalse(builder.build().shouldOverwrite());
+
+ builder.withOverwrite(true);
+ Assert.assertTrue(builder.build().shouldOverwrite());
+
+ try {
+ builder.withSyncFolder(true).build();
+ Assert.fail("Update and overwrite aren't allowed together");
+ } catch (IllegalArgumentException ignore) {
+ }
+ }
+
+ @Test
+ public void testLogPath() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertNull(builder.build().getLogPath());
+
+ final Path logPath = new Path("hdfs://localhost:8020/logs");
+ builder.withLogPath(logPath);
+ Assert.assertEquals(logPath, builder.build().getLogPath());
+ }
+
+ @Test
+ public void testSetBlokcing() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertTrue(builder.build().shouldBlock());
+
+ builder.withBlocking(false);
+ Assert.assertFalse(builder.build().shouldBlock());
+ }
+
+ @Test
+ public void testSetBandwidth() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertEquals(0, builder.build().getMapBandwidth(), DELTA);
+
+ builder.withMapBandwidth(11);
+ Assert.assertEquals(11, builder.build().getMapBandwidth(), DELTA);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSetNonPositiveBandwidth() {
+ new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"))
+ .withMapBandwidth(-11)
+ .build();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSetZeroBandwidth() {
+ new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"))
+ .withMapBandwidth(0)
+ .build();
+ }
+
+ @Test
+ public void testSetSkipCRC() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertFalse(builder.build().shouldSkipCRC());
+
+ final DistCpOptions options = builder.withSyncFolder(true).withCRC(true)
+ .build();
+ Assert.assertTrue(options.shouldSyncFolder());
+ Assert.assertTrue(options.shouldSkipCRC());
+ }
+
+ @Test
+ public void testSetAtomicCommit() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertFalse(builder.build().shouldAtomicCommit());
+
+ builder.withAtomicCommit(true);
+ Assert.assertTrue(builder.build().shouldAtomicCommit());
+
+ try {
+ builder.withSyncFolder(true).build();
+ Assert.fail("Atomic and sync folders were mutually exclusive");
+ } catch (IllegalArgumentException ignore) {
+ }
+ }
+
+ @Test
+ public void testSetWorkPath() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertNull(builder.build().getAtomicWorkPath());
+
+ builder.withAtomicCommit(true);
+ Assert.assertNull(builder.build().getAtomicWorkPath());
+
+ final Path workPath = new Path("hdfs://localhost:8020/work");
+ builder.withAtomicWorkPath(workPath);
+ Assert.assertEquals(workPath, builder.build().getAtomicWorkPath());
+ }
+
+ @Test
+ public void testSetSyncFolders() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertFalse(builder.build().shouldSyncFolder());
+
+ builder.withSyncFolder(true);
+ Assert.assertTrue(builder.build().shouldSyncFolder());
+ }
+
+ @Test
+ public void testSetDeleteMissing() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertFalse(builder.build().shouldDeleteMissing());
+
+ DistCpOptions options = builder.withSyncFolder(true)
+ .withDeleteMissing(true)
+ .build();
+ Assert.assertTrue(options.shouldSyncFolder());
+ Assert.assertTrue(options.shouldDeleteMissing());
+
+ options = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"))
+ .withOverwrite(true)
+ .withDeleteMissing(true)
+ .build();
+ Assert.assertTrue(options.shouldOverwrite());
+ Assert.assertTrue(options.shouldDeleteMissing());
+
+ try {
+ new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"))
+ .withDeleteMissing(true)
+ .build();
+ fail("Delete missing should fail without update or overwrite options");
+ } catch (IllegalArgumentException e) {
+ assertExceptionContains("Delete missing is applicable only with update " +
+ "or overwrite options", e);
+ }
+ try {
+ new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"))
+ .withSyncFolder(true)
+ .withDeleteMissing(true)
+ .withUseDiff("s1", "s2")
+ .build();
+ fail("Should have failed as -delete and -diff are mutually exclusive.");
+ } catch (IllegalArgumentException e) {
+ assertExceptionContains(
+ "-delete and -diff/-rdiff are mutually exclusive.", e);
+ }
+ }
+
+ @Test
+ public void testSetMaps() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertEquals(DistCpConstants.DEFAULT_MAPS,
+ builder.build().getMaxMaps());
+
+ builder.maxMaps(1);
+ Assert.assertEquals(1, builder.build().getMaxMaps());
+
+ builder.maxMaps(0);
+ Assert.assertEquals(1, builder.build().getMaxMaps());
+ }
+
+ @Test
+ public void testSetNumListtatusThreads() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"));
+ // If command line argument isn't set, we expect .getNumListstatusThreads
+ // option to be zero (so that we know when to override conf properties).
+ Assert.assertEquals(0, builder.build().getNumListstatusThreads());
+
+ builder.withNumListstatusThreads(12);
+ Assert.assertEquals(12, builder.build().getNumListstatusThreads());
+
+ builder.withNumListstatusThreads(0);
+ Assert.assertEquals(0, builder.build().getNumListstatusThreads());
+
+ // Ignore large number of threads.
+ builder.withNumListstatusThreads(MAX_NUM_LISTSTATUS_THREADS * 2);
+ Assert.assertEquals(MAX_NUM_LISTSTATUS_THREADS,
+ builder.build().getNumListstatusThreads());
+ }
+
+ @Test
+ public void testSourceListing() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertEquals(new Path("hdfs://localhost:8020/source/first"),
+ builder.build().getSourceFileListing());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testMissingTarget() {
+ new DistCpOptions.Builder(new Path("hdfs://localhost:8020/source/first"),
+ null);
+ }
+
+ @Test
+ public void testToString() {
+ DistCpOptions option = new DistCpOptions.Builder(new Path("abc"),
+ new Path("xyz")).build();
+ String val = "DistCpOptions{atomicCommit=false, syncFolder=false, " +
+ "deleteMissing=false, ignoreFailures=false, overwrite=false, " +
+ "append=false, useDiff=false, useRdiff=false, " +
+ "fromSnapshot=null, toSnapshot=null, " +
+ "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, " +
+ "mapBandwidth=0.0, copyStrategy='uniformsize', preserveStatus=[], " +
+ "atomicWorkPath=null, logPath=null, sourceFileListing=abc, " +
+ "sourcePaths=null, targetPath=xyz, filtersFile='null'," +
+ " blocksPerChunk=0}";
+ String optionString = option.toString();
+ Assert.assertEquals(val, optionString);
+ Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
+ DistCpOptionSwitch.ATOMIC_COMMIT.name());
+ }
+
+ @Test
+ public void testCopyStrategy() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertEquals(DistCpConstants.UNIFORMSIZE,
+ builder.build().getCopyStrategy());
+ builder.withCopyStrategy("dynamic");
+ Assert.assertEquals("dynamic", builder.build().getCopyStrategy());
+ }
+
+ @Test
+ public void testTargetPath() {
+ final DistCpOptions options = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/")).build();
+ Assert.assertEquals(new Path("hdfs://localhost:8020/target/"),
+ options.getTargetPath());
+ }
+
+ @Test
+ public void testPreserve() {
+ DistCpOptions options = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"))
+ .build();
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.BLOCKSIZE));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.REPLICATION));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.USER));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.CHECKSUMTYPE));
+
+ options = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"))
+ .preserve(FileAttribute.ACL)
+ .build();
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.BLOCKSIZE));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.REPLICATION));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.USER));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.CHECKSUMTYPE));
+ Assert.assertTrue(options.shouldPreserve(FileAttribute.ACL));
+
+ options = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"))
+ .preserve(FileAttribute.BLOCKSIZE)
+ .preserve(FileAttribute.REPLICATION)
+ .preserve(FileAttribute.PERMISSION)
+ .preserve(FileAttribute.USER)
+ .preserve(FileAttribute.GROUP)
+ .preserve(FileAttribute.CHECKSUMTYPE)
+ .build();
+
+ Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE));
+ Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION));
+ Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION));
+ Assert.assertTrue(options.shouldPreserve(FileAttribute.USER));
+ Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP));
+ Assert.assertTrue(options.shouldPreserve(FileAttribute.CHECKSUMTYPE));
+ Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR));
+ }
+
+ @Test
+ public void testAppendOption() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"))
+ .withSyncFolder(true)
+ .withAppend(true);
+ Assert.assertTrue(builder.build().shouldAppend());
+
+ try {
+ // make sure -append is only valid when -update is specified
+ new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"))
+ .withAppend(true)
+ .build();
+ fail("Append should fail if update option is not specified");
+ } catch (IllegalArgumentException e) {
+ assertExceptionContains(
+ "Append is valid only with update options", e);
+ }
+
+ try {
+ // make sure -append is invalid when skipCrc is specified
+ new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"))
+ .withSyncFolder(true)
+ .withAppend(true)
+ .withCRC(true)
+ .build();
+ fail("Append should fail if skipCrc option is specified");
+ } catch (IllegalArgumentException e) {
+ assertExceptionContains(
+ "Append is disallowed when skipping CRC", e);
+ }
+ }
+
+ @Test
+ public void testDiffOption() {
+ DistCpOptions options = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"))
+ .withSyncFolder(true)
+ .withUseDiff("s1", "s2")
+ .build();
+ Assert.assertTrue(options.shouldUseDiff());
+ Assert.assertEquals("s1", options.getFromSnapshot());
+ Assert.assertEquals("s2", options.getToSnapshot());
+
+ options = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"))
+ .withSyncFolder(true)
+ .withUseDiff("s1", ".")
+ .build();
+ Assert.assertTrue(options.shouldUseDiff());
+ Assert.assertEquals("s1", options.getFromSnapshot());
+ Assert.assertEquals(".", options.getToSnapshot());
+
+ // make sure -diff is only valid when -update is specified
+ try {
+ new DistCpOptions.Builder(new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"))
+ .withUseDiff("s1", "s2")
+ .build();
+ fail("-diff should fail if -update option is not specified");
+ } catch (IllegalArgumentException e) {
+ assertExceptionContains(
+ "-diff/-rdiff is valid only with -update option", e);
+ }
+
+ try {
+ new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"))
+ .withSyncFolder(true)
+ .withUseDiff("s1", "s2")
+ .withDeleteMissing(true)
+ .build();
+ fail("Should fail as -delete and -diff/-rdiff are mutually exclusive.");
+ } catch (IllegalArgumentException e) {
+ assertExceptionContains(
+ "-delete and -diff/-rdiff are mutually exclusive.", e);
+ }
+
+ try {
+ new DistCpOptions.Builder(new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"))
+ .withUseDiff("s1", "s2")
+ .withDeleteMissing(true)
+ .build();
+ fail("-diff should fail if -update option is not specified");
+ } catch (IllegalArgumentException e) {
+ assertExceptionContains(
+ "-delete and -diff/-rdiff are mutually exclusive.", e);
+ }
+
+ try {
+ new DistCpOptions.Builder(new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"))
+ .withDeleteMissing(true)
+ .withUseDiff("s1", "s2")
+ .build();
+ fail("Should have failed as -delete and -diff are mutually exclusive");
+ } catch (IllegalArgumentException e) {
+ assertExceptionContains(
+ "-delete and -diff/-rdiff are mutually exclusive", e);
+ }
+ }
+
+ @Test
+ public void testExclusionsOption() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/first"),
+ new Path("hdfs://localhost:8020/target/"));
+ Assert.assertNull(builder.build().getFiltersFile());
+
+ builder.withFiltersFile("/tmp/filters.txt");
+ Assert.assertEquals("/tmp/filters.txt", builder.build().getFiltersFile());
+ }
+
+ @Test
+ public void testSetOptionsForSplitLargeFile() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:8020/source/"),
+ new Path("hdfs://localhost:8020/target/"))
+ .withAppend(true)
+ .withSyncFolder(true);
+ Assert.assertFalse(builder.build().shouldPreserve(FileAttribute.BLOCKSIZE));
+ Assert.assertTrue(builder.build().shouldAppend());
+
+ builder.withBlocksPerChunk(5440);
+ Assert.assertTrue(builder.build().shouldPreserve(FileAttribute.BLOCKSIZE));
+ Assert.assertFalse(builder.build().shouldAppend());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
index 94e8604..717b2f0 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
@@ -39,7 +39,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -47,7 +47,7 @@ public class TestDistCpSync {
private MiniDFSCluster cluster;
private final Configuration conf = new HdfsConfiguration();
private DistributedFileSystem dfs;
- private DistCpOptions options;
+ private DistCpContext context;
private final Path source = new Path("/source");
private final Path target = new Path("/target");
private final long BLOCK_SIZE = 1024;
@@ -62,10 +62,13 @@ public class TestDistCpSync {
dfs.mkdirs(source);
dfs.mkdirs(target);
- options = new DistCpOptions(Arrays.asList(source), target);
- options.setSyncFolder(true);
- options.setUseDiff("s1", "s2");
+ final DistCpOptions options = new DistCpOptions.Builder(
+ Collections.singletonList(source), target)
+ .withSyncFolder(true)
+ .withUseDiff("s1", "s2")
+ .build();
options.appendToConf(conf);
+ context = new DistCpContext(options);
conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, target.toString());
conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, target.toString());
@@ -92,34 +95,34 @@ public class TestDistCpSync {
// make sure the source path has been updated to the snapshot path
final Path spath = new Path(source,
HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2");
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, context.getSourcePaths().get(0));
// reset source path in options
- options.setSourcePaths(Arrays.asList(source));
+ context.setSourcePaths(Collections.singletonList(source));
// the source/target does not have the given snapshots
dfs.allowSnapshot(source);
dfs.allowSnapshot(target);
Assert.assertFalse(sync());
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, context.getSourcePaths().get(0));
// reset source path in options
- options.setSourcePaths(Arrays.asList(source));
+ context.setSourcePaths(Collections.singletonList(source));
dfs.createSnapshot(source, "s1");
dfs.createSnapshot(source, "s2");
dfs.createSnapshot(target, "s1");
Assert.assertTrue(sync());
// reset source paths in options
- options.setSourcePaths(Arrays.asList(source));
+ context.setSourcePaths(Collections.singletonList(source));
// changes have been made in target
final Path subTarget = new Path(target, "sub");
dfs.mkdirs(subTarget);
Assert.assertFalse(sync());
// make sure the source path has been updated to the snapshot path
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, context.getSourcePaths().get(0));
// reset source paths in options
- options.setSourcePaths(Arrays.asList(source));
+ context.setSourcePaths(Collections.singletonList(source));
dfs.delete(subTarget, true);
Assert.assertTrue(sync());
}
@@ -137,7 +140,7 @@ public class TestDistCpSync {
}
private boolean sync() throws Exception {
- DistCpSync distCpSync = new DistCpSync(options, conf);
+ DistCpSync distCpSync = new DistCpSync(context, conf);
return distCpSync.sync();
}
@@ -231,7 +234,7 @@ public class TestDistCpSync {
SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2");
System.out.println(report);
- DistCpSync distCpSync = new DistCpSync(options, conf);
+ DistCpSync distCpSync = new DistCpSync(context, conf);
// do the sync
Assert.assertTrue(distCpSync.sync());
@@ -239,24 +242,24 @@ public class TestDistCpSync {
// make sure the source path has been updated to the snapshot path
final Path spath = new Path(source,
HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2");
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, context.getSourcePaths().get(0));
// build copy listing
final Path listingPath = new Path("/tmp/META/fileList.seq");
CopyListing listing = new SimpleCopyListing(conf, new Credentials(), distCpSync);
- listing.buildListing(listingPath, options);
+ listing.buildListing(listingPath, context);
Map<Text, CopyListingFileStatus> copyListing = getListing(listingPath);
CopyMapper copyMapper = new CopyMapper();
StubContext stubContext = new StubContext(conf, null, 0);
- Mapper<Text, CopyListingFileStatus, Text, Text>.Context context =
+ Mapper<Text, CopyListingFileStatus, Text, Text>.Context mapContext =
stubContext.getContext();
// Enable append
- context.getConfiguration().setBoolean(
+ mapContext.getConfiguration().setBoolean(
DistCpOptionSwitch.APPEND.getConfigLabel(), true);
- copyMapper.setup(context);
+ copyMapper.setup(mapContext);
for (Map.Entry<Text, CopyListingFileStatus> entry : copyListing.entrySet()) {
- copyMapper.map(entry.getKey(), entry.getValue(), context);
+ copyMapper.map(entry.getKey(), entry.getValue(), mapContext);
}
// verify that we only list modified and created files/directories
@@ -312,7 +315,12 @@ public class TestDistCpSync {
*/
@Test
public void testSyncWithCurrent() throws Exception {
- options.setUseDiff("s1", ".");
+ final DistCpOptions options = new DistCpOptions.Builder(
+ Collections.singletonList(source), target)
+ .withSyncFolder(true)
+ .withUseDiff("s1", ".")
+ .build();
+ context = new DistCpContext(options);
initData(source);
initData(target);
enableAndCreateFirstSnapshot();
@@ -323,7 +331,7 @@ public class TestDistCpSync {
// do the sync
sync();
// make sure the source path is still unchanged
- Assert.assertEquals(source, options.getSourcePaths().get(0));
+ Assert.assertEquals(source, context.getSourcePaths().get(0));
}
private void initData2(Path dir) throws Exception {
@@ -501,32 +509,32 @@ public class TestDistCpSync {
SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2");
System.out.println(report);
- DistCpSync distCpSync = new DistCpSync(options, conf);
+ DistCpSync distCpSync = new DistCpSync(context, conf);
// do the sync
Assert.assertTrue(distCpSync.sync());
// make sure the source path has been updated to the snapshot path
final Path spath = new Path(source,
HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2");
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, context.getSourcePaths().get(0));
// build copy listing
final Path listingPath = new Path("/tmp/META/fileList.seq");
CopyListing listing = new SimpleCopyListing(conf, new Credentials(), distCpSync);
- listing.buildListing(listingPath, options);
+ listing.buildListing(listingPath, context);
Map<Text, CopyListingFileStatus> copyListing = getListing(listingPath);
CopyMapper copyMapper = new CopyMapper();
StubContext stubContext = new StubContext(conf, null, 0);
- Mapper<Text, CopyListingFileStatus, Text, Text>.Context context =
+ Mapper<Text, CopyListingFileStatus, Text, Text>.Context mapContext =
stubContext.getContext();
// Enable append
- context.getConfiguration().setBoolean(
+ mapContext.getConfiguration().setBoolean(
DistCpOptionSwitch.APPEND.getConfigLabel(), true);
- copyMapper.setup(context);
+ copyMapper.setup(mapContext);
for (Map.Entry<Text, CopyListingFileStatus> entry :
copyListing.entrySet()) {
- copyMapper.map(entry.getKey(), entry.getValue(), context);
+ copyMapper.map(entry.getKey(), entry.getValue(), mapContext);
}
// verify that we only list modified and created files/directories
@@ -729,7 +737,7 @@ public class TestDistCpSync {
boolean threwException = false;
try {
- DistCpSync distCpSync = new DistCpSync(options, conf);
+ DistCpSync distCpSync = new DistCpSync(context, conf);
// do the sync
distCpSync.sync();
} catch (HadoopIllegalArgumentException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java
index fea374e..cca1c53 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java
@@ -56,7 +56,8 @@ public abstract class TestDistCpSyncReverseBase {
private MiniDFSCluster cluster;
private final Configuration conf = new HdfsConfiguration();
private DistributedFileSystem dfs;
- private DistCpOptions options;
+ private DistCpOptions.Builder optionsBuilder;
+ private DistCpContext distCpContext;
private Path source;
private boolean isSrcNotSameAsTgt = true;
private final Path target = new Path("/target");
@@ -139,10 +140,12 @@ public abstract class TestDistCpSyncReverseBase {
}
dfs.mkdirs(target);
- options = new DistCpOptions(Arrays.asList(source), target);
- options.setSyncFolder(true);
- options.setUseRdiff("s2", "s1");
+ optionsBuilder = new DistCpOptions.Builder(Arrays.asList(source), target)
+ .withSyncFolder(true)
+ .withUseRdiff("s2", "s1");
+ final DistCpOptions options = optionsBuilder.build();
options.appendToConf(conf);
+ distCpContext = new DistCpContext(options);
conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, target.toString());
conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, target.toString());
@@ -169,33 +172,33 @@ public abstract class TestDistCpSyncReverseBase {
// make sure the source path has been updated to the snapshot path
final Path spath = new Path(source,
HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s1");
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0));
// reset source path in options
- options.setSourcePaths(Arrays.asList(source));
+ optionsBuilder.withSourcePaths(Arrays.asList(source));
// the source/target does not have the given snapshots
dfs.allowSnapshot(source);
dfs.allowSnapshot(target);
Assert.assertFalse(sync());
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0));
// reset source path in options
- options.setSourcePaths(Arrays.asList(source));
+ optionsBuilder.withSourcePaths(Arrays.asList(source));
this.enableAndCreateFirstSnapshot();
dfs.createSnapshot(target, "s2");
Assert.assertTrue(sync());
// reset source paths in options
- options.setSourcePaths(Arrays.asList(source));
+ optionsBuilder.withSourcePaths(Arrays.asList(source));
// changes have been made in target
final Path subTarget = new Path(target, "sub");
dfs.mkdirs(subTarget);
Assert.assertFalse(sync());
// make sure the source path has been updated to the snapshot path
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0));
// reset source paths in options
- options.setSourcePaths(Arrays.asList(source));
+ optionsBuilder.withSourcePaths(Arrays.asList(source));
dfs.delete(subTarget, true);
Assert.assertTrue(sync());
}
@@ -215,7 +218,8 @@ public abstract class TestDistCpSyncReverseBase {
}
private boolean sync() throws Exception {
- DistCpSync distCpSync = new DistCpSync(options, conf);
+ distCpContext = new DistCpContext(optionsBuilder.build());
+ final DistCpSync distCpSync = new DistCpSync(distCpContext, conf);
return distCpSync.sync();
}
@@ -328,7 +332,7 @@ public abstract class TestDistCpSyncReverseBase {
SnapshotDiffReport report = dfs.getSnapshotDiffReport(target, "s2", "s1");
System.out.println(report);
- DistCpSync distCpSync = new DistCpSync(options, conf);
+ final DistCpSync distCpSync = new DistCpSync(distCpContext, conf);
lsr("Before sync target: ", shell, target);
@@ -340,13 +344,13 @@ public abstract class TestDistCpSyncReverseBase {
// make sure the source path has been updated to the snapshot path
final Path spath = new Path(source,
HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s1");
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0));
// build copy listing
final Path listingPath = new Path("/tmp/META/fileList.seq");
CopyListing listing = new SimpleCopyListing(conf, new Credentials(),
distCpSync);
- listing.buildListing(listingPath, options);
+ listing.buildListing(listingPath, distCpContext);
Map<Text, CopyListingFileStatus> copyListing = getListing(listingPath);
CopyMapper copyMapper = new CopyMapper();
@@ -425,7 +429,7 @@ public abstract class TestDistCpSyncReverseBase {
*/
@Test
public void testSyncWithCurrent() throws Exception {
- options.setUseRdiff(".", "s1");
+ optionsBuilder.withUseRdiff(".", "s1");
if (isSrcNotSameAsTgt) {
initData(source);
}
@@ -440,7 +444,7 @@ public abstract class TestDistCpSyncReverseBase {
final Path spath = new Path(source,
HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s1");
// make sure the source path is still unchanged
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0));
}
private void initData2(Path dir) throws Exception {
@@ -649,7 +653,7 @@ public abstract class TestDistCpSyncReverseBase {
lsrSource("Before sync source: ", shell, source);
lsr("Before sync target: ", shell, target);
- DistCpSync distCpSync = new DistCpSync(options, conf);
+ DistCpSync distCpSync = new DistCpSync(distCpContext, conf);
// do the sync
distCpSync.sync();
@@ -658,12 +662,12 @@ public abstract class TestDistCpSyncReverseBase {
// make sure the source path has been updated to the snapshot path
final Path spath = new Path(source,
HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s1");
- Assert.assertEquals(spath, options.getSourcePaths().get(0));
+ Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0));
// build copy listing
final Path listingPath = new Path("/tmp/META/fileList.seq");
CopyListing listing = new SimpleCopyListing(conf, new Credentials(), distCpSync);
- listing.buildListing(listingPath, options);
+ listing.buildListing(listingPath, distCpContext);
Map<Text, CopyListingFileStatus> copyListing = getListing(listingPath);
CopyMapper copyMapper = new CopyMapper();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
index 5511e09..d6d0542 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
@@ -413,11 +413,13 @@ public class TestDistCpViewFs {
private void runTest(Path listFile, Path target, boolean targetExists,
boolean sync) throws IOException {
- DistCpOptions options = new DistCpOptions(listFile, target);
- options.setSyncFolder(sync);
- options.setTargetPathExists(targetExists);
+ final DistCpOptions options = new DistCpOptions.Builder(listFile, target)
+ .withSyncFolder(sync)
+ .build();
try {
- new DistCp(getConf(), options).execute();
+ final DistCp distcp = new DistCp(getConf(), options);
+ distcp.context.setTargetPathExists(targetExists);
+ distcp.execute();
} catch (Exception e) {
LOG.error("Exception encountered ", e);
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java
index fe2c668..203de1a 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java
@@ -514,10 +514,11 @@ public class TestFileBasedCopyListing {
private void runTest(Path listFile, Path target, boolean targetExists,
boolean sync) throws IOException {
CopyListing listing = new FileBasedCopyListing(config, CREDENTIALS);
- DistCpOptions options = new DistCpOptions(listFile, target);
- options.setSyncFolder(sync);
- options.setTargetPathExists(targetExists);
- listing.buildListing(listFile, options);
+ final DistCpOptions options = new DistCpOptions.Builder(listFile, target)
+ .withSyncFolder(sync).build();
+ final DistCpContext context = new DistCpContext(options);
+ context.setTargetPathExists(targetExists);
+ listing.buildListing(listFile, context);
}
private void checkResult(Path listFile, int count) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java
index 6c03b4e..1c92a9c 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java
@@ -34,7 +34,7 @@ import org.junit.Test;
import java.io.DataOutputStream;
import java.net.URI;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -109,9 +109,12 @@ public class TestGlobbedCopyListing {
Path source = new Path(fileSystemPath.toString() + "/tmp/source");
Path target = new Path(fileSystemPath.toString() + "/tmp/target");
Path listingPath = new Path(fileSystemPath.toString() + "/tmp/META/fileList.seq");
- DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);
- options.setTargetPathExists(false);
- new GlobbedCopyListing(new Configuration(), CREDENTIALS).buildListing(listingPath, options);
+ DistCpOptions options = new DistCpOptions.Builder(
+ Collections.singletonList(source), target).build();
+ DistCpContext context = new DistCpContext(options);
+ context.setTargetPathExists(false);
+ new GlobbedCopyListing(new Configuration(), CREDENTIALS)
+ .buildListing(listingPath, context);
verifyContents(listingPath);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
index ee8e7cc..7574ded 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Cluster;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
-import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.tools.util.TestDistCpUtils;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -493,7 +492,8 @@ public class TestIntegration {
List<Path> sources = new ArrayList<Path>();
sources.add(sourcePath);
- DistCpOptions options = new DistCpOptions(sources, target);
+ DistCpOptions options = new DistCpOptions.Builder(sources, target)
+ .build();
Configuration conf = getConf();
Path stagingDir = JobSubmissionFiles.getStagingDir(
@@ -559,14 +559,16 @@ public class TestIntegration {
private void runTest(Path listFile, Path target, boolean targetExists,
boolean sync, boolean delete,
boolean overwrite) throws IOException {
- DistCpOptions options = new DistCpOptions(listFile, target);
- options.setSyncFolder(sync);
- options.setDeleteMissing(delete);
- options.setOverwrite(overwrite);
- options.setTargetPathExists(targetExists);
- options.setNumListstatusThreads(numListstatusThreads);
+ final DistCpOptions options = new DistCpOptions.Builder(listFile, target)
+ .withSyncFolder(sync)
+ .withDeleteMissing(delete)
+ .withOverwrite(overwrite)
+ .withNumListstatusThreads(numListstatusThreads)
+ .build();
try {
- new DistCp(getConf(), options).execute();
+ final DistCp distCp = new DistCp(getConf(), options);
+ distCp.context.setTargetPathExists(targetExists);
+ distCp.execute();
} catch (Exception e) {
LOG.error("Exception encountered ", e);
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index f94ba97..e7fdc51 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.tools;
-import static org.junit.Assert.assertFalse;
+import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.junit.Assert.fail;
import org.junit.Assert;
@@ -28,7 +28,6 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.tools.DistCpOptions.*;
import org.apache.hadoop.conf.Configuration;
-import java.util.Iterator;
import java.util.NoSuchElementException;
public class TestOptionsParser {
@@ -329,7 +328,7 @@ public class TestOptionsParser {
"100",
"hdfs://localhost:9820/source/first",
"hdfs://localhost:9820/target/"});
- Assert.assertEquals(DistCpOptions.maxNumListstatusThreads,
+ Assert.assertEquals(DistCpOptions.MAX_NUM_LISTSTATUS_THREADS,
options.getNumListstatusThreads());
}
@@ -383,25 +382,6 @@ public class TestOptionsParser {
}
@Test
- public void testToString() {
- DistCpOptions option = new DistCpOptions(new Path("abc"), new Path("xyz"));
- String val = "DistCpOptions{atomicCommit=false, syncFolder=false, "
- + "deleteMissing=false, ignoreFailures=false, overwrite=false, "
- + "append=false, useDiff=false, useRdiff=false, "
- + "fromSnapshot=null, toSnapshot=null, "
- + "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, "
- + "mapBandwidth=0.0, "
- + "copyStrategy='uniformsize', preserveStatus=[], "
- + "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, "
- + "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, "
- + "targetPathExists=true, filtersFile='null', blocksPerChunk=0}";
- String optionString = option.toString();
- Assert.assertEquals(val, optionString);
- Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
- DistCpOptionSwitch.ATOMIC_COMMIT.name());
- }
-
- @Test
public void testCopyStrategy() {
DistCpOptions options = OptionsParser.parse(new String[] {
"-strategy",
@@ -529,13 +509,8 @@ public class TestOptionsParser {
"-f",
"hdfs://localhost:9820/source/first",
"hdfs://localhost:9820/target/"});
- int i = 0;
- Iterator<FileAttribute> attribIterator = options.preserveAttributes();
- while (attribIterator.hasNext()) {
- attribIterator.next();
- i++;
- }
- Assert.assertEquals(i, DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.length() - 2);
+ Assert.assertEquals(DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.length() - 2,
+ options.getPreserveAttributes().size());
try {
OptionsParser.parse(new String[] {
@@ -545,19 +520,18 @@ public class TestOptionsParser {
"hdfs://localhost:9820/target"});
Assert.fail("Invalid preserve attribute");
}
- catch (IllegalArgumentException ignore) {}
catch (NoSuchElementException ignore) {}
- options = OptionsParser.parse(new String[] {
- "-f",
- "hdfs://localhost:9820/source/first",
- "hdfs://localhost:9820/target/"});
- Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION));
- options.preserve(FileAttribute.PERMISSION);
- Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION));
+ Builder builder = new DistCpOptions.Builder(
+ new Path("hdfs://localhost:9820/source/first"),
+ new Path("hdfs://localhost:9820/target/"));
+ Assert.assertFalse(
+ builder.build().shouldPreserve(FileAttribute.PERMISSION));
+ builder.preserve(FileAttribute.PERMISSION);
+ Assert.assertTrue(builder.build().shouldPreserve(FileAttribute.PERMISSION));
- options.preserve(FileAttribute.PERMISSION);
- Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION));
+ builder.preserve(FileAttribute.PERMISSION);
+ Assert.assertTrue(builder.build().shouldPreserve(FileAttribute.PERMISSION));
}
@Test
@@ -756,28 +730,25 @@ public class TestOptionsParser {
}
try {
- options = OptionsParser.parse(new String[] {
- optionStr, "s1", "s2", "-update", "-delete",
+ OptionsParser.parse(new String[] {
+ "-diff", "s1", "s2", "-update", "-delete",
"hdfs://localhost:9820/source/first",
"hdfs://localhost:9820/target/" });
- assertFalse("-delete should be ignored when "
- + optionStr + " is specified",
- options.shouldDeleteMissing());
+ fail("Should fail as -delete and -diff/-rdiff are mutually exclusive");
} catch (IllegalArgumentException e) {
- fail("Got unexpected IllegalArgumentException: " + e.getMessage());
+ assertExceptionContains(
+ "-delete and -diff/-rdiff are mutually exclusive", e);
}
try {
- options = OptionsParser.parse(new String[] {
- optionStr, "s1", "s2", "-delete",
+ OptionsParser.parse(new String[] {
+ "-diff", "s1", "s2", "-delete",
"hdfs://localhost:9820/source/first",
"hdfs://localhost:9820/target/" });
- fail(optionStr + " should fail if -update option is not specified");
+ fail("Should fail as -delete and -diff/-rdiff are mutually exclusive");
} catch (IllegalArgumentException e) {
- assertFalse("-delete should be ignored when -diff is specified",
- options.shouldDeleteMissing());
- GenericTestUtils.assertExceptionContains(
- "-diff/-rdiff is valid only with -update option", e);
+ assertExceptionContains(
+ "-delete and -diff/-rdiff are mutually exclusive", e);
}
try {
@@ -785,10 +756,10 @@ public class TestOptionsParser {
"-delete", "-overwrite",
"hdfs://localhost:9820/source/first",
"hdfs://localhost:9820/target/" });
- fail(optionStr + " should fail if -update option is not specified");
+ fail("Should fail as -delete and -diff are mutually exclusive");
} catch (IllegalArgumentException e) {
- GenericTestUtils.assertExceptionContains(
- "-diff/-rdiff is valid only with -update option", e);
+ assertExceptionContains(
+ "-delete and -diff/-rdiff are mutually exclusive", e);
}
final String optionStrOther = isDiff? "-rdiff" : "-diff";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
index 21a14d3..fb1a64d 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
@@ -19,9 +19,8 @@
package org.apache.hadoop.tools.contract;
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.junit.Assert.*;
-import java.util.Arrays;
+import java.util.Collections;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -184,7 +183,8 @@ public abstract class AbstractContractDistCpTest
* @throws Exception if there is a failure
*/
private void runDistCp(Path src, Path dst) throws Exception {
- DistCpOptions options = new DistCpOptions(Arrays.asList(src), dst);
+ DistCpOptions options = new DistCpOptions.Builder(
+ Collections.singletonList(src), dst).build();
Job job = new DistCp(conf, options).execute();
assertNotNull("Unexpected null job returned from DistCp execution.", job);
assertTrue("DistCp job did not complete.", job.isComplete());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
index 2452d6f..6ee37cc 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.tools.CopyListing;
import org.apache.hadoop.tools.DistCpConstants;
+import org.apache.hadoop.tools.DistCpContext;
import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.GlobbedCopyListing;
@@ -146,15 +147,16 @@ public class TestCopyCommitter {
sourceBase = TestDistCpUtils.createTestSetup(fs, sourcePerm);
targetBase = TestDistCpUtils.createTestSetup(fs, initialPerm);
- DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)),
- new Path("/out"));
- options.preserve(FileAttribute.PERMISSION);
+ final DistCpOptions options = new DistCpOptions.Builder(
+ Collections.singletonList(new Path(sourceBase)), new Path("/out"))
+ .preserve(FileAttribute.PERMISSION).build();
options.appendToConf(conf);
- options.setTargetPathExists(false);
-
+ final DistCpContext context = new DistCpContext(options);
+ context.setTargetPathExists(false);
+
CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
- listing.buildListing(listingFile, options);
+ listing.buildListing(listingFile, context);
conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);
@@ -197,15 +199,15 @@ public class TestCopyCommitter {
String targetBaseAdd = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault());
fs.rename(new Path(targetBaseAdd), new Path(targetBase));
- DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)),
- new Path("/out"));
- options.setSyncFolder(true);
- options.setDeleteMissing(true);
+ final DistCpOptions options = new DistCpOptions.Builder(
+ Collections.singletonList(new Path(sourceBase)), new Path("/out"))
+ .withSyncFolder(true).withDeleteMissing(true).build();
options.appendToConf(conf);
+ final DistCpContext context = new DistCpContext(options);
CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
- listing.buildListing(listingFile, options);
+ listing.buildListing(listingFile, context);
conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);
conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase);
@@ -266,15 +268,15 @@ public class TestCopyCommitter {
TestDistCpUtils.createFile(fs, targetBase + "/9");
TestDistCpUtils.createFile(fs, targetBase + "/A");
- DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)),
- new Path("/out"));
- options.setSyncFolder(true);
- options.setDeleteMissing(true);
+ final DistCpOptions options = new DistCpOptions.Builder(
+ Collections.singletonList(new Path(sourceBase)), new Path("/out"))
+ .withSyncFolder(true).withDeleteMissing(true).build();
options.appendToConf(conf);
+ final DistCpContext context = new DistCpContext(options);
CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
- listing.buildListing(listingFile, options);
+ listing.buildListing(listingFile, context);
conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);
conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
index 78e2262..5315137 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.tools.CopyListing;
import org.apache.hadoop.tools.CopyListingFileStatus;
+import org.apache.hadoop.tools.DistCpContext;
import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.tools.StubContext;
import org.apache.hadoop.security.Credentials;
@@ -74,9 +75,9 @@ public class TestUniformSizeInputFormat {
List<Path> sourceList = new ArrayList<Path>();
sourceList.add(sourcePath);
- final DistCpOptions distCpOptions = new DistCpOptions(sourceList, targetPath);
- distCpOptions.setMaxMaps(nMaps);
- return distCpOptions;
+ return new DistCpOptions.Builder(sourceList, targetPath)
+ .maxMaps(nMaps)
+ .build();
}
private static int createFile(String path, int fileSize) throws Exception {
@@ -100,14 +101,14 @@ public class TestUniformSizeInputFormat {
}
public void testGetSplits(int nMaps) throws Exception {
- DistCpOptions options = getOptions(nMaps);
+ DistCpContext context = new DistCpContext(getOptions(nMaps));
Configuration configuration = new Configuration();
configuration.set("mapred.map.tasks",
- String.valueOf(options.getMaxMaps()));
+ String.valueOf(context.getMaxMaps()));
Path listFile = new Path(cluster.getFileSystem().getUri().toString()
+ "/tmp/testGetSplits_1/fileList.seq");
- CopyListing.getCopyListing(configuration, CREDENTIALS, options).
- buildListing(listFile, options);
+ CopyListing.getCopyListing(configuration, CREDENTIALS, context)
+ .buildListing(listFile, context);
JobContext jobContext = new JobContextImpl(configuration, new JobID());
UniformSizeInputFormat uniformSizeInputFormat = new UniformSizeInputFormat();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26172a94/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
index bb2dd9d..87290ca 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.tools.mapred.lib;
import org.apache.hadoop.tools.DistCpConstants;
+import org.apache.hadoop.tools.DistCpContext;
import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -84,9 +85,9 @@ public class TestDynamicInputFormat {
List<Path> sourceList = new ArrayList<Path>();
sourceList.add(sourcePath);
- DistCpOptions options = new DistCpOptions(sourceList, targetPath);
- options.setMaxMaps(NUM_SPLITS);
- return options;
+ return new DistCpOptions.Builder(sourceList, targetPath)
+ .maxMaps(NUM_SPLITS)
+ .build();
}
private static void createFile(String path) throws Exception {
@@ -110,13 +111,13 @@ public class TestDynamicInputFormat {
@Test
public void testGetSplits() throws Exception {
- DistCpOptions options = getOptions();
+ final DistCpContext context = new DistCpContext(getOptions());
Configuration configuration = new Configuration();
configuration.set("mapred.map.tasks",
- String.valueOf(options.getMaxMaps()));
- CopyListing.getCopyListing(configuration, CREDENTIALS, options).buildListing(
- new Path(cluster.getFileSystem().getUri().toString()
- +"/tmp/testDynInputFormat/fileList.seq"), options);
+ String.valueOf(context.getMaxMaps()));
+ CopyListing.getCopyListing(configuration, CREDENTIALS, context)
+ .buildListing(new Path(cluster.getFileSystem().getUri().toString()
+ +"/tmp/testDynInputFormat/fileList.seq"), context);
JobContext jobContext = new JobContextImpl(configuration, new JobID());
DynamicInputFormat<Text, CopyListingFileStatus> inputFormat =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HADOOP-11794. Enable distcp to
copy blocks in parallel. Contributed by Yongjun Zhang, Wei-Chiu Chuang,
Xiao Chen, Rosie Li.
Posted by ae...@apache.org.
HADOOP-11794. Enable distcp to copy blocks in parallel. Contributed by Yongjun Zhang, Wei-Chiu Chuang, Xiao Chen, Rosie Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf3fb585
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf3fb585
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf3fb585
Branch: refs/heads/HDFS-7240
Commit: bf3fb585aaf2b179836e139c041fc87920a3c886
Parents: 144f1cf
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Thu Mar 30 17:01:15 2017 -0700
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Thu Mar 30 17:38:56 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 22 +-
.../org/apache/hadoop/tools/CopyListing.java | 37 +-
.../hadoop/tools/CopyListingFileStatus.java | 87 ++++-
.../java/org/apache/hadoop/tools/DistCp.java | 52 +++
.../apache/hadoop/tools/DistCpOptionSwitch.java | 10 +
.../org/apache/hadoop/tools/DistCpOptions.java | 22 +-
.../org/apache/hadoop/tools/OptionsParser.java | 36 +-
.../apache/hadoop/tools/SimpleCopyListing.java | 83 +++--
.../hadoop/tools/mapred/CopyCommitter.java | 174 ++++++++-
.../apache/hadoop/tools/mapred/CopyMapper.java | 40 +-
.../tools/mapred/RetriableFileCopyCommand.java | 26 +-
.../tools/mapred/UniformSizeInputFormat.java | 5 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 111 +++++-
.../src/site/markdown/DistCp.md.vm | 1 +
.../apache/hadoop/tools/TestDistCpSystem.java | 368 +++++++++++++++++--
.../apache/hadoop/tools/TestOptionsParser.java | 2 +-
.../hadoop/tools/mapred/TestCopyCommitter.java | 5 +-
17 files changed, 971 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 1329195..9b782f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -862,7 +862,27 @@ public class DFSTestUtil {
out.write(toAppend);
}
}
-
+
+ /**
+ * Append specified length of bytes to a given file, starting with new block.
+ * @param fs The file system
+ * @param p Path of the file to append
+ * @param length Length of bytes to append to the file
+ * @throws IOException
+ */
+ public static void appendFileNewBlock(DistributedFileSystem fs,
+ Path p, int length) throws IOException {
+ assert fs.exists(p);
+ assert length >= 0;
+ byte[] toAppend = new byte[length];
+ Random random = new Random();
+ random.nextBytes(toAppend);
+ try (FSDataOutputStream out = fs.append(p,
+ EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null)) {
+ out.write(toAppend);
+ }
+ }
+
/**
* @return url content as string (UTF-8 encoding assumed)
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
index 481aa61..9ebf9d2 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
@@ -145,12 +145,22 @@ public abstract class CopyListing extends Configured {
Configuration config = getConf();
FileSystem fs = pathToListFile.getFileSystem(config);
- Path sortedList = DistCpUtils.sortListing(fs, config, pathToListFile);
+ final boolean splitLargeFile = options.splitLargeFile();
+
+ // When splitLargeFile is enabled, we don't randomize the copylist
+ // earlier, so we don't do the sorting here. For a file that has
+ // multiple entries due to split, we check here that their
+ // <chunkOffset, chunkLength> is continuous.
+ //
+ Path checkPath = splitLargeFile?
+ pathToListFile : DistCpUtils.sortListing(fs, config, pathToListFile);
SequenceFile.Reader reader = new SequenceFile.Reader(
- config, SequenceFile.Reader.file(sortedList));
+ config, SequenceFile.Reader.file(checkPath));
try {
Text lastKey = new Text("*"); //source relative path can never hold *
+ long lastChunkOffset = -1;
+ long lastChunkLength = -1;
CopyListingFileStatus lastFileStatus = new CopyListingFileStatus();
Text currentKey = new Text();
@@ -161,8 +171,21 @@ public abstract class CopyListing extends Configured {
if (currentKey.equals(lastKey)) {
CopyListingFileStatus currentFileStatus = new CopyListingFileStatus();
reader.getCurrentValue(currentFileStatus);
- throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " +
- currentFileStatus.getPath() + " would cause duplicates. Aborting");
+ if (!splitLargeFile) {
+ throw new DuplicateFileException("File " + lastFileStatus.getPath()
+ + " and " + currentFileStatus.getPath()
+ + " would cause duplicates. Aborting");
+ } else {
+ if (lastChunkOffset + lastChunkLength !=
+ currentFileStatus.getChunkOffset()) {
+ throw new InvalidInputException("File " + lastFileStatus.getPath()
+ + " " + lastChunkOffset + "," + lastChunkLength
+ + " and " + currentFileStatus.getPath()
+ + " " + currentFileStatus.getChunkOffset() + ","
+ + currentFileStatus.getChunkLength()
+ + " are not continuous. Aborting");
+ }
+ }
}
reader.getCurrentValue(lastFileStatus);
if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
@@ -181,8 +204,12 @@ public abstract class CopyListing extends Configured {
xAttrSupportCheckFsSet.add(lastFsUri);
}
}
- lastKey.set(currentKey);
+ lastKey.set(currentKey);
+ if (splitLargeFile) {
+ lastChunkOffset = lastFileStatus.getChunkOffset();
+ lastChunkLength = lastFileStatus.getChunkLength();
+ }
if (options.shouldUseDiff() && LOG.isDebugEnabled()) {
LOG.debug("Copy list entry " + idx + ": " +
lastFileStatus.getPath().toUri().getPath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
index 00d4b32..29c59ac 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
@@ -74,6 +74,14 @@ public final class CopyListingFileStatus implements Writable {
private List<AclEntry> aclEntries;
private Map<String, byte[]> xAttrs;
+ // <chunkOffset, chunkLength> represents the offset and length of a file
+ // chunk in number of bytes.
+ // used when splitting a large file to chunks to copy in parallel.
+ // If a file is not large enough to split, chunkOffset would be 0 and
+ // chunkLength would be the length of the file.
+ private long chunkOffset = 0;
+ private long chunkLength = Long.MAX_VALUE;
+
/**
* Default constructor.
*/
@@ -96,11 +104,32 @@ public final class CopyListingFileStatus implements Writable {
fileStatus.getPath());
}
+ public CopyListingFileStatus(FileStatus fileStatus,
+ long chunkOffset, long chunkLength) {
+ this(fileStatus.getLen(), fileStatus.isDirectory(),
+ fileStatus.getReplication(), fileStatus.getBlockSize(),
+ fileStatus.getModificationTime(), fileStatus.getAccessTime(),
+ fileStatus.getPermission(), fileStatus.getOwner(),
+ fileStatus.getGroup(),
+ fileStatus.getPath());
+ this.chunkOffset = chunkOffset;
+ this.chunkLength = chunkLength;
+ }
+
@SuppressWarnings("checkstyle:parameternumber")
public CopyListingFileStatus(long length, boolean isdir,
int blockReplication, long blocksize, long modificationTime,
long accessTime, FsPermission permission, String owner, String group,
Path path) {
+ this(length, isdir, blockReplication, blocksize, modificationTime,
+ accessTime, permission, owner, group, path, 0, Long.MAX_VALUE);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ public CopyListingFileStatus(long length, boolean isdir,
+ int blockReplication, long blocksize, long modificationTime,
+ long accessTime, FsPermission permission, String owner, String group,
+ Path path, long chunkOffset, long chunkLength) {
this.length = length;
this.isdir = isdir;
this.blockReplication = (short)blockReplication;
@@ -117,6 +146,23 @@ public final class CopyListingFileStatus implements Writable {
this.owner = (owner == null) ? "" : owner;
this.group = (group == null) ? "" : group;
this.path = path;
+ this.chunkOffset = chunkOffset;
+ this.chunkLength = chunkLength;
+ }
+
+ public CopyListingFileStatus(CopyListingFileStatus other) {
+ this.length = other.length;
+ this.isdir = other.isdir;
+ this.blockReplication = other.blockReplication;
+ this.blocksize = other.blocksize;
+ this.modificationTime = other.modificationTime;
+ this.accessTime = other.accessTime;
+ this.permission = other.permission;
+ this.owner = other.owner;
+ this.group = other.group;
+ this.path = new Path(other.path.toUri());
+ this.chunkOffset = other.chunkOffset;
+ this.chunkLength = other.chunkLength;
}
public Path getPath() {
@@ -200,6 +246,31 @@ public final class CopyListingFileStatus implements Writable {
this.xAttrs = xAttrs;
}
+ public long getChunkOffset() {
+ return chunkOffset;
+ }
+
+ public void setChunkOffset(long offset) {
+ this.chunkOffset = offset;
+ }
+
+ public long getChunkLength() {
+ return chunkLength;
+ }
+
+ public void setChunkLength(long chunkLength) {
+ this.chunkLength = chunkLength;
+ }
+
+ public boolean isSplit() {
+ return getChunkLength() != Long.MAX_VALUE &&
+ getChunkLength() != getLen();
+ }
+
+ public long getSizeToCopy() {
+ return isSplit()? getChunkLength() : getLen();
+ }
+
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN);
@@ -244,6 +315,9 @@ public final class CopyListingFileStatus implements Writable {
} else {
out.writeInt(NO_XATTRS);
}
+
+ out.writeLong(chunkOffset);
+ out.writeLong(chunkLength);
}
@Override
@@ -292,6 +366,9 @@ public final class CopyListingFileStatus implements Writable {
} else {
xAttrs = null;
}
+
+ chunkOffset = in.readLong();
+ chunkLength = in.readLong();
}
@Override
@@ -317,8 +394,14 @@ public final class CopyListingFileStatus implements Writable {
public String toString() {
StringBuilder sb = new StringBuilder(super.toString());
sb.append('{');
- sb.append("aclEntries = " + aclEntries);
- sb.append(", xAttrs = " + xAttrs);
+ sb.append(this.getPath().toString());
+ sb.append(" length = ").append(this.getLen());
+ sb.append(" aclEntries = ").append(aclEntries);
+ sb.append(", xAttrs = ").append(xAttrs);
+ if (isSplit()) {
+ sb.append(", chunkOffset = ").append(this.getChunkOffset());
+ sb.append(", chunkLength = ").append(this.getChunkLength());
+ }
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index ab58e9c..8c2fa24 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.Cluster;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
+import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.CopyListing.*;
import org.apache.hadoop.tools.mapred.CopyMapper;
import org.apache.hadoop.tools.mapred.CopyOutputFormat;
@@ -134,6 +135,7 @@ public class DistCp extends Configured implements Tool {
try {
inputOptions = (OptionsParser.parse(argv));
+ setOptionsForSplitLargeFile();
setTargetPathExists();
LOG.info("Input Options: " + inputOptions);
} catch (Throwable e) {
@@ -235,6 +237,56 @@ public class DistCp extends Configured implements Tool {
getConf().setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS,
targetExists);
}
+
+ /**
+ * Check if concat is supported by fs.
+ * Throws UnsupportedOperationException if not.
+ */
+ private void checkConcatSupport(FileSystem fs) {
+ try {
+ Path[] src = null;
+ Path tgt = null;
+ fs.concat(tgt, src);
+ } catch (UnsupportedOperationException use) {
+ throw new UnsupportedOperationException(
+ DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() +
+ " is not supported since the target file system doesn't" +
+ " support concat.", use);
+ } catch (Exception e) {
+ // Ignore other exception
+ }
+ }
+
+ /**
+ * Set up needed options for splitting large files.
+ */
+ private void setOptionsForSplitLargeFile() throws IOException {
+ if (!inputOptions.splitLargeFile()) {
+ return;
+ }
+ Path target = inputOptions.getTargetPath();
+ FileSystem targetFS = target.getFileSystem(getConf());
+ checkConcatSupport(targetFS);
+
+ LOG.info("Enabling preserving blocksize since "
+ + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " is passed.");
+ inputOptions.preserve(FileAttribute.BLOCKSIZE);
+
+ LOG.info("Set " +
+ DistCpOptionSwitch.APPEND.getSwitch()
+ + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()
+ + " is passed.");
+ inputOptions.setAppend(false);
+
+ LOG.info("Set " +
+ DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES
+ + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()
+ + " is passed.");
+ getConf().setBoolean(
+ DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false);
+ }
+
+
/**
* Create Job object for submitting it, with all the configuration
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index fb47d76..ced9b54 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -169,6 +169,16 @@ public enum DistCpOptionSwitch {
new Option("sizelimit", true, "(Deprecated!) Limit number of files " +
"copied to <= n bytes")),
+ BLOCKS_PER_CHUNK("",
+ new Option("blocksperchunk", true, "If set to a positive value, files"
+ + "with more blocks than this value will be split into chunks of "
+ + "<blocksperchunk> blocks to be transferred in parallel, and "
+ + "reassembled on the destination. By default, <blocksperchunk> is "
+ + "0 and the files will be transmitted in their entirety without "
+ + "splitting. This switch is only applicable when the source file "
+ + "system implements getBlockLocations method and the target file "
+ + "system implements concat method")),
+
/**
* Specify bandwidth per map in MB, accepts bandwidth as a fraction
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index 8c37ff3..9822d83 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -97,7 +97,11 @@ public class DistCpOptions {
// targetPathExist is a derived field, it's initialized in the
// beginning of distcp.
private boolean targetPathExists = true;
-
+
+ // Size of chunk in number of blocks when splitting large file into chunks
+ // to copy in parallel. Default is 0 and file are not splitted.
+ private int blocksPerChunk = 0;
+
public static enum FileAttribute{
REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES;
@@ -166,6 +170,7 @@ public class DistCpOptions {
this.targetPath = that.getTargetPath();
this.targetPathExists = that.getTargetPathExists();
this.filtersFile = that.getFiltersFile();
+ this.blocksPerChunk = that.blocksPerChunk;
}
}
@@ -578,6 +583,18 @@ public class DistCpOptions {
this.filtersFile = filtersFilename;
}
+ public final void setBlocksPerChunk(int csize) {
+ this.blocksPerChunk = csize;
+ }
+
+ public final int getBlocksPerChunk() {
+ return blocksPerChunk;
+ }
+
+ public final boolean splitLargeFile() {
+ return blocksPerChunk > 0;
+ }
+
void validate() {
if ((useDiff || useRdiff) && deleteMissing) {
// -delete and -diff/-rdiff are mutually exclusive. For backward
@@ -669,6 +686,8 @@ public class DistCpOptions {
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.FILTERS,
filtersFile);
}
+ DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BLOCKS_PER_CHUNK,
+ String.valueOf(blocksPerChunk));
}
/**
@@ -704,6 +723,7 @@ public class DistCpOptions {
", targetPath=" + targetPath +
", targetPathExists=" + targetPathExists +
", filtersFile='" + filtersFile + '\'' +
+ ", blocksPerChunk=" + blocksPerChunk +
'}';
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
index d0f82ca..8881264 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
@@ -172,11 +172,44 @@ public class OptionsParser {
DistCpOptionSwitch.FILTERS.getSwitch()));
}
+ parseBlocksPerChunk(command, option);
+
option.validate();
return option;
}
+
+ /**
+ * A helper method to parse chunk size in number of blocks.
+ * Used when breaking large file into chunks to copy in parallel.
+ *
+ * @param command command line arguments
+ */
+ private static void parseBlocksPerChunk(CommandLine command,
+ DistCpOptions option) {
+ boolean hasOption =
+ command.hasOption(DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch());
+ LOG.info("parseChunkSize: " +
+ DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " " + hasOption);
+ if (hasOption) {
+ String chunkSizeString = getVal(command,
+ DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch().trim());
+ try {
+ int csize = Integer.parseInt(chunkSizeString);
+ if (csize < 0) {
+ csize = 0;
+ }
+ LOG.info("Set distcp blocksPerChunk to " + csize);
+ option.setBlocksPerChunk(csize);
+ }
+ catch (NumberFormatException e) {
+ throw new IllegalArgumentException("blocksPerChunk is invalid: "
+ + chunkSizeString, e);
+ }
+ }
+ }
+
/**
* parseSizeLimit is a helper method for parsing the deprecated
* argument SIZE_LIMIT.
@@ -211,8 +244,7 @@ public class OptionsParser {
DistCpOptionSwitch.FILE_LIMIT.getSwitch().trim());
try {
Integer.parseInt(fileLimitString);
- }
- catch (NumberFormatException e) {
+ } catch (NumberFormatException e) {
throw new IllegalArgumentException("File-limit is invalid: "
+ fileLimitString, e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
index 105e4f2..af91347 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.tools;
import com.google.common.collect.Lists;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
@@ -47,6 +48,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
+import java.util.LinkedList;
import static org.apache.hadoop.tools.DistCpConstants
.HDFS_RESERVED_RAW_DIRECTORY_NAME;
@@ -240,10 +242,10 @@ public class SimpleCopyListing extends CopyListing {
final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs();
- CopyListingFileStatus fileCopyListingStatus =
+ LinkedList<CopyListingFileStatus> fileCopyListingStatus =
DistCpUtils.toCopyListingFileStatus(sourceFS, fileStatus,
- preserveAcls, preserveXAttrs, preserveRawXAttrs);
-
+ preserveAcls, preserveXAttrs, preserveRawXAttrs,
+ options.getBlocksPerChunk());
writeToFileListingRoot(fileListWriter, fileCopyListingStatus,
sourceRoot, options);
}
@@ -348,9 +350,10 @@ public class SimpleCopyListing extends CopyListing {
FileStatus[] sourceFiles = sourceFS.listStatus(path);
boolean explore = (sourceFiles != null && sourceFiles.length > 0);
if (!explore || rootStatus.isDirectory()) {
- CopyListingFileStatus rootCopyListingStatus =
- DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus,
- preserveAcls, preserveXAttrs, preserveRawXAttrs);
+ LinkedList<CopyListingFileStatus> rootCopyListingStatus =
+ DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus,
+ preserveAcls, preserveXAttrs, preserveRawXAttrs,
+ options.getBlocksPerChunk());
writeToFileListingRoot(fileListWriter, rootCopyListingStatus,
sourcePathRoot, options);
}
@@ -360,20 +363,20 @@ public class SimpleCopyListing extends CopyListing {
if (LOG.isDebugEnabled()) {
LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy.");
}
- CopyListingFileStatus sourceCopyListingStatus =
- DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus,
- preserveAcls && sourceStatus.isDirectory(),
- preserveXAttrs && sourceStatus.isDirectory(),
- preserveRawXAttrs && sourceStatus.isDirectory());
- if (randomizeFileListing) {
- addToFileListing(statusList,
- new FileStatusInfo(sourceCopyListingStatus, sourcePathRoot),
- fileListWriter);
- } else {
- writeToFileListing(fileListWriter, sourceCopyListingStatus,
- sourcePathRoot);
+ LinkedList<CopyListingFileStatus> sourceCopyListingStatus =
+ DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus,
+ preserveAcls && sourceStatus.isDirectory(),
+ preserveXAttrs && sourceStatus.isDirectory(),
+ preserveRawXAttrs && sourceStatus.isDirectory(),
+ options.getBlocksPerChunk());
+ for (CopyListingFileStatus fs : sourceCopyListingStatus) {
+ if (randomizeFileListing) {
+ addToFileListing(statusList,
+ new FileStatusInfo(fs, sourcePathRoot), fileListWriter);
+ } else {
+ writeToFileListing(fileListWriter, fs, sourcePathRoot);
+ }
}
-
if (sourceStatus.isDirectory()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding source dir for traverse: " + sourceStatus.getPath());
@@ -641,18 +644,20 @@ public class SimpleCopyListing extends CopyListing {
LOG.debug("Recording source-path: " + child.getPath() + " for copy.");
}
if (workResult.getSuccess()) {
- CopyListingFileStatus childCopyListingStatus =
+ LinkedList<CopyListingFileStatus> childCopyListingStatus =
DistCpUtils.toCopyListingFileStatus(sourceFS, child,
preserveAcls && child.isDirectory(),
preserveXAttrs && child.isDirectory(),
- preserveRawXattrs && child.isDirectory());
- if (randomizeFileListing) {
- addToFileListing(fileStatuses,
- new FileStatusInfo(childCopyListingStatus, sourcePathRoot),
- fileListWriter);
- } else {
- writeToFileListing(fileListWriter, childCopyListingStatus,
- sourcePathRoot);
+ preserveRawXattrs && child.isDirectory(),
+ options.getBlocksPerChunk());
+
+ for (CopyListingFileStatus fs : childCopyListingStatus) {
+ if (randomizeFileListing) {
+ addToFileListing(fileStatuses,
+ new FileStatusInfo(fs, sourcePathRoot), fileListWriter);
+ } else {
+ writeToFileListing(fileListWriter, fs, sourcePathRoot);
+ }
}
}
if (retry < maxRetries) {
@@ -675,19 +680,21 @@ public class SimpleCopyListing extends CopyListing {
}
private void writeToFileListingRoot(SequenceFile.Writer fileListWriter,
- CopyListingFileStatus fileStatus, Path sourcePathRoot,
+ LinkedList<CopyListingFileStatus> fileStatus, Path sourcePathRoot,
DistCpOptions options) throws IOException {
boolean syncOrOverwrite = options.shouldSyncFolder() ||
options.shouldOverwrite();
- if (fileStatus.getPath().equals(sourcePathRoot) &&
- fileStatus.isDirectory() && syncOrOverwrite) {
- // Skip the root-paths when syncOrOverwrite
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skip " + fileStatus.getPath());
- }
- return;
+ for (CopyListingFileStatus fs : fileStatus) {
+ if (fs.getPath().equals(sourcePathRoot) &&
+ fs.isDirectory() && syncOrOverwrite) {
+ // Skip the root-paths when syncOrOverwrite
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skip " + fs.getPath());
+ }
+ return;
+ }
+ writeToFileListing(fileListWriter, fs, sourcePathRoot);
}
- writeToFileListing(fileListWriter, fileStatus, sourcePathRoot);
}
private void writeToFileListing(SequenceFile.Writer fileListWriter,
@@ -707,7 +714,7 @@ public class SimpleCopyListing extends CopyListing {
fileListWriter.sync();
if (!fileStatus.isDirectory()) {
- totalBytesToCopy += fileStatus.getLen();
+ totalBytesToCopy += fileStatus.getSizeToCopy();
} else {
totalDirs++;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
index 75cefb4..6ddaab9 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -34,14 +35,17 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.tools.CopyListing;
import org.apache.hadoop.tools.CopyListingFileStatus;
import org.apache.hadoop.tools.DistCpConstants;
+import org.apache.hadoop.tools.DistCpOptionSwitch;
import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.GlobbedCopyListing;
import org.apache.hadoop.tools.util.DistCpUtils;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.LinkedList;
import java.util.List;
/**
@@ -63,7 +67,8 @@ public class CopyCommitter extends FileOutputCommitter {
private boolean syncFolder = false;
private boolean overwrite = false;
private boolean targetPathExists = true;
-
+ private boolean ignoreFailures = false;
+
/**
* Create a output committer
*
@@ -82,8 +87,13 @@ public class CopyCommitter extends FileOutputCommitter {
Configuration conf = jobContext.getConfiguration();
syncFolder = conf.getBoolean(DistCpConstants.CONF_LABEL_SYNC_FOLDERS, false);
overwrite = conf.getBoolean(DistCpConstants.CONF_LABEL_OVERWRITE, false);
- targetPathExists = conf.getBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true);
-
+ targetPathExists = conf.getBoolean(
+ DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true);
+ ignoreFailures = conf.getBoolean(
+ DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
+
+ concatFileChunks(conf);
+
super.commitJob(jobContext);
cleanupTempFiles(jobContext);
@@ -169,9 +179,112 @@ public class CopyCommitter extends FileOutputCommitter {
}
}
+ private boolean isFileNotFoundException(IOException e) {
+ if (e instanceof FileNotFoundException) {
+ return true;
+ }
+
+ if (e instanceof RemoteException) {
+ return ((RemoteException)e).unwrapRemoteException()
+ instanceof FileNotFoundException;
+ }
+
+ return false;
+ }
+
+ /**
+ * Concat chunk files for the same file into one.
+ * Iterate through copy listing, identify chunk files for the same file,
+ * concat them into one.
+ */
+ private void concatFileChunks(Configuration conf) throws IOException {
+
+ LOG.info("concat file chunks ...");
+
+ String spath = conf.get(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH);
+ if (spath == null || spath.isEmpty()) {
+ return;
+ }
+ Path sourceListing = new Path(spath);
+ SequenceFile.Reader sourceReader = new SequenceFile.Reader(conf,
+ SequenceFile.Reader.file(sourceListing));
+ Path targetRoot =
+ new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
+
+ try {
+ CopyListingFileStatus srcFileStatus = new CopyListingFileStatus();
+ Text srcRelPath = new Text();
+ CopyListingFileStatus lastFileStatus = null;
+ LinkedList<Path> allChunkPaths = new LinkedList<Path>();
+
+ // Iterate over every source path that was copied.
+ while (sourceReader.next(srcRelPath, srcFileStatus)) {
+ if (srcFileStatus.isDirectory()) {
+ continue;
+ }
+ Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath);
+ Path targetFileChunkPath =
+ DistCpUtils.getSplitChunkPath(targetFile, srcFileStatus);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" add " + targetFileChunkPath + " to concat.");
+ }
+ allChunkPaths.add(targetFileChunkPath);
+ if (srcFileStatus.getChunkOffset() + srcFileStatus.getChunkLength()
+ == srcFileStatus.getLen()) {
+ // This is the last chunk of the splits, consolidate allChunkPaths
+ try {
+ concatFileChunks(conf, targetFile, allChunkPaths);
+ } catch (IOException e) {
+ // If the concat failed because a chunk file doesn't exist,
+ // then we assume that the CopyMapper has skipped copying this
+ // file, and we ignore the exception here.
+ // If a chunk file should have been created but it was not, then
+ // the CopyMapper would have failed.
+ if (!isFileNotFoundException(e)) {
+ String emsg = "Failed to concat chunk files for " + targetFile;
+ if (!ignoreFailures) {
+ throw new IOException(emsg, e);
+ } else {
+ LOG.warn(emsg, e);
+ }
+ }
+ }
+ allChunkPaths.clear();
+ lastFileStatus = null;
+ } else {
+ if (lastFileStatus == null) {
+ lastFileStatus = new CopyListingFileStatus(srcFileStatus);
+ } else {
+ // Two neighboring chunks have to be consecutive ones for the same
+ // file, for them to be merged
+ if (!srcFileStatus.getPath().equals(lastFileStatus.getPath()) ||
+ srcFileStatus.getChunkOffset() !=
+ (lastFileStatus.getChunkOffset() +
+ lastFileStatus.getChunkLength())) {
+ String emsg = "Inconsistent sequence file: current " +
+ "chunk file " + srcFileStatus + " doesnt match prior " +
+ "entry " + lastFileStatus;
+ if (!ignoreFailures) {
+ throw new IOException(emsg);
+ } else {
+ LOG.warn(emsg + ", skipping concat this set.");
+ }
+ } else {
+ lastFileStatus.setChunkOffset(srcFileStatus.getChunkOffset());
+ lastFileStatus.setChunkLength(srcFileStatus.getChunkLength());
+ }
+ }
+ }
+ }
+ } finally {
+ IOUtils.closeStream(sourceReader);
+ }
+ }
+
// This method changes the target-directories' file-attributes (owner,
// user/group permissions, etc.) based on the corresponding source directories.
- private void preserveFileAttributesForDirectories(Configuration conf) throws IOException {
+ private void preserveFileAttributesForDirectories(Configuration conf)
+ throws IOException {
String attrSymbols = conf.get(DistCpConstants.CONF_LABEL_PRESERVE_STATUS);
final boolean syncOrOverwrite = syncFolder || overwrite;
@@ -325,4 +438,57 @@ public class CopyCommitter extends FileOutputCommitter {
", Unable to move to " + finalDir);
}
}
+
+ /**
+ * Concat the passed chunk files into one and rename it the targetFile.
+ */
+ private void concatFileChunks(Configuration conf, Path targetFile,
+ LinkedList<Path> allChunkPaths) throws IOException {
+ if (allChunkPaths.size() == 1) {
+ return;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("concat " + targetFile + " allChunkSize+ "
+ + allChunkPaths.size());
+ }
+ FileSystem dstfs = targetFile.getFileSystem(conf);
+
+ Path firstChunkFile = allChunkPaths.removeFirst();
+ Path[] restChunkFiles = new Path[allChunkPaths.size()];
+ allChunkPaths.toArray(restChunkFiles);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("concat: firstchunk: " + dstfs.getFileStatus(firstChunkFile));
+ int i = 0;
+ for (Path f : restChunkFiles) {
+ LOG.debug("concat: other chunk: " + i + ": " + dstfs.getFileStatus(f));
+ ++i;
+ }
+ }
+ dstfs.concat(firstChunkFile, restChunkFiles);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("concat: result: " + dstfs.getFileStatus(firstChunkFile));
+ }
+ rename(dstfs, firstChunkFile, targetFile);
+ }
+
+ /**
+ * Rename tmp to dst on destFileSys.
+ * @param destFileSys the file ssystem
+ * @param tmp the source path
+ * @param dst the destination path
+ * @throws IOException if renaming failed
+ */
+ private static void rename(FileSystem destFileSys, Path tmp, Path dst)
+ throws IOException {
+ try {
+ if (destFileSys.exists(dst)) {
+ destFileSys.delete(dst, true);
+ }
+ destFileSys.rename(tmp, dst);
+ } catch (IOException ioe) {
+ throw new IOException("Fail to rename tmp file (=" + tmp
+ + ") to destination file (=" + dst + ")", ioe);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
index e1873f1..d6b3ba8 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
@@ -156,10 +156,12 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
sourceFS = sourcePath.getFileSystem(conf);
final boolean preserveXAttrs =
fileAttributes.contains(FileAttribute.XATTR);
- sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS,
- sourceFS.getFileStatus(sourcePath),
- fileAttributes.contains(FileAttribute.ACL),
- preserveXAttrs, preserveRawXattrs);
+ sourceCurrStatus = DistCpUtils.toCopyListingFileStatusHelper(sourceFS,
+ sourceFS.getFileStatus(sourcePath),
+ fileAttributes.contains(FileAttribute.ACL),
+ preserveXAttrs, preserveRawXattrs,
+ sourceFileStatus.getChunkOffset(),
+ sourceFileStatus.getChunkLength());
} catch (FileNotFoundException e) {
throw new IOException(new RetriableFileCopyCommand.CopyReadException(e));
}
@@ -173,7 +175,8 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
LOG.debug("Path could not be found: " + target, ignore);
}
- if (targetStatus != null && (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) {
+ if (targetStatus != null &&
+ (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) {
throw new IOException("Can't replace " + target + ". Target is " +
getFileType(targetStatus) + ", Source is " + getFileType(sourceCurrStatus));
}
@@ -183,19 +186,28 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
return;
}
- FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, targetStatus);
+ FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target,
+ targetStatus);
+
+ Path tmpTarget = target;
if (action == FileAction.SKIP) {
LOG.info("Skipping copy of " + sourceCurrStatus.getPath()
+ " to " + target);
updateSkipCounters(context, sourceCurrStatus);
context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath()));
+
} else {
- copyFileWithRetry(description, sourceCurrStatus, target, context,
+ if (sourceCurrStatus.isSplit()) {
+ tmpTarget = DistCpUtils.getSplitChunkPath(target, sourceCurrStatus);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("copying " + sourceCurrStatus + " " + tmpTarget);
+ }
+ copyFileWithRetry(description, sourceCurrStatus, tmpTarget, context,
action, fileAttributes);
}
-
- DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus,
- fileAttributes, preserveRawXattrs);
+ DistCpUtils.preserve(target.getFileSystem(conf), tmpTarget,
+ sourceCurrStatus, fileAttributes, preserveRawXattrs);
} catch (IOException exception) {
handleFailures(exception, sourceFileStatus, target, context);
}
@@ -261,8 +273,12 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
private void handleFailures(IOException exception,
CopyListingFileStatus sourceFileStatus, Path target, Context context)
throws IOException, InterruptedException {
- LOG.error("Failure in copying " + sourceFileStatus.getPath() + " to " +
- target, exception);
+ LOG.error("Failure in copying " + sourceFileStatus.getPath() +
+ (sourceFileStatus.isSplit()? ","
+ + " offset=" + sourceFileStatus.getChunkOffset()
+ + " chunkLength=" + sourceFileStatus.getChunkLength()
+ : "") +
+ " to " + target, exception);
if (ignoreFailures &&
ExceptionUtils.indexOfType(exception, CopyReadException.class) != -1) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
index 06acd78..2c17fef 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
@@ -118,17 +118,21 @@ public class RetriableFileCopyCommand extends RetriableCommand {
.contains(FileAttribute.CHECKSUMTYPE) ? sourceFS
.getFileChecksum(sourcePath) : null;
- final long offset = action == FileAction.APPEND ? targetFS.getFileStatus(
- target).getLen() : 0;
+ long offset = (action == FileAction.APPEND) ?
+ targetFS.getFileStatus(target).getLen() : source.getChunkOffset();
long bytesRead = copyToFile(targetPath, targetFS, source,
offset, context, fileAttributes, sourceChecksum);
- compareFileLengths(source, targetPath, configuration, bytesRead
- + offset);
+ if (!source.isSplit()) {
+ compareFileLengths(source, targetPath, configuration, bytesRead
+ + offset);
+ }
//At this point, src&dest lengths are same. if length==0, we skip checksum
if ((bytesRead != 0) && (!skipCrc)) {
- compareCheckSums(sourceFS, source.getPath(), sourceChecksum,
- targetFS, targetPath);
+ if (!source.isSplit()) {
+ compareCheckSums(sourceFS, source.getPath(), sourceChecksum,
+ targetFS, targetPath);
+ }
}
// it's not append case, thus we first write to a temporary file, rename
// it to the target path.
@@ -249,16 +253,26 @@ public class RetriableFileCopyCommand extends RetriableCommand {
ThrottledInputStream inStream = null;
long totalBytesRead = 0;
+ long chunkLength = source2.getChunkLength();
+ boolean finished = false;
try {
inStream = getInputStream(source, context.getConfiguration());
int bytesRead = readBytes(inStream, buf, sourceOffset);
while (bytesRead >= 0) {
+ if (chunkLength > 0 &&
+ (totalBytesRead + bytesRead) >= chunkLength) {
+ bytesRead = (int)(chunkLength - totalBytesRead);
+ finished = true;
+ }
totalBytesRead += bytesRead;
if (action == FileAction.APPEND) {
sourceOffset += bytesRead;
}
outStream.write(buf, 0, bytesRead);
updateContextStatus(totalBytesRead, context, source2);
+ if (finished) {
+ break;
+ }
bytesRead = readBytes(inStream, buf, sourceOffset);
}
outStream.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
index 3e86d09..d1c18ea 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
@@ -99,7 +99,8 @@ public class UniformSizeInputFormat
while (reader.next(srcRelPath, srcFileStatus)) {
// If adding the current file would cause the bytes per map to exceed
// limit. Add the current file to new split
- if (currentSplitSize + srcFileStatus.getLen() > nBytesPerSplit && lastPosition != 0) {
+ if (currentSplitSize + srcFileStatus.getChunkLength() > nBytesPerSplit
+ && lastPosition != 0) {
FileSplit split = new FileSplit(listingFilePath, lastSplitStart,
lastPosition - lastSplitStart, null);
if (LOG.isDebugEnabled()) {
@@ -109,7 +110,7 @@ public class UniformSizeInputFormat
lastSplitStart = lastPosition;
currentSplitSize = 0;
}
- currentSplitSize += srcFileStatus.getLen();
+ currentSplitSize += srcFileStatus.getChunkLength();
lastPosition = reader.getPosition();
}
if (lastPosition > lastSplitStart) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
index 76bc4c5..e315b84 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
@@ -19,9 +19,11 @@
package org.apache.hadoop.tools.util;
import com.google.common.collect.Maps;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -30,6 +32,7 @@ import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputFormat;
@@ -44,6 +47,7 @@ import org.apache.hadoop.util.StringUtils;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.EnumSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -116,7 +120,7 @@ public class DistCpUtils {
* @return Class implementing the strategy specified in options.
*/
public static Class<? extends InputFormat> getStrategy(Configuration conf,
- DistCpOptions options) {
+ DistCpOptions options) {
String confLabel = "distcp."
+ StringUtils.toLowerCase(options.getCopyStrategy())
+ ".strategy" + ".impl";
@@ -298,6 +302,86 @@ public class DistCpUtils {
}
/**
+ * Converts FileStatus to a list of CopyListingFileStatus.
+ * The resulted list contains either one CopyListingFileStatus per chunk of
+ * file-blocks (if file-size exceeds blockSize * blocksPerChunk, and there
+ * are more blocks in the file than blocksperChunk), or a single
+ * CopyListingFileStatus for the entire file (if file-size is too small to
+ * split).
+ * If preserving ACLs, populates the CopyListingFileStatus with the ACLs.
+ * If preserving XAttrs, populates the CopyListingFileStatus with the XAttrs.
+ *
+ * @param fileSystem FileSystem containing the file
+ * @param fileStatus FileStatus of file
+ * @param preserveAcls boolean true if preserving ACLs
+ * @param preserveXAttrs boolean true if preserving XAttrs
+ * @param preserveRawXAttrs boolean true if preserving raw.* XAttrs
+ * @param blocksPerChunk size of chunks when copying chunks in parallel
+ * @return list of CopyListingFileStatus
+ * @throws IOException if there is an I/O error
+ */
+ public static LinkedList<CopyListingFileStatus> toCopyListingFileStatus(
+ FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls,
+ boolean preserveXAttrs, boolean preserveRawXAttrs, int blocksPerChunk)
+ throws IOException {
+ LinkedList<CopyListingFileStatus> copyListingFileStatus =
+ new LinkedList<CopyListingFileStatus>();
+
+ final CopyListingFileStatus clfs = toCopyListingFileStatusHelper(
+ fileSystem, fileStatus, preserveAcls,
+ preserveXAttrs, preserveRawXAttrs,
+ 0, fileStatus.getLen());
+ final long blockSize = fileStatus.getBlockSize();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("toCopyListing: " + fileStatus + " chunkSize: "
+ + blocksPerChunk + " isDFS: " +
+ (fileSystem instanceof DistributedFileSystem));
+ }
+ if ((blocksPerChunk > 0) &&
+ !fileStatus.isDirectory() &&
+ (fileStatus.getLen() > blockSize * blocksPerChunk)) {
+ // split only when the file size is larger than the intended chunk size
+ final BlockLocation[] blockLocations;
+ blockLocations = fileSystem.getFileBlockLocations(fileStatus, 0,
+ fileStatus.getLen());
+
+ int numBlocks = blockLocations.length;
+ long curPos = 0;
+ if (numBlocks <= blocksPerChunk) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" add file " + clfs);
+ }
+ copyListingFileStatus.add(clfs);
+ } else {
+ int i = 0;
+ while (i < numBlocks) {
+ long curLength = 0;
+ for (int j = 0; j < blocksPerChunk && i < numBlocks; ++j, ++i) {
+ curLength += blockLocations[i].getLength();
+ }
+ if (curLength > 0) {
+ CopyListingFileStatus clfs1 = new CopyListingFileStatus(clfs);
+ clfs1.setChunkOffset(curPos);
+ clfs1.setChunkLength(curLength);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" add file chunk " + clfs1);
+ }
+ copyListingFileStatus.add(clfs1);
+ curPos += curLength;
+ }
+ }
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" add file/dir " + clfs);
+ }
+ copyListingFileStatus.add(clfs);
+ }
+
+ return copyListingFileStatus;
+ }
+
+ /**
* Converts a FileStatus to a CopyListingFileStatus. If preserving ACLs,
* populates the CopyListingFileStatus with the ACLs. If preserving XAttrs,
* populates the CopyListingFileStatus with the XAttrs.
@@ -307,13 +391,17 @@ public class DistCpUtils {
* @param preserveAcls boolean true if preserving ACLs
* @param preserveXAttrs boolean true if preserving XAttrs
* @param preserveRawXAttrs boolean true if preserving raw.* XAttrs
+ * @param chunkOffset chunk offset in bytes
+ * @param chunkLength chunk length in bytes
+ * @return CopyListingFileStatus
* @throws IOException if there is an I/O error
*/
- public static CopyListingFileStatus toCopyListingFileStatus(
+ public static CopyListingFileStatus toCopyListingFileStatusHelper(
FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls,
- boolean preserveXAttrs, boolean preserveRawXAttrs) throws IOException {
+ boolean preserveXAttrs, boolean preserveRawXAttrs,
+ long chunkOffset, long chunkLength) throws IOException {
CopyListingFileStatus copyListingFileStatus =
- new CopyListingFileStatus(fileStatus);
+ new CopyListingFileStatus(fileStatus, chunkOffset, chunkLength);
if (preserveAcls) {
FsPermission perm = fileStatus.getPermission();
if (perm.getAclBit()) {
@@ -470,4 +558,19 @@ public class DistCpUtils {
return (sourceChecksum == null || targetChecksum == null ||
sourceChecksum.equals(targetChecksum));
}
+
+ /*
+ * Return the Path for a given chunk.
+ * Used when splitting large file into chunks to copy in parallel.
+ * @param targetFile path to target file
+ * @param srcFileStatus source file status in copy listing
+ * @return path to the chunk specified by the parameters to store
+ * in target cluster temporarily
+ */
+ public static Path getSplitChunkPath(Path targetFile,
+ CopyListingFileStatus srcFileStatus) {
+ return new Path(targetFile.toString()
+ + ".____distcpSplit____" + srcFileStatus.getChunkOffset()
+ + "." + srcFileStatus.getChunkLength());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
index 41a6e94..a77deb2 100644
--- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
+++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
@@ -237,6 +237,7 @@ Flag | Description | Notes
`-rdiff <newSnapshot> <oldSnapshot>` | Use snapshot diff report between given two snapshots to identify what has been changed on the target since the snapshot `<oldSnapshot>` was created on the target, and apply the diff reversely to the target, and copy modified files from the source's `<oldSnapshot>`, to make the target the same as `<oldSnapshot>`. | This option is valid only with `-update` option and the following conditions should be satisfied. <ol><li>Both the source and the target FileSystem must be DistributedFileSystem. The source and the target can be two different clusters/paths, or they can be exactly the same cluster/path. In the latter case, modified files are copied from target's `<oldSnapshot>` to target's current state).</li> <li> Two snapshots `<newSnapshot>` and `<oldSnapshot>` have been created on the target FS, and `<oldSnapshot>` is older than `<newSnapshot>`. No change has been made on target since `<newSnapshot>` was created on the target. </li> <li> The sour
ce has the same snapshot `<oldSnapshot>`, which has the same content as the `<oldSnapshot>` on the target. All the files/directories in the target's `<oldSnapshot>` are the same with source's `<oldSnapshot>`.</li> </ol> |
`-numListstatusThreads` | Number of threads to use for building file listing | At most 40 threads.
`-skipcrccheck` | Whether to skip CRC checks between source and target paths. |
+`-blocksperchunk <blocksperchunk>` | Number of blocks per chunk. When specified, split files into chunks to copy in parallel | If set to a positive value, files with more blocks than this value will be split into chunks of `<blocksperchunk>` blocks to be transferred in parallel, and reassembled on the destination. By default, `<blocksperchunk>` is 0 and the files will be transmitted in their entirety without splitting. This switch is only applicable when the source file system implements getBlockLocations method and the target file system implements concat method. |
Architecture of DistCp
----------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
index e3018a0..b2266b3 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
@@ -23,17 +23,27 @@ import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.OutputStream;
+import java.io.PrintStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
+import java.util.Random;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.util.ToolRunner;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -47,11 +57,15 @@ import org.junit.rules.Timeout;
*/
public class TestDistCpSystem {
+ private static final Log LOG =
+ LogFactory.getLog(TestDistCpSystem.class);
+
@Rule
public Timeout globalTimeout = new Timeout(30000);
private static final String SRCDAT = "srcdat";
private static final String DSTDAT = "dstdat";
+ private static final long BLOCK_SIZE = 1024;
private static MiniDFSCluster cluster;
private static Configuration conf;
@@ -63,27 +77,76 @@ public class TestDistCpSystem {
this.path = path;
this.isDir = isDir;
}
- String getPath() { return path; }
- boolean isDirectory() { return isDir; }
+
+ String getPath() {
+ return path;
+ }
+
+ boolean isDirectory() {
+ return isDir;
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws IOException {
+ conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+ cluster.waitActive();
+ }
+
+ @AfterClass
+ public static void afterClass() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ static String execCmd(FsShell shell, String... args) throws Exception {
+ ByteArrayOutputStream baout = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(baout, true);
+ PrintStream old = System.out;
+ System.setOut(out);
+ shell.run(args);
+ out.close();
+ System.setOut(old);
+ return baout.toString();
}
- private void createFiles(FileSystem fs, String topdir,
- FileEntry[] entries) throws IOException {
+ private void createFiles(DistributedFileSystem fs, String topdir,
+ FileEntry[] entries, long chunkSize) throws IOException {
+ long seed = System.currentTimeMillis();
+ Random rand = new Random(seed);
+ short replicationFactor = 2;
for (FileEntry entry : entries) {
- Path newpath = new Path(topdir + "/" + entry.getPath());
+ Path newPath = new Path(topdir + "/" + entry.getPath());
if (entry.isDirectory()) {
- fs.mkdirs(newpath);
+ fs.mkdirs(newPath);
} else {
- OutputStream out = fs.create(newpath);
- try {
- out.write((topdir + "/" + entry).getBytes());
- out.write("\n".getBytes());
- } finally {
- out.close();
+ long fileSize = BLOCK_SIZE *100;
+ int bufSize = 128;
+ if (chunkSize == -1) {
+ DFSTestUtil.createFile(fs, newPath, bufSize,
+ fileSize, BLOCK_SIZE, replicationFactor, seed);
+ } else {
+ // Create a variable length block file, by creating
+ // one block of half block size at the chunk boundary
+ long seg1 = chunkSize * BLOCK_SIZE - BLOCK_SIZE / 2;
+ long seg2 = fileSize - seg1;
+ DFSTestUtil.createFile(fs, newPath, bufSize,
+ seg1, BLOCK_SIZE, replicationFactor, seed);
+ DFSTestUtil.appendFileNewBlock(fs, newPath, (int)seg2);
}
}
+ seed = System.currentTimeMillis() + rand.nextLong();
}
}
+
+ private void createFiles(DistributedFileSystem fs, String topdir,
+ FileEntry[] entries) throws IOException {
+ createFiles(fs, topdir, entries, -1);
+ }
private static FileStatus[] getFileStatus(FileSystem fs,
String topdir, FileEntry[] files) throws IOException {
@@ -104,18 +167,19 @@ public class TestDistCpSystem {
}
private void testPreserveUserHelper(String testRoot,
- FileEntry[] srcEntries,
- FileEntry[] dstEntries,
- boolean createSrcDir,
- boolean createTgtDir,
- boolean update) throws Exception {
+ FileEntry[] srcEntries,
+ FileEntry[] dstEntries,
+ boolean createSrcDir,
+ boolean createTgtDir,
+ boolean update) throws Exception {
final String testSrcRel = SRCDAT;
final String testSrc = testRoot + "/" + testSrcRel;
final String testDstRel = DSTDAT;
final String testDst = testRoot + "/" + testDstRel;
String nnUri = FileSystem.getDefaultUri(conf).toString();
- FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
+ DistributedFileSystem fs = (DistributedFileSystem)
+ FileSystem.get(URI.create(nnUri), conf);
fs.mkdirs(new Path(testRoot));
if (createSrcDir) {
fs.mkdirs(new Path(testSrc));
@@ -129,8 +193,8 @@ public class TestDistCpSystem {
for(int i = 0; i < srcEntries.length; i++) {
fs.setOwner(srcstats[i].getPath(), "u" + i, null);
}
- String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc,
- nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst};
+ String[] args = update? new String[]{"-pub", "-update", nnUri+testSrc,
+ nnUri+testDst} : new String[]{"-pub", nnUri+testSrc, nnUri+testDst};
ToolRunner.run(conf, new DistCp(), args);
@@ -145,20 +209,263 @@ public class TestDistCpSystem {
deldir(fs, testRoot);
}
- @BeforeClass
- public static void beforeClass() throws IOException {
- conf = new Configuration();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- cluster.waitActive();
+ private void compareFiles(FileSystem fs, FileStatus srcStat,
+ FileStatus dstStat) throws Exception {
+ LOG.info("Comparing " + srcStat + " and " + dstStat);
+ assertEquals(srcStat.isDirectory(), dstStat.isDirectory());
+ assertEquals(srcStat.getReplication(), dstStat.getReplication());
+ assertEquals("File POSIX permission should match",
+ srcStat.getPermission(), dstStat.getPermission());
+ assertEquals("File user ownership should match",
+ srcStat.getOwner(), dstStat.getOwner());
+ assertEquals("File group ownership should match",
+ srcStat.getGroup(), dstStat.getGroup());
+ // TODO; check ACL attributes
+
+ if (srcStat.isDirectory()) {
+ return;
+ }
+
+ assertEquals("File length should match (" + srcStat.getPath() + ")",
+ srcStat.getLen(), dstStat.getLen());
+
+ FSDataInputStream srcIn = fs.open(srcStat.getPath());
+ FSDataInputStream dstIn = fs.open(dstStat.getPath());
+ try {
+ byte[] readSrc = new byte[(int)
+ HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT];
+ byte[] readDst = new byte[(int)
+ HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT];
+
+ int srcBytesRead = 0, tgtBytesRead = 0;
+ int srcIdx = 0, tgtIdx = 0;
+ long totalComparedBytes = 0;
+ while (true) {
+ if (srcBytesRead == 0) {
+ srcBytesRead = srcIn.read(readSrc);
+ srcIdx = 0;
+ }
+ if (tgtBytesRead == 0) {
+ tgtBytesRead = dstIn.read(readDst);
+ tgtIdx = 0;
+ }
+ if (srcBytesRead == 0 || tgtBytesRead == 0) {
+ LOG.info("______ compared src and dst files for "
+ + totalComparedBytes + " bytes, content match.");
+ if (srcBytesRead != tgtBytesRead) {
+ Assert.fail("Read mismatching size, compared "
+ + totalComparedBytes + " bytes between src and dst file "
+ + srcStat + " and " + dstStat);
+ }
+ if (totalComparedBytes != srcStat.getLen()) {
+ Assert.fail("Only read/compared " + totalComparedBytes +
+ " bytes between src and dst file " + srcStat +
+ " and " + dstStat);
+ } else {
+ // success
+ break;
+ }
+ }
+ for (; srcIdx < srcBytesRead && tgtIdx < tgtBytesRead;
+ ++srcIdx, ++tgtIdx) {
+ if (readSrc[srcIdx] != readDst[tgtIdx]) {
+ Assert.fail("src and dst file does not match at "
+ + totalComparedBytes + " between "
+ + srcStat + " and " + dstStat);
+ }
+ ++totalComparedBytes;
+ }
+ LOG.info("______ compared src and dst files for "
+ + totalComparedBytes + " bytes, content match. FileLength: "
+ + srcStat.getLen());
+ if (totalComparedBytes == srcStat.getLen()) {
+ LOG.info("______ Final:" + srcIdx + " "
+ + srcBytesRead + " " + tgtIdx + " " + tgtBytesRead);
+ break;
+ }
+ if (srcIdx == srcBytesRead) {
+ srcBytesRead = 0;
+ }
+ if (tgtIdx == tgtBytesRead) {
+ tgtBytesRead = 0;
+ }
+ }
+ } finally {
+ if (srcIn != null) {
+ srcIn.close();
+ }
+ if (dstIn != null) {
+ dstIn.close();
+ }
+ }
}
- @AfterClass
- public static void afterClass() throws IOException {
- if (cluster != null) {
- cluster.shutdown();
+ // WC: needed because the current distcp does not create target dirs
+ private void createDestDir(FileSystem fs, String testDst,
+ FileStatus[] srcStats, FileEntry[] srcFiles) throws IOException {
+ fs.mkdirs(new Path(testDst));
+
+ for (int i=0; i<srcStats.length; i++) {
+ FileStatus srcStat = srcStats[i];
+ if (srcStat.isDirectory()) {
+ Path dstPath = new Path(testDst, srcFiles[i].getPath());
+ fs.mkdirs(dstPath);
+ fs.setOwner(dstPath, srcStat.getOwner(), srcStat.getGroup());
+ }
+ }
+ }
+
+ private void copyAndVerify(final DistributedFileSystem fs,
+ final FileEntry[] srcFiles, final FileStatus[] srcStats,
+ final String testDst,
+ final String[] args) throws Exception {
+ final String testRoot = "/testdir";
+ FsShell shell = new FsShell(fs.getConf());
+
+ LOG.info("ls before distcp");
+ LOG.info(execCmd(shell, "-lsr", testRoot));
+
+ LOG.info("_____ running distcp: " + args[0] + " " + args[1]);
+ ToolRunner.run(conf, new DistCp(), args);
+
+ LOG.info("ls after distcp");
+ LOG.info(execCmd(shell, "-lsr", testRoot));
+
+ FileStatus[] dstStat = getFileStatus(fs, testDst, srcFiles);
+ for (int i=0; i< dstStat.length; i++) {
+ compareFiles(fs, srcStats[i], dstStat[i]);
}
}
+ private void chunkCopy(FileEntry[] srcFiles) throws Exception {
+ final String testRoot = "/testdir";
+ final String testSrcRel = SRCDAT;
+ final String testSrc = testRoot + "/" + testSrcRel;
+ final String testDstRel = DSTDAT;
+ final String testDst = testRoot + "/" + testDstRel;
+ long chunkSize = 8;
+
+ String nnUri = FileSystem.getDefaultUri(conf).toString();
+ DistributedFileSystem fs = (DistributedFileSystem)
+ FileSystem.get(URI.create(nnUri), conf);
+
+ createFiles(fs, testRoot, srcFiles, chunkSize);
+
+ FileStatus[] srcStats = getFileStatus(fs, testRoot, srcFiles);
+ for (int i = 0; i < srcFiles.length; i++) {
+ fs.setOwner(srcStats[i].getPath(), "u" + i, "g" + i);
+ }
+ // get file status after updating owners
+ srcStats = getFileStatus(fs, testRoot, srcFiles);
+
+ createDestDir(fs, testDst, srcStats, srcFiles);
+
+ String[] args = new String[] {"-pugp", "-blocksperchunk",
+ String.valueOf(chunkSize),
+ nnUri + testSrc, nnUri + testDst};
+
+ copyAndVerify(fs, srcFiles, srcStats, testDst, args);
+ // Do it again
+ copyAndVerify(fs, srcFiles, srcStats, testDst, args);
+
+ // modify last file and rerun distcp with -update option
+ LOG.info("Modify a file and copy again");
+ for(int i=srcFiles.length-1; i >=0; --i) {
+ if (!srcFiles[i].isDirectory()) {
+ LOG.info("Modifying " + srcStats[i].getPath());
+ DFSTestUtil.appendFileNewBlock(fs, srcStats[i].getPath(),
+ (int)BLOCK_SIZE * 3);
+ break;
+ }
+ }
+ // get file status after modifying file
+ srcStats = getFileStatus(fs, testRoot, srcFiles);
+
+ args = new String[] {"-pugp", "-update", "-blocksperchunk",
+ String.valueOf(chunkSize),
+ nnUri + testSrc, nnUri + testDst + "/" + testSrcRel};
+
+ copyAndVerify(fs, srcFiles, srcStats, testDst, args);
+
+ deldir(fs, testRoot);
+ }
+
+ @Test
+ public void testRecursiveChunkCopy() throws Exception {
+ FileEntry[] srcFiles = {
+ new FileEntry(SRCDAT, true),
+ new FileEntry(SRCDAT + "/file0", false),
+ new FileEntry(SRCDAT + "/dir1", true),
+ new FileEntry(SRCDAT + "/dir2", true),
+ new FileEntry(SRCDAT + "/dir1/file1", false)
+ };
+ chunkCopy(srcFiles);
+ }
+
+ @Test
+ public void testChunkCopyOneFile() throws Exception {
+ FileEntry[] srcFiles = {
+ new FileEntry(SRCDAT, true),
+ new FileEntry(SRCDAT + "/file0", false)
+ };
+ chunkCopy(srcFiles);
+ }
+
+ @Test
+ public void testDistcpLargeFile() throws Exception {
+ FileEntry[] srcfiles = {
+ new FileEntry(SRCDAT, true),
+ new FileEntry(SRCDAT + "/file", false)
+ };
+
+ final String testRoot = "/testdir";
+ final String testSrcRel = SRCDAT;
+ final String testSrc = testRoot + "/" + testSrcRel;
+ final String testDstRel = DSTDAT;
+ final String testDst = testRoot + "/" + testDstRel;
+
+ String nnUri = FileSystem.getDefaultUri(conf).toString();
+ DistributedFileSystem fs =
+ (DistributedFileSystem) FileSystem.get(URI.create(nnUri), conf);
+ fs.mkdirs(new Path(testRoot));
+ fs.mkdirs(new Path(testSrc));
+ fs.mkdirs(new Path(testDst));
+ long chunkSize = 6;
+ createFiles(fs, testRoot, srcfiles, chunkSize);
+
+ String srcFileName = testRoot + Path.SEPARATOR + srcfiles[1].getPath();
+ Path srcfile = new Path(srcFileName);
+
+ if(!cluster.getFileSystem().exists(srcfile)){
+ throw new Exception("src not exist");
+ }
+
+ final long srcLen = fs.getFileStatus(srcfile).getLen();
+
+ FileStatus[] srcstats = getFileStatus(fs, testRoot, srcfiles);
+ for (int i = 0; i < srcfiles.length; i++) {
+ fs.setOwner(srcstats[i].getPath(), "u" + i, null);
+ }
+ String[] args = new String[] {
+ "-blocksperchunk",
+ String.valueOf(chunkSize),
+ nnUri + testSrc,
+ nnUri + testDst
+ };
+
+ LOG.info("_____ running distcp: " + args[0] + " " + args[1]);
+ ToolRunner.run(conf, new DistCp(), args);
+
+ String realTgtPath = testDst;
+ FileStatus[] dststat = getFileStatus(fs, realTgtPath, srcfiles);
+ assertEquals("File length should match", srcLen,
+ dststat[dststat.length - 1].getLen());
+
+ this.compareFiles(fs, srcstats[srcstats.length-1],
+ dststat[dststat.length-1]);
+ deldir(fs, testRoot);
+ }
+
@Test
public void testPreserveUseNonEmptyDir() throws Exception {
String testRoot = "/testdir." + getMethodName();
@@ -180,7 +487,6 @@ public class TestDistCpSystem {
testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false);
}
-
@Test
public void testPreserveUserEmptyDir() throws Exception {
String testRoot = "/testdir." + getMethodName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index efe4627..f94ba97 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -394,7 +394,7 @@ public class TestOptionsParser {
+ "copyStrategy='uniformsize', preserveStatus=[], "
+ "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, "
+ "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, "
- + "targetPathExists=true, filtersFile='null'}";
+ + "targetPathExists=true, filtersFile='null', blocksPerChunk=0}";
String optionString = option.toString();
Assert.assertEquals(val, optionString);
Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3fb585/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
index 2e9a350..2452d6f 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
@@ -81,6 +81,10 @@ public class TestCopyCommitter {
@Before
public void createMetaFolder() {
config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta");
+ // Unset listing file path since the config is shared by
+ // multiple tests, and some test doesn't set it, such as
+ // testNoCommitAction, but the distcp code will check it.
+ config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, "");
Path meta = new Path("/meta");
try {
cluster.getFileSystem().mkdirs(meta);
@@ -326,7 +330,6 @@ public class TestCopyCommitter {
committer.commitJob(jobContext);
Assert.assertFalse(fs.exists(new Path(workPath)));
Assert.assertTrue(fs.exists(new Path(finalPath)));
-
} catch (IOException e) {
LOG.error("Exception encountered while testing for preserve status", e);
Assert.fail("Atomic commit failure");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-6342. Make TimelineV2Client's
drain timeout after stop configurable (Haibo Chen via Varun Saxena)
Posted by ae...@apache.org.
YARN-6342. Make TimelineV2Client's drain timeout after stop configurable (Haibo Chen via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c2bc444
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c2bc444
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c2bc444
Branch: refs/heads/HDFS-7240
Commit: 7c2bc444b3d6750aafeed9b530c8e5b1bf95c1f4
Parents: 1309c58
Author: Varun Saxena <va...@apache.org>
Authored: Fri Mar 31 02:02:57 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Fri Mar 31 02:02:57 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 10 ++++++++++
.../hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java | 9 ++++++---
.../src/main/resources/yarn-default.xml | 9 +++++++++
3 files changed, 25 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c2bc444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index c239249..81cb8c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2095,6 +2095,16 @@ public class YarnConfiguration extends Configuration {
public static final int DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE = 10;
+
+ /**
+ * The time period for which timeline v2 client will wait for draining
+ * leftover entities after stop.
+ */
+ public static final String TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS =
+ TIMELINE_SERVICE_CLIENT_PREFIX + "drain-entities.timeout.ms";
+ public static final long DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS
+ = 2000L;
+
// mark app-history related configs @Private as application history is going
// to be integrated into the timeline service
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c2bc444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java
index cef7e5f..5d88f70 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java
@@ -289,7 +289,7 @@ public class TimelineV2ClientImpl extends TimelineV2Client {
* Time period for which the timelineclient will wait for draining after
* stop.
*/
- private static final long DRAIN_TIME_PERIOD = 2000L;
+ private final long drainTimeoutPeriod;
private int numberOfAsyncsToMerge;
private final BlockingQueue<EntitiesHolder> timelineEntityQueue;
@@ -300,6 +300,9 @@ public class TimelineV2ClientImpl extends TimelineV2Client {
numberOfAsyncsToMerge =
conf.getInt(YarnConfiguration.NUMBER_OF_ASYNC_ENTITIES_TO_MERGE,
YarnConfiguration.DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE);
+ drainTimeoutPeriod = conf.getLong(
+ YarnConfiguration.TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS,
+ YarnConfiguration.DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS);
}
Runnable createRunnable() {
@@ -330,7 +333,7 @@ public class TimelineV2ClientImpl extends TimelineV2Client {
// Try to drain the remaining entities to be published @ the max for
// 2 seconds
long timeTillweDrain =
- System.currentTimeMillis() + DRAIN_TIME_PERIOD;
+ System.currentTimeMillis() + drainTimeoutPeriod;
while (!timelineEntityQueue.isEmpty()) {
publishWithoutBlockingOnQueue(timelineEntityQueue.poll());
if (System.currentTimeMillis() > timeTillweDrain) {
@@ -449,7 +452,7 @@ public class TimelineV2ClientImpl extends TimelineV2Client {
LOG.info("Stopping TimelineClient.");
executor.shutdownNow();
try {
- executor.awaitTermination(DRAIN_TIME_PERIOD, TimeUnit.MILLISECONDS);
+ executor.awaitTermination(drainTimeoutPeriod, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c2bc444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 727e2c9..bdd4de5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2102,6 +2102,15 @@
</property>
<property>
+ <description>
+ The time period for which timeline v2 client will wait for draining
+ leftover entities after stop.
+ </description>
+ <name>yarn.timeline-service.client.drain-entities.timeout.ms</name>
+ <value>2000</value>
+ </property>
+
+ <property>
<description>Enable timeline server to recover state after starting. If
true, then yarn.timeline-service.state-store-class must be specified.
</description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-6377.
NMTimelinePublisher#serviceStop does not stop timeline clients (Haibo Chen
via Varun Saxena)
Posted by ae...@apache.org.
YARN-6377. NMTimelinePublisher#serviceStop does not stop timeline clients (Haibo Chen via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4b5aa84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4b5aa84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4b5aa84
Branch: refs/heads/HDFS-7240
Commit: a4b5aa8493e0bd9006f44291d265c28ab86497e1
Parents: b883857
Author: Varun Saxena <va...@apache.org>
Authored: Sun Apr 2 04:54:12 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Sun Apr 2 04:54:12 2017 +0530
----------------------------------------------------------------------
.../nodemanager/timelineservice/NMTimelinePublisher.java | 8 ++++++++
.../timelineservice/TestNMTimelinePublisher.java | 10 +++++++++-
2 files changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4b5aa84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index ce2c656..8aaae79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -101,6 +101,14 @@ public class NMTimelinePublisher extends CompositeService {
this.nodeId = context.getNodeId();
}
+ @Override
+ protected void serviceStop() throws Exception {
+ for(ApplicationId app : appToClientMap.keySet()) {
+ stopTimelineClient(app);
+ }
+ super.serviceStop();
+ }
+
@VisibleForTesting
Map<ApplicationId, TimelineV2Client> getAppToClientMap() {
return appToClientMap;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4b5aa84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
index e116122..0b8eaa9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -53,14 +54,21 @@ public class TestNMTimelinePublisher {
final DummyTimelineClient timelineClient = new DummyTimelineClient(null);
when(context.getNodeId()).thenReturn(NodeId.newInstance("localhost", 0));
when(context.getHttpPort()).thenReturn(0);
+
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+
NMTimelinePublisher publisher = new NMTimelinePublisher(context) {
public void createTimelineClient(ApplicationId appId) {
if (!getAppToClientMap().containsKey(appId)) {
+ timelineClient.init(getConfig());
+ timelineClient.start();
getAppToClientMap().put(appId, timelineClient);
}
}
};
- publisher.init(new Configuration());
+ publisher.init(conf);
publisher.start();
ApplicationId appId = ApplicationId.newInstance(0, 1);
publisher.createTimelineClient(appId);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-6050. AMs can't be scheduled
on racks or nodes (rkanter)
Posted by ae...@apache.org.
YARN-6050. AMs can't be scheduled on racks or nodes (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9bae6720
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9bae6720
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9bae6720
Branch: refs/heads/HDFS-7240
Commit: 9bae6720cb8432efd78c909dc624c00e367cedf5
Parents: 64ea62c
Author: Robert Kanter <rk...@apache.org>
Authored: Mon Mar 27 17:22:43 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Mon Mar 27 17:23:13 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/mapred/YARNRunner.java | 4 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 2 +-
.../records/ApplicationSubmissionContext.java | 57 +++-
.../src/main/proto/yarn_protos.proto | 2 +-
.../pb/ApplicationSubmissionContextPBImpl.java | 64 +++-
.../nodelabels/CommonNodeLabelsManager.java | 22 ++
.../server/resourcemanager/RMAppManager.java | 104 +++++--
.../server/resourcemanager/RMServerUtils.java | 68 ++++-
.../server/resourcemanager/rmapp/RMApp.java | 3 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 22 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 35 ++-
.../scheduler/AbstractYarnScheduler.java | 5 +
.../scheduler/ClusterNodeTracker.java | 65 +++-
.../scheduler/ResourceScheduler.java | 9 +
.../scheduler/common/fica/FiCaSchedulerApp.java | 10 +-
.../resourcemanager/webapp/dao/AppInfo.java | 2 +-
.../yarn/server/resourcemanager/MockRM.java | 45 ++-
.../server/resourcemanager/TestAppManager.java | 195 +++++++++++-
.../resourcemanager/TestClientRMService.java | 8 +-
.../TestNodeBlacklistingOnAMFailures.java | 184 ++++++++++++
.../resourcemanager/TestRMServerUtils.java | 297 +++++++++++++++++++
.../applicationsmanager/MockAsm.java | 6 +-
.../metrics/TestSystemMetricsPublisher.java | 3 +-
.../server/resourcemanager/rmapp/MockRMApp.java | 11 +-
.../rmapp/TestRMAppTransitions.java | 9 +-
.../attempt/TestRMAppAttemptTransitions.java | 19 +-
.../capacity/TestApplicationLimits.java | 4 +-
.../TestApplicationLimitsByPartition.java | 4 +-
.../capacity/TestCapacityScheduler.java | 2 +-
.../scheduler/capacity/TestLeafQueue.java | 3 +-
.../scheduler/fair/TestFairScheduler.java | 78 +++++
.../webapp/TestRMWebServicesApps.java | 10 +-
32 files changed, 1212 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 228c6af..2339c79 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -589,7 +590,8 @@ public class YARNRunner implements ClientProtocol {
amResourceRequest.setCapability(capability);
amResourceRequest.setNumContainers(1);
amResourceRequest.setNodeLabelExpression(amNodelabelExpression.trim());
- appContext.setAMContainerResourceRequest(amResourceRequest);
+ appContext.setAMContainerResourceRequests(
+ Collections.singletonList(amResourceRequest));
}
// set labels for the Job containers
appContext.setNodeLabelExpression(jobConf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index 279c8ce..c2bda62 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -571,7 +571,7 @@ public class TestYARNRunner {
buildSubmitContext(yarnRunner, jobConf);
assertEquals(appSubCtx.getNodeLabelExpression(), "GPU");
- assertEquals(appSubCtx.getAMContainerResourceRequest()
+ assertEquals(appSubCtx.getAMContainerResourceRequests().get(0)
.getNodeLabelExpression(), "highMem");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index e562aaa..4f1d147 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.api.records;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -100,7 +102,7 @@ public abstract class ApplicationSubmissionContext {
amReq.setNumContainers(1);
amReq.setRelaxLocality(true);
amReq.setNodeLabelExpression(amContainerLabelExpression);
- context.setAMContainerResourceRequest(amReq);
+ context.setAMContainerResourceRequests(Collections.singletonList(amReq));
return context;
}
@@ -159,7 +161,8 @@ public abstract class ApplicationSubmissionContext {
context.setApplicationType(applicationType);
context.setKeepContainersAcrossApplicationAttempts(keepContainers);
context.setNodeLabelExpression(appLabelExpression);
- context.setAMContainerResourceRequest(resourceRequest);
+ context.setAMContainerResourceRequests(
+ Collections.singletonList(resourceRequest));
return context;
}
@@ -454,30 +457,62 @@ public abstract class ApplicationSubmissionContext {
public abstract void setNodeLabelExpression(String nodeLabelExpression);
/**
- * Get ResourceRequest of AM container, if this is not null, scheduler will
- * use this to acquire resource for AM container.
- *
+ * Get the ResourceRequest of the AM container.
+ *
+ * If this is not null, scheduler will use this to acquire resource for AM
+ * container.
+ *
* If this is null, scheduler will assemble a ResourceRequest by using
* <em>getResource</em> and <em>getPriority</em> of
* <em>ApplicationSubmissionContext</em>.
- *
- * Number of containers and Priority will be ignore.
- *
- * @return ResourceRequest of AM container
+ *
+ * Number of containers and Priority will be ignored.
+ *
+ * @return ResourceRequest of the AM container
+ * @deprecated See {@link #getAMContainerResourceRequests()}
*/
@Public
@Evolving
+ @Deprecated
public abstract ResourceRequest getAMContainerResourceRequest();
/**
- * Set ResourceRequest of AM container
- * @param request of AM container
+ * Set ResourceRequest of the AM container
+ * @param request of the AM container
+ * @deprecated See {@link #setAMContainerResourceRequests(List)}
*/
@Public
@Evolving
+ @Deprecated
public abstract void setAMContainerResourceRequest(ResourceRequest request);
/**
+ * Get the ResourceRequests of the AM container.
+ *
+ * If this is not null, scheduler will use this to acquire resource for AM
+ * container.
+ *
+ * If this is null, scheduler will use the ResourceRequest as determined by
+ * <em>getAMContainerResourceRequest</em> and its behavior.
+ *
+ * Number of containers and Priority will be ignored.
+ *
+ * @return List of ResourceRequests of the AM container
+ */
+ @Public
+ @Evolving
+ public abstract List<ResourceRequest> getAMContainerResourceRequests();
+
+ /**
+ * Set ResourceRequests of the AM container.
+ * @param requests of the AM container
+ */
+ @Public
+ @Evolving
+ public abstract void setAMContainerResourceRequests(
+ List<ResourceRequest> requests);
+
+ /**
* Get the attemptFailuresValidityInterval in milliseconds for the application
*
* @return the attemptFailuresValidityInterval
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 3b26a5c..587354a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -378,7 +378,7 @@ message ApplicationSubmissionContextProto {
optional LogAggregationContextProto log_aggregation_context = 14;
optional ReservationIdProto reservation_id = 15;
optional string node_label_expression = 16;
- optional ResourceRequestProto am_container_resource_request = 17;
+ repeated ResourceRequestProto am_container_resource_request = 17;
repeated ApplicationTimeoutMapProto application_timeouts = 18;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
index 66930e7..0148d0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -66,7 +68,7 @@ extends ApplicationSubmissionContext {
private ContainerLaunchContext amContainer = null;
private Resource resource = null;
private Set<String> applicationTags = null;
- private ResourceRequest amResourceRequest = null;
+ private List<ResourceRequest> amResourceRequests = null;
private LogAggregationContext logAggregationContext = null;
private ReservationId reservationId = null;
private Map<ApplicationTimeoutType, Long> applicationTimeouts = null;
@@ -127,9 +129,10 @@ extends ApplicationSubmissionContext {
builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
- if (this.amResourceRequest != null) {
- builder.setAmContainerResourceRequest(
- convertToProtoFormat(this.amResourceRequest));
+ if (this.amResourceRequests != null) {
+ builder.clearAmContainerResourceRequest();
+ builder.addAllAmContainerResourceRequest(
+ convertToProtoFormat(this.amResourceRequests));
}
if (this.logAggregationContext != null) {
builder.setLogAggregationContext(
@@ -430,13 +433,23 @@ extends ApplicationSubmissionContext {
private PriorityProto convertToProtoFormat(Priority t) {
return ((PriorityPBImpl)t).getProto();
}
-
- private ResourceRequestPBImpl convertFromProtoFormat(ResourceRequestProto p) {
- return new ResourceRequestPBImpl(p);
+
+ private List<ResourceRequest> convertFromProtoFormat(
+ List<ResourceRequestProto> ps) {
+ List<ResourceRequest> rs = new ArrayList<>();
+ for (ResourceRequestProto p : ps) {
+ rs.add(new ResourceRequestPBImpl(p));
+ }
+ return rs;
}
- private ResourceRequestProto convertToProtoFormat(ResourceRequest t) {
- return ((ResourceRequestPBImpl)t).getProto();
+ private List<ResourceRequestProto> convertToProtoFormat(
+ List<ResourceRequest> ts) {
+ List<ResourceRequestProto> rs = new ArrayList<>(ts.size());
+ for (ResourceRequest t : ts) {
+ rs.add(((ResourceRequestPBImpl)t).getProto());
+ }
+ return rs;
}
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
@@ -485,25 +498,46 @@ extends ApplicationSubmissionContext {
}
@Override
+ @Deprecated
public ResourceRequest getAMContainerResourceRequest() {
+ List<ResourceRequest> reqs = getAMContainerResourceRequests();
+ if (reqs == null || reqs.isEmpty()) {
+ return null;
+ }
+ return getAMContainerResourceRequests().get(0);
+ }
+
+ @Override
+ public List<ResourceRequest> getAMContainerResourceRequests() {
ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
- if (this.amResourceRequest != null) {
- return amResourceRequest;
+ if (this.amResourceRequests != null) {
+ return amResourceRequests;
} // Else via proto
- if (!p.hasAmContainerResourceRequest()) {
+ if (p.getAmContainerResourceRequestCount() == 0) {
return null;
}
- amResourceRequest = convertFromProtoFormat(p.getAmContainerResourceRequest());
- return amResourceRequest;
+ amResourceRequests =
+ convertFromProtoFormat(p.getAmContainerResourceRequestList());
+ return amResourceRequests;
}
@Override
+ @Deprecated
public void setAMContainerResourceRequest(ResourceRequest request) {
maybeInitBuilder();
if (request == null) {
builder.clearAmContainerResourceRequest();
}
- this.amResourceRequest = request;
+ this.amResourceRequests = Collections.singletonList(request);
+ }
+
+ @Override
+ public void setAMContainerResourceRequests(List<ResourceRequest> requests) {
+ maybeInitBuilder();
+ if (requests == null) {
+ builder.clearAmContainerResourceRequest();
+ }
+ this.amResourceRequests = requests;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index f3f4ba0..60ade2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -801,6 +801,28 @@ public class CommonNodeLabelsManager extends AbstractService {
}
}
+ /**
+ * Get nodes that have no labels.
+ *
+ * @return set of nodes with no labels
+ */
+ public Set<NodeId> getNodesWithoutALabel() {
+ try {
+ readLock.lock();
+ Set<NodeId> nodes = new HashSet<>();
+ for (Host host : nodeCollections.values()) {
+ for (NodeId nodeId : host.nms.keySet()) {
+ if (getLabelsByNode(nodeId).isEmpty()) {
+ nodes.add(nodeId);
+ }
+ }
+ }
+ return Collections.unmodifiableSet(nodes);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
/**
* Get mapping of labels to nodes for all the labels.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 1f446e6..e0cff7b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -17,7 +17,9 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager;
+import java.util.Collections;
import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -31,6 +33,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -337,14 +341,16 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
// has been disabled. Reject the recovery of this application if it
// is true and give clear message so that user can react properly.
if (!appContext.getUnmanagedAM() &&
- application.getAMResourceRequest() == null &&
+ (application.getAMResourceRequests() == null ||
+ application.getAMResourceRequests().isEmpty()) &&
!YarnConfiguration.areNodeLabelsEnabled(this.conf)) {
// check application submission context and see if am resource request
// or application itself contains any node label expression.
- ResourceRequest amReqFromAppContext =
- appContext.getAMContainerResourceRequest();
- String labelExp = (amReqFromAppContext != null) ?
- amReqFromAppContext.getNodeLabelExpression() : null;
+ List<ResourceRequest> amReqsFromAppContext =
+ appContext.getAMContainerResourceRequests();
+ String labelExp =
+ (amReqsFromAppContext != null && !amReqsFromAppContext.isEmpty()) ?
+ amReqsFromAppContext.get(0).getNodeLabelExpression() : null;
if (labelExp == null) {
labelExp = appContext.getNodeLabelExpression();
}
@@ -379,9 +385,9 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
}
ApplicationId applicationId = submissionContext.getApplicationId();
- ResourceRequest amReq = null;
+ List<ResourceRequest> amReqs = null;
try {
- amReq = validateAndCreateResourceRequest(submissionContext, isRecovery);
+ amReqs = validateAndCreateResourceRequest(submissionContext, isRecovery);
} catch (InvalidLabelResourceRequestException e) {
// This can happen if the application had been submitted and run
// with Node Label enabled but recover with Node Label disabled.
@@ -444,7 +450,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
submissionContext.getQueue(),
submissionContext, this.scheduler, this.masterService,
submitTime, submissionContext.getApplicationType(),
- submissionContext.getApplicationTags(), amReq, startTime);
+ submissionContext.getApplicationTags(), amReqs, startTime);
// Concurrent app submissions with same applicationId will fail here
// Concurrent app submissions with different applicationIds will not
// influence each other
@@ -470,7 +476,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
return application;
}
- private ResourceRequest validateAndCreateResourceRequest(
+ private List<ResourceRequest> validateAndCreateResourceRequest(
ApplicationSubmissionContext submissionContext, boolean isRecovery)
throws InvalidResourceRequestException {
// Validation of the ApplicationSubmissionContext needs to be completed
@@ -480,33 +486,77 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
// Check whether AM resource requirements are within required limits
if (!submissionContext.getUnmanagedAM()) {
- ResourceRequest amReq = submissionContext.getAMContainerResourceRequest();
- if (amReq == null) {
- amReq = BuilderUtils
- .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, submissionContext.getResource(), 1);
- }
-
- // set label expression for AM container
- if (null == amReq.getNodeLabelExpression()) {
- amReq.setNodeLabelExpression(submissionContext
- .getNodeLabelExpression());
+ List<ResourceRequest> amReqs =
+ submissionContext.getAMContainerResourceRequests();
+ if (amReqs == null || amReqs.isEmpty()) {
+ if (submissionContext.getResource() != null) {
+ amReqs = Collections.singletonList(BuilderUtils
+ .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+ ResourceRequest.ANY, submissionContext.getResource(), 1));
+ } else {
+ throw new InvalidResourceRequestException("Invalid resource request, "
+ + "no resources requested");
+ }
}
try {
- SchedulerUtils.normalizeAndValidateRequest(amReq,
- scheduler.getMaximumResourceCapability(),
- submissionContext.getQueue(), scheduler, isRecovery, rmContext);
+ // Find the ANY request and ensure there's only one
+ ResourceRequest anyReq = null;
+ for (ResourceRequest amReq : amReqs) {
+ if (amReq.getResourceName().equals(ResourceRequest.ANY)) {
+ if (anyReq == null) {
+ anyReq = amReq;
+ } else {
+ throw new InvalidResourceRequestException("Invalid resource "
+ + "request, only one resource request with "
+ + ResourceRequest.ANY + " is allowed");
+ }
+ }
+ }
+ if (anyReq == null) {
+ throw new InvalidResourceRequestException("Invalid resource request, "
+ + "no resource request specified with " + ResourceRequest.ANY);
+ }
+
+ // Make sure that all of the requests agree with the ANY request
+ // and have correct values
+ for (ResourceRequest amReq : amReqs) {
+ amReq.setCapability(anyReq.getCapability());
+ amReq.setExecutionTypeRequest(
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ amReq.setNumContainers(1);
+ amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY);
+ }
+
+ // set label expression for AM ANY request if not set
+ if (null == anyReq.getNodeLabelExpression()) {
+ anyReq.setNodeLabelExpression(submissionContext
+ .getNodeLabelExpression());
+ }
+
+ // Put ANY request at the front
+ if (!amReqs.get(0).equals(anyReq)) {
+ amReqs.remove(anyReq);
+ amReqs.add(0, anyReq);
+ }
+
+ // Normalize all requests
+ for (ResourceRequest amReq : amReqs) {
+ SchedulerUtils.normalizeAndValidateRequest(amReq,
+ scheduler.getMaximumResourceCapability(),
+ submissionContext.getQueue(), scheduler, isRecovery, rmContext);
+
+ amReq.setCapability(
+ scheduler.getNormalizedResource(amReq.getCapability()));
+ }
+ return amReqs;
} catch (InvalidResourceRequestException e) {
LOG.warn("RM app submission failed in validating AM resource request"
+ " for application " + submissionContext.getApplicationId(), e);
throw e;
}
-
- amReq.setCapability(scheduler.getNormalizedResource(amReq.getCapability()));
- return amReq;
}
-
+
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 0aa7a2c..35b0c98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -21,13 +21,16 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.AccessControlException;
@@ -41,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -557,19 +561,65 @@ public class RMServerUtils {
*
* @param rmContext context
* @param conf configuration
- * @param amreq am resource request
+ * @param amReqs am resource requests
* @return applicable node count
*/
public static int getApplicableNodeCountForAM(RMContext rmContext,
- Configuration conf, ResourceRequest amreq) {
+ Configuration conf, List<ResourceRequest> amReqs) {
+ // Determine the list of nodes that are eligible based on the strict
+ // resource requests
+ Set<NodeId> nodesForReqs = new HashSet<>();
+ for (ResourceRequest amReq : amReqs) {
+ if (amReq.getRelaxLocality() &&
+ !amReq.getResourceName().equals(ResourceRequest.ANY)) {
+ nodesForReqs.addAll(
+ rmContext.getScheduler().getNodeIds(amReq.getResourceName()));
+ }
+ }
+
if (YarnConfiguration.areNodeLabelsEnabled(conf)) {
- RMNodeLabelsManager labelManager = rmContext.getNodeLabelManager();
- String amNodeLabelExpression = amreq.getNodeLabelExpression();
- amNodeLabelExpression = (amNodeLabelExpression == null
- || amNodeLabelExpression.trim().isEmpty())
- ? RMNodeLabelsManager.NO_LABEL : amNodeLabelExpression;
- return labelManager.getActiveNMCountPerLabel(amNodeLabelExpression);
+ // Determine the list of nodes that are eligible based on the node label
+ String amNodeLabelExpression = amReqs.get(0).getNodeLabelExpression();
+ Set<NodeId> nodesForLabels =
+ getNodeIdsForLabel(rmContext, amNodeLabelExpression);
+ if (nodesForLabels != null && !nodesForLabels.isEmpty()) {
+ // If only node labels, strip out any wildcard NodeIds and return
+ if (nodesForReqs.isEmpty()) {
+ for (Iterator<NodeId> it = nodesForLabels.iterator(); it.hasNext();) {
+ if (it.next().getPort() == 0) {
+ it.remove();
+ }
+ }
+ return nodesForLabels.size();
+ } else {
+ // The NodeIds common to both the strict resource requests and the
+ // node label is the eligible set
+ return Sets.intersection(nodesForReqs, nodesForLabels).size();
+ }
+ }
+ }
+
+ // If no strict resource request NodeIds nor node label NodeIds, then just
+ // return the entire cluster
+ if (nodesForReqs.isEmpty()) {
+ return rmContext.getScheduler().getNumClusterNodes();
+ }
+ // No node label NodeIds, so return the strict resource request NodeIds
+ return nodesForReqs.size();
+ }
+
+ private static Set<NodeId> getNodeIdsForLabel(RMContext rmContext,
+ String label) {
+ label = (label == null || label.trim().isEmpty())
+ ? RMNodeLabelsManager.NO_LABEL : label;
+ if (label.equals(RMNodeLabelsManager.NO_LABEL)) {
+ // NO_LABEL nodes aren't tracked directly
+ return rmContext.getNodeLabelManager().getNodesWithoutALabel();
+ } else {
+ Map<String, Set<NodeId>> labelsToNodes =
+ rmContext.getNodeLabelManager().getLabelsToNodes(
+ Collections.singleton(label));
+ return labelsToNodes.get(label);
}
- return rmContext.getScheduler().getNumClusterNodes();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index b3a87a6..43fd1fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import java.util.Collection;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -269,7 +270,7 @@ public interface RMApp extends EventHandler<RMAppEvent> {
ReservationId getReservationId();
- ResourceRequest getAMResourceRequest();
+ List<ResourceRequest> getAMResourceRequests();
Map<NodeId, LogAggregationReport> getLogAggregationReportsForApp();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index f24908b..531844c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -196,7 +196,7 @@ public class RMAppImpl implements RMApp, Recoverable {
private RMAppEvent eventCausingFinalSaving;
private RMAppState targetedFinalState;
private RMAppState recoveredFinalState;
- private ResourceRequest amReq;
+ private List<ResourceRequest> amReqs;
private CallerContext callerContext;
@@ -424,10 +424,10 @@ public class RMAppImpl implements RMApp, Recoverable {
ApplicationSubmissionContext submissionContext, YarnScheduler scheduler,
ApplicationMasterService masterService, long submitTime,
String applicationType, Set<String> applicationTags,
- ResourceRequest amReq) {
+ List<ResourceRequest> amReqs) {
this(applicationId, rmContext, config, name, user, queue, submissionContext,
scheduler, masterService, submitTime, applicationType, applicationTags,
- amReq, -1);
+ amReqs, -1);
}
public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
@@ -435,7 +435,7 @@ public class RMAppImpl implements RMApp, Recoverable {
ApplicationSubmissionContext submissionContext, YarnScheduler scheduler,
ApplicationMasterService masterService, long submitTime,
String applicationType, Set<String> applicationTags,
- ResourceRequest amReq, long startTime) {
+ List<ResourceRequest> amReqs, long startTime) {
this.systemClock = SystemClock.getInstance();
@@ -458,7 +458,7 @@ public class RMAppImpl implements RMApp, Recoverable {
}
this.applicationType = applicationType;
this.applicationTags = applicationTags;
- this.amReq = amReq;
+ this.amReqs = amReqs;
if (submissionContext.getPriority() != null) {
this.applicationPriority = Priority
.newInstance(submissionContext.getPriority().getPriority());
@@ -987,7 +987,7 @@ public class RMAppImpl implements RMApp, Recoverable {
if (amBlacklistingEnabled && !submissionContext.getUnmanagedAM()) {
currentAMBlacklistManager = new SimpleBlacklistManager(
RMServerUtils.getApplicableNodeCountForAM(rmContext, conf,
- getAMResourceRequest()),
+ getAMResourceRequests()),
blacklistDisableThreshold);
} else {
currentAMBlacklistManager = new DisabledBlacklistManager();
@@ -995,7 +995,7 @@ public class RMAppImpl implements RMApp, Recoverable {
}
RMAppAttempt attempt =
new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
- submissionContext, conf, amReq, this, currentAMBlacklistManager);
+ submissionContext, conf, amReqs, this, currentAMBlacklistManager);
attempts.put(appAttemptId, attempt);
currentAttempt = attempt;
}
@@ -1690,8 +1690,8 @@ public class RMAppImpl implements RMApp, Recoverable {
}
@Override
- public ResourceRequest getAMResourceRequest() {
- return this.amReq;
+ public List<ResourceRequest> getAMResourceRequests() {
+ return this.amReqs;
}
@Override
@@ -1964,7 +1964,9 @@ public class RMAppImpl implements RMApp, Recoverable {
public String getAmNodeLabelExpression() {
String amNodeLabelExpression = null;
if (!getApplicationSubmissionContext().getUnmanagedAM()) {
- amNodeLabelExpression = getAMResourceRequest().getNodeLabelExpression();
+ amNodeLabelExpression =
+ getAMResourceRequests() != null && !getAMResourceRequests().isEmpty()
+ ? getAMResourceRequests().get(0).getNodeLabelExpression() : null;
amNodeLabelExpression = (amNodeLabelExpression == null)
? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : amNodeLabelExpression;
amNodeLabelExpression = (amNodeLabelExpression.trim().isEmpty())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 5c0f48e..19503e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -192,7 +192,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
private Object transitionTodo;
private RMAppAttemptMetrics attemptMetrics = null;
- private ResourceRequest amReq = null;
+ private List<ResourceRequest> amReqs = null;
private BlacklistManager blacklistedNodesForAM = null;
private String amLaunchDiagnostics;
@@ -485,16 +485,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
- Configuration conf, ResourceRequest amReq, RMApp rmApp) {
+ Configuration conf, List<ResourceRequest> amReqs, RMApp rmApp) {
this(appAttemptId, rmContext, scheduler, masterService, submissionContext,
- conf, amReq, rmApp, new DisabledBlacklistManager());
+ conf, amReqs, rmApp, new DisabledBlacklistManager());
}
public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
- Configuration conf, ResourceRequest amReq, RMApp rmApp,
+ Configuration conf, List<ResourceRequest> amReqs, RMApp rmApp,
BlacklistManager amBlacklistManager) {
this.conf = conf;
this.applicationAttemptId = appAttemptId;
@@ -514,7 +514,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.attemptMetrics =
new RMAppAttemptMetrics(applicationAttemptId, rmContext);
- this.amReq = amReq;
+ this.amReqs = amReqs;
this.blacklistedNodesForAM = amBlacklistManager;
final int diagnosticsLimitKC = getDiagnosticsLimitKCOrThrow(conf);
@@ -1090,18 +1090,21 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
// will be passed to scheduler, and scheduler will deduct the number after
// AM container allocated
- // Currently, following fields are all hard code,
+ // Currently, following fields are all hard coded,
// TODO: change these fields when we want to support
- // priority/resource-name/relax-locality specification for AM containers
- // allocation.
- appAttempt.amReq.setNumContainers(1);
- appAttempt.amReq.setPriority(AM_CONTAINER_PRIORITY);
- appAttempt.amReq.setResourceName(ResourceRequest.ANY);
- appAttempt.amReq.setRelaxLocality(true);
-
- appAttempt.getAMBlacklistManager().refreshNodeHostCount(
+ // priority or multiple containers AM container allocation.
+ for (ResourceRequest amReq : appAttempt.amReqs) {
+ amReq.setNumContainers(1);
+ amReq.setPriority(AM_CONTAINER_PRIORITY);
+ }
+
+ int numNodes =
RMServerUtils.getApplicableNodeCountForAM(appAttempt.rmContext,
- appAttempt.conf, appAttempt.amReq));
+ appAttempt.conf, appAttempt.amReqs);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting node count for blacklist to " + numNodes);
+ }
+ appAttempt.getAMBlacklistManager().refreshNodeHostCount(numNodes);
ResourceBlacklistRequest amBlacklist =
appAttempt.getAMBlacklistManager().getBlacklistUpdates();
@@ -1114,7 +1117,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
Allocation amContainerAllocation =
appAttempt.scheduler.allocate(
appAttempt.applicationAttemptId,
- Collections.singletonList(appAttempt.amReq),
+ appAttempt.amReqs,
EMPTY_CONTAINER_RELEASE_LIST,
amBlacklist.getBlacklistAdditions(),
amBlacklist.getBlacklistRemovals(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 213839d..b954bdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -1257,4 +1257,9 @@ public abstract class AbstractYarnScheduler
rmContainer.getLastConfirmedResource(), null)));
}
}
+
+ @Override
+ public List<NodeId> getNodeIds(String resourceName) {
+ return nodeTracker.getNodeIdsByResourceName(resourceName);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index e487f69..010e645 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -268,6 +268,9 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
/**
* Convenience method to filter nodes based on a condition.
+ *
+ * @param nodeFilter A {@link NodeFilter} for filtering the nodes
+ * @return A list of filtered nodes
*/
public List<N> getNodes(NodeFilter nodeFilter) {
List<N> nodeList = new ArrayList<>();
@@ -288,6 +291,37 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
return nodeList;
}
+ public List<NodeId> getAllNodeIds() {
+ return getNodeIds(null);
+ }
+
+ /**
+ * Convenience method to filter nodes based on a condition.
+ *
+ * @param nodeFilter A {@link NodeFilter} for filtering the nodes
+ * @return A list of filtered nodes
+ */
+ public List<NodeId> getNodeIds(NodeFilter nodeFilter) {
+ List<NodeId> nodeList = new ArrayList<>();
+ readLock.lock();
+ try {
+ if (nodeFilter == null) {
+ for (N node : nodes.values()) {
+ nodeList.add(node.getNodeID());
+ }
+ } else {
+ for (N node : nodes.values()) {
+ if (nodeFilter.accept(node)) {
+ nodeList.add(node.getNodeID());
+ }
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return nodeList;
+ }
+
/**
* Convenience method to sort nodes.
*
@@ -320,11 +354,38 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
resourceName != null && !resourceName.isEmpty());
List<N> retNodes = new ArrayList<>();
if (ResourceRequest.ANY.equals(resourceName)) {
- return getAllNodes();
+ retNodes.addAll(getAllNodes());
} else if (nodeNameToNodeMap.containsKey(resourceName)) {
retNodes.add(nodeNameToNodeMap.get(resourceName));
} else if (nodesPerRack.containsKey(resourceName)) {
- return nodesPerRack.get(resourceName);
+ retNodes.addAll(nodesPerRack.get(resourceName));
+ } else {
+ LOG.info(
+ "Could not find a node matching given resourceName " + resourceName);
+ }
+ return retNodes;
+ }
+
+ /**
+ * Convenience method to return list of {@link NodeId} corresponding to
+ * resourceName passed in the {@link ResourceRequest}.
+ *
+ * @param resourceName Host/rack name of the resource, or
+ * {@link ResourceRequest#ANY}
+ * @return list of {@link NodeId} that match the resourceName
+ */
+ public List<NodeId> getNodeIdsByResourceName(final String resourceName) {
+ Preconditions.checkArgument(
+ resourceName != null && !resourceName.isEmpty());
+ List<NodeId> retNodes = new ArrayList<>();
+ if (ResourceRequest.ANY.equals(resourceName)) {
+ retNodes.addAll(getAllNodeIds());
+ } else if (nodeNameToNodeMap.containsKey(resourceName)) {
+ retNodes.add(nodeNameToNodeMap.get(resourceName).getNodeID());
+ } else if (nodesPerRack.containsKey(resourceName)) {
+ for (N node : nodesPerRack.get(resourceName)) {
+ retNodes.add(node.getNodeID());
+ }
} else {
LOG.info(
"Could not find a node matching given resourceName " + resourceName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
index 5649ccf..d96d625 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
@@ -19,10 +19,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.io.IOException;
+import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
@@ -49,4 +51,11 @@ public interface ResourceScheduler extends YarnScheduler, Recoverable {
* @throws IOException
*/
void reinitialize(Configuration conf, RMContext rmContext) throws IOException;
+
+ /**
+ * Get the {@link NodeId} available in the cluster by resource name.
+ * @param resourceName resource name
+ * @return the number of available {@link NodeId} by resource name.
+ */
+ List<NodeId> getNodeIds(String resourceName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index fea29bb..5c0b718 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -141,18 +141,20 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
Resource amResource;
String partition;
- if (rmApp == null || rmApp.getAMResourceRequest() == null) {
+ if (rmApp == null || rmApp.getAMResourceRequests() == null
+ || rmApp.getAMResourceRequests().isEmpty()) {
// the rmApp may be undefined (the resource manager checks for this too)
// and unmanaged applications do not provide an amResource request
// in these cases, provide a default using the scheduler
amResource = rmContext.getScheduler().getMinimumResourceCapability();
partition = CommonNodeLabelsManager.NO_LABEL;
} else {
- amResource = rmApp.getAMResourceRequest().getCapability();
+ amResource = rmApp.getAMResourceRequests().get(0).getCapability();
partition =
- (rmApp.getAMResourceRequest().getNodeLabelExpression() == null)
+ (rmApp.getAMResourceRequests().get(0)
+ .getNodeLabelExpression() == null)
? CommonNodeLabelsManager.NO_LABEL
- : rmApp.getAMResourceRequest().getNodeLabelExpression();
+ : rmApp.getAMResourceRequests().get(0).getNodeLabelExpression();
}
setAppAMNodePartitionName(partition);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 4e85b67..10e627a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -229,7 +229,7 @@ public class AppInfo {
appNodeLabelExpression =
app.getApplicationSubmissionContext().getNodeLabelExpression();
amNodeLabelExpression = (unmanagedApplication) ? null
- : app.getAMResourceRequest().getNodeLabelExpression();
+ : app.getAMResourceRequests().get(0).getNodeLabelExpression();
// Setting partition based resource usage of application
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index f9f42ad..aca2fc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -24,6 +24,7 @@ import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -678,6 +679,17 @@ public class MockRM extends ResourceManager {
tokensConf);
}
+ public RMApp submitApp(List<ResourceRequest> amResourceRequests)
+ throws Exception {
+ return submitApp(amResourceRequests, "app1",
+ "user", null, false, null,
+ super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true,
+ false, false, null, 0, null, true,
+ amResourceRequests.get(0).getPriority(),
+ amResourceRequests.get(0).getNodeLabelExpression(), null, null);
+ }
+
public RMApp submitApp(Resource capability, String name, String user,
Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
int maxAppAttempts, Credentials ts, String appType,
@@ -688,6 +700,30 @@ public class MockRM extends ResourceManager {
Map<ApplicationTimeoutType, Long> applicationTimeouts,
ByteBuffer tokensConf)
throws Exception {
+ priority = (priority == null) ? Priority.newInstance(0) : priority;
+ ResourceRequest amResourceRequest = ResourceRequest.newInstance(
+ priority, ResourceRequest.ANY, capability, 1);
+ if (amLabel != null && !amLabel.isEmpty()) {
+ amResourceRequest.setNodeLabelExpression(amLabel.trim());
+ }
+ return submitApp(Collections.singletonList(amResourceRequest), name, user,
+ acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted,
+ keepContainers, isAppIdProvided, applicationId,
+ attemptFailuresValidityInterval, logAggregationContext,
+ cancelTokensWhenComplete, priority, amLabel, applicationTimeouts,
+ tokensConf);
+ }
+
+ public RMApp submitApp(List<ResourceRequest> amResourceRequests, String name,
+ String user, Map<ApplicationAccessType, String> acls, boolean unmanaged,
+ String queue, int maxAppAttempts, Credentials ts, String appType,
+ boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
+ ApplicationId applicationId, long attemptFailuresValidityInterval,
+ LogAggregationContext logAggregationContext,
+ boolean cancelTokensWhenComplete, Priority priority, String amLabel,
+ Map<ApplicationTimeoutType, Long> applicationTimeouts,
+ ByteBuffer tokensConf)
+ throws Exception {
ApplicationId appId = isAppIdProvided ? applicationId : null;
ApplicationClientProtocol client = getClientRMService();
if (! isAppIdProvided) {
@@ -718,7 +754,6 @@ public class MockRM extends ResourceManager {
sub.setApplicationType(appType);
ContainerLaunchContext clc = Records
.newRecord(ContainerLaunchContext.class);
- sub.setResource(capability);
clc.setApplicationACLs(acls);
if (ts != null && UserGroupInformation.isSecurityEnabled()) {
DataOutputBuffer dob = new DataOutputBuffer();
@@ -733,12 +768,12 @@ public class MockRM extends ResourceManager {
sub.setLogAggregationContext(logAggregationContext);
}
sub.setCancelTokensWhenComplete(cancelTokensWhenComplete);
- ResourceRequest amResourceRequest = ResourceRequest.newInstance(
- Priority.newInstance(0), ResourceRequest.ANY, capability, 1);
if (amLabel != null && !amLabel.isEmpty()) {
- amResourceRequest.setNodeLabelExpression(amLabel.trim());
+ for (ResourceRequest amResourceRequest : amResourceRequests) {
+ amResourceRequest.setNodeLabelExpression(amLabel.trim());
+ }
}
- sub.setAMContainerResourceRequest(amResourceRequest);
+ sub.setAMContainerResourceRequests(amResourceRequests);
req.setApplicationSubmissionContext(sub);
UserGroupInformation fakeUser =
UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 892f8ba..eb69efa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -31,6 +31,8 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
@@ -50,6 +52,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -57,11 +61,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
@@ -72,6 +78,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@@ -312,7 +319,7 @@ public class TestAppManager{
ResourceRequest resReg =
ResourceRequest.newInstance(Priority.newInstance(0),
ResourceRequest.ANY, Resource.newInstance(1024, 1), 1);
- sub.setAMContainerResourceRequest(resReg);
+ sub.setAMContainerResourceRequests(Collections.singletonList(resReg));
req.setApplicationSubmissionContext(sub);
sub.setAMContainerSpec(mock(ContainerLaunchContext.class));
try {
@@ -522,8 +529,157 @@ public class TestAppManager{
Assert.assertEquals("app event type is wrong before", RMAppEventType.KILL, appEventType);
}
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testRMAppSubmitAMContainerResourceRequests() throws Exception {
+ asContext.setResource(Resources.createResource(1024));
+ asContext.setAMContainerResourceRequest(
+ ResourceRequest.newInstance(Priority.newInstance(0),
+ ResourceRequest.ANY, Resources.createResource(1024), 1, true));
+ List<ResourceRequest> reqs = new ArrayList<>();
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
+ ResourceRequest.ANY, Resources.createResource(1025), 1, false));
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
+ "/rack", Resources.createResource(1025), 1, false));
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
+ "/rack/node", Resources.createResource(1025), 1, true));
+ asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
+ // getAMContainerResourceRequest uses the first entry of
+ // getAMContainerResourceRequests
+ Assert.assertEquals(reqs.get(0), asContext.getAMContainerResourceRequest());
+ Assert.assertEquals(reqs, asContext.getAMContainerResourceRequests());
+ RMApp app = testRMAppSubmit();
+ for (ResourceRequest req : reqs) {
+ req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ }
+ // setAMContainerResourceRequests has priority over
+ // setAMContainerResourceRequest and setResource
+ Assert.assertEquals(reqs, app.getAMResourceRequests());
+ }
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testRMAppSubmitAMContainerResourceRequest() throws Exception {
+ asContext.setResource(Resources.createResource(1024));
+ asContext.setAMContainerResourceRequests(null);
+ ResourceRequest req =
+ ResourceRequest.newInstance(Priority.newInstance(0),
+ ResourceRequest.ANY, Resources.createResource(1025), 1, true);
+ asContext.setAMContainerResourceRequest(cloneResourceRequest(req));
+ // getAMContainerResourceRequests uses a singleton list of
+ // getAMContainerResourceRequest
+ Assert.assertEquals(req, asContext.getAMContainerResourceRequest());
+ Assert.assertEquals(req, asContext.getAMContainerResourceRequests().get(0));
+ Assert.assertEquals(1, asContext.getAMContainerResourceRequests().size());
+ RMApp app = testRMAppSubmit();
+ req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ // setAMContainerResourceRequest has priority over setResource
+ Assert.assertEquals(Collections.singletonList(req),
+ app.getAMResourceRequests());
+ }
+
@Test
- public void testRMAppSubmit() throws Exception {
+ public void testRMAppSubmitResource() throws Exception {
+ asContext.setResource(Resources.createResource(1024));
+ asContext.setAMContainerResourceRequests(null);
+ RMApp app = testRMAppSubmit();
+ // setResource
+ Assert.assertEquals(Collections.singletonList(
+ ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+ ResourceRequest.ANY, Resources.createResource(1024), 1, true, "")),
+ app.getAMResourceRequests());
+ }
+
+ @Test
+ public void testRMAppSubmitNoResourceRequests() throws Exception {
+ asContext.setResource(null);
+ asContext.setAMContainerResourceRequests(null);
+ try {
+ testRMAppSubmit();
+ Assert.fail("Should have failed due to no ResourceRequest");
+ } catch (InvalidResourceRequestException e) {
+ Assert.assertEquals(
+ "Invalid resource request, no resources requested",
+ e.getMessage());
+ }
+ }
+
+ @Test
+ public void testRMAppSubmitAMContainerResourceRequestsDisagree()
+ throws Exception {
+ asContext.setResource(null);
+ List<ResourceRequest> reqs = new ArrayList<>();
+ ResourceRequest anyReq = ResourceRequest.newInstance(
+ Priority.newInstance(1),
+ ResourceRequest.ANY, Resources.createResource(1024), 1, false, "label1",
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ reqs.add(anyReq);
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(2),
+ "/rack", Resources.createResource(1025), 2, false, "",
+ ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC)));
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(3),
+ "/rack/node", Resources.createResource(1026), 3, true, "",
+ ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC)));
+ asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
+ RMApp app = testRMAppSubmit();
+ // It should force the requests to all agree on these points
+ for (ResourceRequest req : reqs) {
+ req.setCapability(anyReq.getCapability());
+ req.setExecutionTypeRequest(
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ req.setNumContainers(1);
+ req.setPriority(Priority.newInstance(0));
+ }
+ Assert.assertEquals(reqs, app.getAMResourceRequests());
+ }
+
+ @Test
+ public void testRMAppSubmitAMContainerResourceRequestsNoAny()
+ throws Exception {
+ asContext.setResource(null);
+ List<ResourceRequest> reqs = new ArrayList<>();
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(1),
+ "/rack", Resources.createResource(1025), 1, false));
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(1),
+ "/rack/node", Resources.createResource(1025), 1, true));
+ asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
+ // getAMContainerResourceRequest uses the first entry of
+ // getAMContainerResourceRequests
+ Assert.assertEquals(reqs, asContext.getAMContainerResourceRequests());
+ try {
+ testRMAppSubmit();
+ Assert.fail("Should have failed due to missing ANY ResourceRequest");
+ } catch (InvalidResourceRequestException e) {
+ Assert.assertEquals(
+ "Invalid resource request, no resource request specified with *",
+ e.getMessage());
+ }
+ }
+
+ @Test
+ public void testRMAppSubmitAMContainerResourceRequestsTwoManyAny()
+ throws Exception {
+ asContext.setResource(null);
+ List<ResourceRequest> reqs = new ArrayList<>();
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(1),
+ ResourceRequest.ANY, Resources.createResource(1025), 1, false));
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(1),
+ ResourceRequest.ANY, Resources.createResource(1025), 1, false));
+ asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
+ // getAMContainerResourceRequest uses the first entry of
+ // getAMContainerResourceRequests
+ Assert.assertEquals(reqs, asContext.getAMContainerResourceRequests());
+ try {
+ testRMAppSubmit();
+ Assert.fail("Should have failed due to too many ANY ResourceRequests");
+ } catch (InvalidResourceRequestException e) {
+ Assert.assertEquals(
+ "Invalid resource request, only one resource request with * is " +
+ "allowed", e.getMessage());
+ }
+ }
+
+ private RMApp testRMAppSubmit() throws Exception {
appMonitor.submitApplication(asContext, "test");
RMApp app = rmContext.getRMApps().get(appId);
Assert.assertNotNull("app is null", app);
@@ -534,12 +690,14 @@ public class TestAppManager{
// wait for event to be processed
int timeoutSecs = 0;
- while ((getAppEventType() == RMAppEventType.KILL) &&
+ while ((getAppEventType() == RMAppEventType.KILL) &&
timeoutSecs++ < 20) {
Thread.sleep(1000);
}
Assert.assertEquals("app event type sent is wrong", RMAppEventType.START,
getAppEventType());
+
+ return app;
}
@Test
@@ -737,6 +895,15 @@ public class TestAppManager{
ResourceCalculator rs = mock(ResourceCalculator.class);
when(scheduler.getResourceCalculator()).thenReturn(rs);
+ when(scheduler.getNormalizedResource(any()))
+ .thenAnswer(new Answer<Resource>() {
+ @Override
+ public Resource answer(InvocationOnMock invocationOnMock)
+ throws Throwable {
+ return (Resource) invocationOnMock.getArguments()[0];
+ }
+ });
+
return scheduler;
}
@@ -753,4 +920,26 @@ public class TestAppManager{
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
}
+ private static ResourceRequest cloneResourceRequest(ResourceRequest req) {
+ return ResourceRequest.newInstance(
+ Priority.newInstance(req.getPriority().getPriority()),
+ new String(req.getResourceName()),
+ Resource.newInstance(req.getCapability().getMemorySize(),
+ req.getCapability().getVirtualCores()),
+ req.getNumContainers(),
+ req.getRelaxLocality(),
+ req.getNodeLabelExpression() != null
+ ? new String(req.getNodeLabelExpression()) : null,
+ ExecutionTypeRequest.newInstance(
+ req.getExecutionTypeRequest().getExecutionType()));
+ }
+
+ private static List<ResourceRequest> cloneResourceRequests(
+ List<ResourceRequest> reqs) {
+ List<ResourceRequest> cloneReqs = new ArrayList<>();
+ for (ResourceRequest req : reqs) {
+ cloneReqs.add(cloneResourceRequest(req));
+ }
+ return cloneReqs;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 7a67aa8..f0e60a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -38,6 +38,7 @@ import java.security.AccessControlException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -1307,9 +1308,9 @@ public class TestClientRMService {
spy(new RMAppImpl(applicationId3, rmContext, config, null, null,
queueName, asContext, yarnScheduler, null,
System.currentTimeMillis(), "YARN", null,
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- Resource.newInstance(1024, 1), 1)){
+ Resource.newInstance(1024, 1), 1))){
@Override
public ApplicationReport createAndGetApplicationReport(
String clientUserName, boolean allowAccess) {
@@ -1323,7 +1324,8 @@ public class TestClientRMService {
return report;
}
});
- app.getAMResourceRequest().setNodeLabelExpression(amNodeLabelExpression);
+ app.getAMResourceRequests().get(0)
+ .setNodeLabelExpression(amNodeLabelExpression);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
ApplicationId.newInstance(123456, 1), 1);
RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
index c80a799..b4adf48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -28,6 +29,9 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -157,6 +161,186 @@ public class TestNodeBlacklistingOnAMFailures {
}
@Test(timeout = 100000)
+ public void testNodeBlacklistingOnAMFailureStrictNodeLocality()
+ throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED,
+ true);
+
+ DrainDispatcher dispatcher = new DrainDispatcher();
+ MockRM rm = startRM(conf, dispatcher);
+ CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
+
+ // Register 5 nodes, so that we can blacklist atleast one if AM container
+ // is failed. As per calculation it will be like, 5nodes * 0.2 (default)=1.
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService());
+ nm1.registerNode();
+
+ MockNM nm2 =
+ new MockNM("127.0.0.2:2345", 8000, rm.getResourceTrackerService());
+ nm2.registerNode();
+
+ MockNM nm3 =
+ new MockNM("127.0.0.3:2345", 8000, rm.getResourceTrackerService());
+ nm3.registerNode();
+
+ MockNM nm4 =
+ new MockNM("127.0.0.4:2345", 8000, rm.getResourceTrackerService());
+ nm4.registerNode();
+
+ MockNM nm5 =
+ new MockNM("127.0.0.5:2345", 8000, rm.getResourceTrackerService());
+ nm5.registerNode();
+
+ // Specify a strict locality on nm2
+ List<ResourceRequest> reqs = new ArrayList<>();
+ ResourceRequest nodeReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), nm2.getNodeId().getHost(),
+ Resource.newInstance(200, 1), 1, true);
+ ResourceRequest rackReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), "/default-rack",
+ Resource.newInstance(200, 1), 1, false);
+ ResourceRequest anyReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), ResourceRequest.ANY,
+ Resource.newInstance(200, 1), 1, false);
+ reqs.add(anyReq);
+ reqs.add(rackReq);
+ reqs.add(nodeReq);
+ RMApp app = rm.submitApp(reqs);
+
+ MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2);
+ ContainerId amContainerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ RMContainer rmContainer = scheduler.getRMContainer(amContainerId);
+ NodeId nodeWhereAMRan = rmContainer.getAllocatedNode();
+ Assert.assertEquals(nm2.getNodeId(), nodeWhereAMRan);
+
+ // Set the exist status to INVALID so that we can verify that the system
+ // automatically blacklisting the node
+ makeAMContainerExit(rm, amContainerId, nm2, ContainerExitStatus.INVALID);
+
+ // restart the am
+ RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm);
+ System.out.println("New AppAttempt launched " + attempt.getAppAttemptId());
+
+ nm2.nodeHeartbeat(true);
+ dispatcher.await();
+
+ // Now the AM container should be allocated
+ MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
+
+ MockAM am2 = rm.sendAMLaunched(attempt.getAppAttemptId());
+ rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED);
+ amContainerId =
+ ContainerId.newContainerId(am2.getApplicationAttemptId(), 1);
+ rmContainer = scheduler.getRMContainer(amContainerId);
+ nodeWhereAMRan = rmContainer.getAllocatedNode();
+
+ // The second AM should be on the same node because the strict locality
+ // made the eligible nodes only 1, so the blacklisting threshold kicked in
+ System.out.println("AM ran on " + nodeWhereAMRan);
+ Assert.assertEquals(nm2.getNodeId(), nodeWhereAMRan);
+
+ am2.registerAppAttempt();
+ rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
+ }
+
+ @Test(timeout = 100000)
+ public void testNodeBlacklistingOnAMFailureRelaxedNodeLocality()
+ throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED,
+ true);
+
+ DrainDispatcher dispatcher = new DrainDispatcher();
+ MockRM rm = startRM(conf, dispatcher);
+ CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
+
+ // Register 5 nodes, so that we can blacklist atleast one if AM container
+ // is failed. As per calculation it will be like, 5nodes * 0.2 (default)=1.
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService());
+ nm1.registerNode();
+
+ MockNM nm2 =
+ new MockNM("127.0.0.2:2345", 8000, rm.getResourceTrackerService());
+ nm2.registerNode();
+
+ MockNM nm3 =
+ new MockNM("127.0.0.3:2345", 8000, rm.getResourceTrackerService());
+ nm3.registerNode();
+
+ MockNM nm4 =
+ new MockNM("127.0.0.4:2345", 8000, rm.getResourceTrackerService());
+ nm4.registerNode();
+
+ MockNM nm5 =
+ new MockNM("127.0.0.5:2345", 8000, rm.getResourceTrackerService());
+ nm5.registerNode();
+
+ // Specify a relaxed locality on nm2
+ List<ResourceRequest> reqs = new ArrayList<>();
+ ResourceRequest nodeReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), nm2.getNodeId().getHost(),
+ Resource.newInstance(200, 1), 1, true);
+ ResourceRequest rackReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), "/default-rack",
+ Resource.newInstance(200, 1), 1, true);
+ ResourceRequest anyReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), ResourceRequest.ANY,
+ Resource.newInstance(200, 1), 1, true);
+ reqs.add(anyReq);
+ reqs.add(rackReq);
+ reqs.add(nodeReq);
+ RMApp app = rm.submitApp(reqs);
+
+ MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2);
+ ContainerId amContainerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ RMContainer rmContainer = scheduler.getRMContainer(amContainerId);
+ NodeId nodeWhereAMRan = rmContainer.getAllocatedNode();
+ Assert.assertEquals(nm2.getNodeId(), nodeWhereAMRan);
+
+ // Set the exist status to INVALID so that we can verify that the system
+ // automatically blacklisting the node
+ makeAMContainerExit(rm, amContainerId, nm2, ContainerExitStatus.INVALID);
+
+ // restart the am
+ RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm);
+ System.out.println("New AppAttempt launched " + attempt.getAppAttemptId());
+
+ nm2.nodeHeartbeat(true);
+ nm1.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ nm4.nodeHeartbeat(true);
+ nm5.nodeHeartbeat(true);
+ dispatcher.await();
+
+ // Now the AM container should be allocated
+ MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
+
+ MockAM am2 = rm.sendAMLaunched(attempt.getAppAttemptId());
+ rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED);
+ amContainerId =
+ ContainerId.newContainerId(am2.getApplicationAttemptId(), 1);
+ rmContainer = scheduler.getRMContainer(amContainerId);
+ nodeWhereAMRan = rmContainer.getAllocatedNode();
+
+ // The second AM should be on a different node because the relaxed locality
+ // made the app schedulable on other nodes and nm2 is blacklisted
+ System.out.println("AM ran on " + nodeWhereAMRan);
+ Assert.assertNotEquals(nm2.getNodeId(), nodeWhereAMRan);
+
+ am2.registerAppAttempt();
+ rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
+ }
+
+ @Test(timeout = 100000)
public void testNoBlacklistingForNonSystemErrors() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HADOOP-14216. Improve
Configuration XML Parsing Performance (jeagles)
Posted by ae...@apache.org.
HADOOP-14216. Improve Configuration XML Parsing Performance (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/523f467d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/523f467d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/523f467d
Branch: refs/heads/HDFS-7240
Commit: 523f467d939d80e2bc162e1f47be497109783061
Parents: 84d787b
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Wed Mar 29 10:12:02 2017 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Wed Mar 29 10:12:02 2017 -0500
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/pom.xml | 10 +
.../org/apache/hadoop/conf/Configuration.java | 317 +++++++++++--------
.../apache/hadoop/conf/TestConfiguration.java | 62 +++-
hadoop-project/pom.xml | 10 +
4 files changed, 266 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/523f467d/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 961c902..ee82df0 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -313,6 +313,16 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.woodstox</groupId>
+ <artifactId>stax2-api</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml</groupId>
+ <artifactId>aalto-xml</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/523f467d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index da4f6ac..36845e3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.conf;
+import com.fasterxml.aalto.stax.InputFactoryImpl;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.google.common.annotations.VisibleForTesting;
@@ -65,9 +66,11 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
-import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
@@ -93,14 +96,10 @@ import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
-import org.w3c.dom.Attr;
-import org.w3c.dom.DOMException;
+import org.codehaus.stax2.XMLInputFactory2;
+import org.codehaus.stax2.XMLStreamReader2;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.w3c.dom.Text;
-import org.xml.sax.SAXException;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -280,7 +279,13 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* the key most recently
*/
private Map<String, String[]> updatingResource;
-
+
+ /**
+ * Specify exact input factory to avoid time finding correct one.
+ * Factory is reusable across un-synchronized threads once initialized
+ */
+ private static final XMLInputFactory2 factory = new InputFactoryImpl();
+
/**
* Class to keep the information about the keys which replace the deprecated
* ones.
@@ -689,7 +694,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
addDefaultResource("hadoop-site.xml");
}
}
-
+
private Properties properties;
private Properties overlay;
private ClassLoader classLoader;
@@ -2613,8 +2618,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return configMap;
}
- private Document parse(DocumentBuilder builder, URL url)
- throws IOException, SAXException {
+ private XMLStreamReader parse(URL url)
+ throws IOException, XMLStreamException {
if (!quietmode) {
if (LOG.isDebugEnabled()) {
LOG.debug("parsing URL " + url);
@@ -2630,23 +2635,18 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
// with other users.
connection.setUseCaches(false);
}
- return parse(builder, connection.getInputStream(), url.toString());
+ return parse(connection.getInputStream(), url.toString());
}
- private Document parse(DocumentBuilder builder, InputStream is,
- String systemId) throws IOException, SAXException {
+ private XMLStreamReader parse(InputStream is,
+ String systemId) throws IOException, XMLStreamException {
if (!quietmode) {
LOG.debug("parsing input stream " + is);
}
if (is == null) {
return null;
}
- try {
- return (systemId == null) ? builder.parse(is) : builder.parse(is,
- systemId);
- } finally {
- is.close();
- }
+ return factory.createXMLStreamReader(systemId, is);
}
private void loadResources(Properties properties,
@@ -2666,37 +2666,20 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
}
- private Resource loadResource(Properties properties, Resource wrapper, boolean quiet) {
+ private Resource loadResource(Properties properties,
+ Resource wrapper, boolean quiet) {
String name = UNKNOWN_RESOURCE;
try {
Object resource = wrapper.getResource();
name = wrapper.getName();
-
- DocumentBuilderFactory docBuilderFactory
- = DocumentBuilderFactory.newInstance();
- //ignore all comments inside the xml file
- docBuilderFactory.setIgnoringComments(true);
-
- //allow includes in the xml file
- docBuilderFactory.setNamespaceAware(true);
- try {
- docBuilderFactory.setXIncludeAware(true);
- } catch (UnsupportedOperationException e) {
- LOG.error("Failed to set setXIncludeAware(true) for parser "
- + docBuilderFactory
- + ":" + e,
- e);
- }
- DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
- Document doc = null;
- Element root = null;
+ XMLStreamReader2 reader = null;
boolean returnCachedProperties = false;
-
+
if (resource instanceof URL) { // an URL resource
- doc = parse(builder, (URL)resource);
+ reader = (XMLStreamReader2)parse((URL)resource);
} else if (resource instanceof String) { // a CLASSPATH resource
URL url = getResource((String)resource);
- doc = parse(builder, url);
+ reader = (XMLStreamReader2)parse(url);
} else if (resource instanceof Path) { // a file resource
// Can't use FileSystem API or we get an infinite loop
// since FileSystem uses Configuration API. Use java.io.File instead.
@@ -2706,104 +2689,188 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
if (!quiet) {
LOG.debug("parsing File " + file);
}
- doc = parse(builder, new BufferedInputStream(
+ reader = (XMLStreamReader2)parse(new BufferedInputStream(
new FileInputStream(file)), ((Path)resource).toString());
}
} else if (resource instanceof InputStream) {
- doc = parse(builder, (InputStream) resource, null);
+ reader = (XMLStreamReader2)parse((InputStream)resource, null);
returnCachedProperties = true;
} else if (resource instanceof Properties) {
overlay(properties, (Properties)resource);
- } else if (resource instanceof Element) {
- root = (Element)resource;
}
- if (root == null) {
- if (doc == null) {
- if (quiet) {
- return null;
- }
- throw new RuntimeException(resource + " not found");
+ if (reader == null) {
+ if (quiet) {
+ return null;
}
- root = doc.getDocumentElement();
+ throw new RuntimeException(resource + " not found");
}
Properties toAddTo = properties;
if(returnCachedProperties) {
toAddTo = new Properties();
}
- if (!"configuration".equals(root.getTagName()))
- LOG.fatal("bad conf file: top-level element not <configuration>");
- NodeList props = root.getChildNodes();
DeprecationContext deprecations = deprecationContext.get();
- for (int i = 0; i < props.getLength(); i++) {
- Node propNode = props.item(i);
- if (!(propNode instanceof Element))
- continue;
- Element prop = (Element)propNode;
- if ("configuration".equals(prop.getTagName())) {
- loadResource(toAddTo, new Resource(prop, name), quiet);
- continue;
- }
- if (!"property".equals(prop.getTagName()))
- LOG.warn("bad conf file: element not <property>");
-
- String attr = null;
- String value = null;
- boolean finalParameter = false;
- LinkedList<String> source = new LinkedList<String>();
-
- Attr propAttr = prop.getAttributeNode("name");
- if (propAttr != null)
- attr = StringInterner.weakIntern(propAttr.getValue());
- propAttr = prop.getAttributeNode("value");
- if (propAttr != null)
- value = StringInterner.weakIntern(propAttr.getValue());
- propAttr = prop.getAttributeNode("final");
- if (propAttr != null)
- finalParameter = "true".equals(propAttr.getValue());
- propAttr = prop.getAttributeNode("source");
- if (propAttr != null)
- source.add(StringInterner.weakIntern(propAttr.getValue()));
-
- NodeList fields = prop.getChildNodes();
- for (int j = 0; j < fields.getLength(); j++) {
- Node fieldNode = fields.item(j);
- if (!(fieldNode instanceof Element))
- continue;
- Element field = (Element)fieldNode;
- if ("name".equals(field.getTagName()) && field.hasChildNodes())
- attr = StringInterner.weakIntern(
- ((Text)field.getFirstChild()).getData().trim());
- if ("value".equals(field.getTagName()) && field.hasChildNodes())
- value = StringInterner.weakIntern(
- ((Text)field.getFirstChild()).getData());
- if ("final".equals(field.getTagName()) && field.hasChildNodes())
- finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
- if ("source".equals(field.getTagName()) && field.hasChildNodes())
- source.add(StringInterner.weakIntern(
- ((Text)field.getFirstChild()).getData()));
- }
- source.add(name);
-
- // Ignore this parameter if it has already been marked as 'final'
- if (attr != null) {
- if (deprecations.getDeprecatedKeyMap().containsKey(attr)) {
- DeprecatedKeyInfo keyInfo =
- deprecations.getDeprecatedKeyMap().get(attr);
- keyInfo.clearAccessed();
- for (String key:keyInfo.newKeys) {
- // update new keys with deprecated key's value
- loadProperty(toAddTo, name, key, value, finalParameter,
- source.toArray(new String[source.size()]));
+
+ StringBuilder token = new StringBuilder();
+ String confName = null;
+ String confValue = null;
+ boolean confFinal = false;
+ boolean fallbackAllowed = false;
+ boolean fallbackEntered = false;
+ boolean parseToken = false;
+ LinkedList<String> confSource = new LinkedList<String>();
+
+ while (reader.hasNext()) {
+ switch (reader.next()) {
+ case XMLStreamConstants.START_ELEMENT:
+ switch (reader.getLocalName()) {
+ case "property":
+ confName = null;
+ confValue = null;
+ confFinal = false;
+ confSource.clear();
+
+ // First test for short format configuration
+ int attrCount = reader.getAttributeCount();
+ for (int i = 0; i < attrCount; i++) {
+ String propertyAttr = reader.getAttributeLocalName(i);
+ if ("name".equals(propertyAttr)) {
+ confName = StringInterner.weakIntern(
+ reader.getAttributeValue(i));
+ } else if ("value".equals(propertyAttr)) {
+ confValue = StringInterner.weakIntern(
+ reader.getAttributeValue(i));
+ } else if ("final".equals(propertyAttr)) {
+ confFinal = "true".equals(reader.getAttributeValue(i));
+ } else if ("source".equals(propertyAttr)) {
+ confSource.add(StringInterner.weakIntern(
+ reader.getAttributeValue(i)));
+ }
+ }
+ break;
+ case "name":
+ case "value":
+ case "final":
+ case "source":
+ parseToken = true;
+ token.setLength(0);
+ break;
+ case "include":
+ if (!"xi".equals(reader.getPrefix())) {
+ break;
+ }
+ // Determine href for xi:include
+ String confInclude = null;
+ attrCount = reader.getAttributeCount();
+ for (int i = 0; i < attrCount; i++) {
+ String attrName = reader.getAttributeLocalName(i);
+ if ("href".equals(attrName)) {
+ confInclude = reader.getAttributeValue(i);
+ }
+ }
+ if (confInclude == null) {
+ break;
+ }
+ // Determine if the included resource is a classpath resource
+ // otherwise fallback to a file resource
+ // xi:include are treated as inline and retain current source
+ URL include = getResource(confInclude);
+ if (include != null) {
+ Resource classpathResource = new Resource(include, name);
+ loadResource(properties, classpathResource, quiet);
+ } else {
+ File href = new File(confInclude);
+ if (!href.isAbsolute()) {
+ // Included resources are relative to the current resource
+ File baseFile = new File(name).getParentFile();
+ href = new File(baseFile, href.getPath());
+ }
+ if (!href.exists()) {
+ // Resource errors are non-fatal iff there is 1 xi:fallback
+ fallbackAllowed = true;
+ break;
+ }
+ Resource uriResource = new Resource(href.toURI().toURL(), name);
+ loadResource(properties, uriResource, quiet);
}
+ break;
+ case "fallback":
+ if (!"xi".equals(reader.getPrefix())) {
+ break;
+ }
+ fallbackEntered = true;
+ break;
+ case "configuration":
+ break;
+ default:
+ break;
}
- else {
- loadProperty(toAddTo, name, attr, value, finalParameter,
- source.toArray(new String[source.size()]));
+ break;
+
+ case XMLStreamConstants.CHARACTERS:
+ if (parseToken) {
+ char[] text = reader.getTextCharacters();
+ token.append(text, reader.getTextStart(), reader.getTextLength());
}
+ break;
+
+ case XMLStreamConstants.END_ELEMENT:
+ switch (reader.getLocalName()) {
+ case "name":
+ if (token.length() > 0) {
+ confName = StringInterner.weakIntern(token.toString().trim());
+ }
+ break;
+ case "value":
+ if (token.length() > 0) {
+ confValue = StringInterner.weakIntern(token.toString());
+ }
+ break;
+ case "final":
+ confFinal = "true".equals(token.toString());
+ break;
+ case "source":
+ confSource.add(StringInterner.weakIntern(token.toString()));
+ break;
+ case "include":
+ if (!"xi".equals(reader.getPrefix())) {
+ break;
+ }
+ if (fallbackAllowed && !fallbackEntered) {
+ throw new IOException("Fetch fail on include with no "
+ + "fallback while loading '" + name + "'");
+ }
+ fallbackAllowed = false;
+ fallbackEntered = false;
+ break;
+ case "property":
+ if (confName == null || (!fallbackAllowed && fallbackEntered)) {
+ break;
+ }
+ confSource.add(name);
+ DeprecatedKeyInfo keyInfo =
+ deprecations.getDeprecatedKeyMap().get(confName);
+ if (keyInfo != null) {
+ keyInfo.clearAccessed();
+ for (String key : keyInfo.newKeys) {
+ // update new keys with deprecated key's value
+ loadProperty(toAddTo, name, key, confValue, confFinal,
+ confSource.toArray(new String[confSource.size()]));
+ }
+ } else {
+ loadProperty(toAddTo, name, confName, confValue, confFinal,
+ confSource.toArray(new String[confSource.size()]));
+ }
+ break;
+ default:
+ break;
+ }
+ default:
+ break;
}
}
-
+ reader.close();
+
if (returnCachedProperties) {
overlay(properties, toAddTo);
return new Resource(toAddTo, name);
@@ -2812,15 +2879,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
} catch (IOException e) {
LOG.fatal("error parsing conf " + name, e);
throw new RuntimeException(e);
- } catch (DOMException e) {
+ } catch (XMLStreamException e) {
LOG.fatal("error parsing conf " + name, e);
throw new RuntimeException(e);
- } catch (SAXException e) {
- LOG.fatal("error parsing conf " + name, e);
- throw new RuntimeException(e);
- } catch (ParserConfigurationException e) {
- LOG.fatal("error parsing conf " + name , e);
- throw new RuntimeException(e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/523f467d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 1f97786..6f4c26c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -99,10 +99,22 @@ public class TestConfiguration extends TestCase {
out.close();
}
- private void addInclude(String filename) throws IOException{
- out.write("<xi:include href=\"" + filename + "\" xmlns:xi=\"http://www.w3.org/2001/XInclude\" />\n ");
+ private void startInclude(String filename) throws IOException {
+ out.write("<xi:include href=\"" + filename + "\" xmlns:xi=\"http://www.w3.org/2001/XInclude\" >\n ");
}
-
+
+ private void endInclude() throws IOException{
+ out.write("</xi:include>\n ");
+ }
+
+ private void startFallback() throws IOException {
+ out.write("<xi:fallback>\n ");
+ }
+
+ private void endFallback() throws IOException {
+ out.write("</xi:fallback>\n ");
+ }
+
public void testInputStreamResource() throws Exception {
StringWriter writer = new StringWriter();
out = new BufferedWriter(writer);
@@ -507,7 +519,8 @@ public class TestConfiguration extends TestCase {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
- addInclude(CONFIG2);
+ startInclude(CONFIG2);
+ endInclude();
appendProperty("e","f");
appendProperty("g","h");
endConfig();
@@ -522,6 +535,44 @@ public class TestConfiguration extends TestCase {
tearDown();
}
+ public void testIncludesWithFallback() throws Exception {
+ tearDown();
+ out=new BufferedWriter(new FileWriter(CONFIG2));
+ startConfig();
+ appendProperty("a","b");
+ appendProperty("c","d");
+ endConfig();
+
+ out=new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ startInclude(CONFIG2);
+ startFallback();
+ appendProperty("a", "b.fallback");
+ appendProperty("c", "d.fallback", true);
+ endFallback();
+ endInclude();
+ appendProperty("e","f");
+ appendProperty("g","h");
+ startInclude("MissingConfig.xml");
+ startFallback();
+ appendProperty("i", "j.fallback");
+ appendProperty("k", "l.fallback", true);
+ endFallback();
+ endInclude();
+ endConfig();
+
+ // verify that the includes file contains all properties
+ Path fileResource = new Path(CONFIG);
+ conf.addResource(fileResource);
+ assertEquals("b", conf.get("a"));
+ assertEquals("d", conf.get("c"));
+ assertEquals("f", conf.get("e"));
+ assertEquals("h", conf.get("g"));
+ assertEquals("j.fallback", conf.get("i"));
+ assertEquals("l.fallback", conf.get("k"));
+ tearDown();
+ }
+
public void testRelativeIncludes() throws Exception {
tearDown();
String relConfig = new File("./tmp/test-config.xml").getAbsolutePath();
@@ -536,7 +587,8 @@ public class TestConfiguration extends TestCase {
out = new BufferedWriter(new FileWriter(relConfig));
startConfig();
// Add the relative path instead of the absolute one.
- addInclude(new File(relConfig2).getName());
+ startInclude(new File(relConfig2).getName());
+ endInclude();
appendProperty("c", "d");
endConfig();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/523f467d/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 68e8b5c..e55308f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -870,6 +870,16 @@
<version>3.1.1</version>
</dependency>
<dependency>
+ <groupId>org.codehaus.woodstox</groupId>
+ <artifactId>stax2-api</artifactId>
+ <version>3.1.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml</groupId>
+ <artifactId>aalto-xml</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+ <dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>${jackson.version}</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org