You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by br...@apache.org on 2018/11/21 19:06:15 UTC
[01/50] [abbrv] hadoop git commit: HDDS-801. Quasi close the
container when close is not executed via Ratis. Contributed by Nanda kumar.
[Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/HDFS-13891 175fd4cc4 -> 864cad231 (forced update)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index 631283c..1ff6655 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
@@ -960,7 +959,7 @@ public class TestNodeManager {
TestUtils.getRandomPipelineReports());
eq.fireEvent(DATANODE_COMMAND,
new CommandForDatanode<>(datanodeDetails.getUuid(),
- new CloseContainerCommand(1L, ReplicationType.STAND_ALONE,
+ new CloseContainerCommand(1L,
PipelineID.randomId())));
eq.processAll(1000L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index 0f35607..16c9f22 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -21,7 +21,6 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CloseContainerCommandProto;
@@ -66,6 +65,7 @@ import org.apache.hadoop.ozone.container.common.states.endpoint
import org.apache.hadoop.ozone.container.common.states.endpoint
.VersionEndpointTask;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
import org.apache.hadoop.test.GenericTestUtils;
@@ -75,6 +75,7 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.mockito.Mockito;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import static org.mockito.Mockito.mock;
@@ -309,8 +310,10 @@ public class TestEndPoint {
OzoneContainer ozoneContainer = mock(OzoneContainer.class);
when(ozoneContainer.getNodeReport()).thenReturn(TestUtils
.createNodeReport(getStorageReports(UUID.randomUUID())));
- when(ozoneContainer.getContainerReport()).thenReturn(
+ ContainerController controller = Mockito.mock(ContainerController.class);
+ when(controller.getContainerReport()).thenReturn(
TestUtils.getRandomContainerReports(10));
+ when(ozoneContainer.getController()).thenReturn(controller);
when(ozoneContainer.getPipelineReport()).thenReturn(
TestUtils.getRandomPipelineReports());
RegisterEndpointTask endpointTask =
@@ -433,7 +436,6 @@ public class TestEndPoint {
.setCloseContainerCommandProto(
CloseContainerCommandProto.newBuilder().setCmdId(1)
.setContainerID(1)
- .setReplicationType(ReplicationType.RATIS)
.setPipelineID(PipelineID.randomId().getProtobuf())
.build())
.setCommandType(Type.closeContainerCommand)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot b/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot
index fb58520..0162f9e 100644
--- a/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot
@@ -42,6 +42,6 @@ Check webui static resources
Should contain ${result} 200
Start freon testing
- ${result} = Execute ozone freon randomkeys --numOfVolumes 5 --numOfBuckets 5 --numOfKeys 5 --numOfThreads 10
+ ${result} = Execute ozone freon randomkeys --numOfVolumes 5 --numOfBuckets 5 --numOfKeys 5 --numOfThreads 1
Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125
Should Not Contain ${result} ERROR
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index 3083660..6427dae 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -176,14 +176,14 @@ public class TestMiniOzoneCluster {
TestUtils.randomDatanodeDetails(), ozoneConf)
) {
HashSet<Integer> ports = new HashSet<Integer>();
- assertTrue(ports.add(sm1.getContainer().getContainerServerPort()));
- assertTrue(ports.add(sm2.getContainer().getContainerServerPort()));
- assertTrue(ports.add(sm3.getContainer().getContainerServerPort()));
+ assertTrue(ports.add(sm1.getContainer().getReadChannel().getIPCPort()));
+ assertTrue(ports.add(sm2.getContainer().getReadChannel().getIPCPort()));
+ assertTrue(ports.add(sm3.getContainer().getReadChannel().getIPCPort()));
// Assert that ratis is also on a different port.
- assertTrue(ports.add(sm1.getContainer().getRatisContainerServerPort()));
- assertTrue(ports.add(sm2.getContainer().getRatisContainerServerPort()));
- assertTrue(ports.add(sm3.getContainer().getRatisContainerServerPort()));
+ assertTrue(ports.add(sm1.getContainer().getWriteChannel().getIPCPort()));
+ assertTrue(ports.add(sm2.getContainer().getWriteChannel().getIPCPort()));
+ assertTrue(ports.add(sm3.getContainer().getWriteChannel().getIPCPort()));
}
@@ -199,9 +199,9 @@ public class TestMiniOzoneCluster {
TestUtils.randomDatanodeDetails(), ozoneConf)
) {
HashSet<Integer> ports = new HashSet<Integer>();
- assertTrue(ports.add(sm1.getContainer().getContainerServerPort()));
- assertFalse(ports.add(sm2.getContainer().getContainerServerPort()));
- assertFalse(ports.add(sm3.getContainer().getContainerServerPort()));
+ assertTrue(ports.add(sm1.getContainer().getReadChannel().getIPCPort()));
+ assertFalse(ports.add(sm2.getContainer().getReadChannel().getIPCPort()));
+ assertFalse(ports.add(sm3.getContainer().getReadChannel().getIPCPort()));
assertEquals(ports.iterator().next().intValue(),
conf.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index 405ce8e..abd60a1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -134,7 +134,7 @@ public class TestCloseContainerHandlingByClient {
.setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
.build();
- waitForContainerClose(keyName, key, HddsProtos.ReplicationType.RATIS);
+ waitForContainerClose(keyName, key);
key.write(data);
key.flush();
key.close();
@@ -162,11 +162,12 @@ public class TestCloseContainerHandlingByClient {
Assert.assertTrue(key.getOutputStream() instanceof ChunkGroupOutputStream);
//get the name of a valid container
OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
- .setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
+ .setBucketName(bucketName)
+ .setType(HddsProtos.ReplicationType.RATIS)
.setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
.build();
- waitForContainerClose(keyName, key, HddsProtos.ReplicationType.RATIS);
+ waitForContainerClose(keyName, key);
key.close();
// read the key from OM again and match the length.The length will still
// be the equal to the original data size.
@@ -199,7 +200,7 @@ public class TestCloseContainerHandlingByClient {
.setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
.build();
- waitForContainerClose(keyName, key, HddsProtos.ReplicationType.RATIS);
+ waitForContainerClose(keyName, key);
// write 1 more block worth of data. It will fail and new block will be
// allocated
key.write(ContainerTestHelper.getFixedLengthString(keyString, blockSize)
@@ -249,7 +250,7 @@ public class TestCloseContainerHandlingByClient {
.setFactor(HddsProtos.ReplicationFactor.THREE).setKeyName(keyName)
.build();
- waitForContainerClose(keyName, key, HddsProtos.ReplicationType.RATIS);
+ waitForContainerClose(keyName, key);
key.close();
// read the key from OM again and match the length.The length will still
@@ -291,7 +292,7 @@ public class TestCloseContainerHandlingByClient {
.setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
.build();
- waitForContainerClose(keyName, key, HddsProtos.ReplicationType.RATIS);
+ waitForContainerClose(keyName, key);
// write 3 more chunks worth of data. It will fail and new block will be
// allocated. This write completes 4 blocks worth of data written to key
data = Arrays.copyOfRange(writtenData, 3 * blockSize + chunkSize, keyLen);
@@ -321,7 +322,7 @@ public class TestCloseContainerHandlingByClient {
}
private void waitForContainerClose(String keyName,
- OzoneOutputStream outputStream, HddsProtos.ReplicationType type)
+ OzoneOutputStream outputStream)
throws Exception {
ChunkGroupOutputStream groupOutputStream =
(ChunkGroupOutputStream) outputStream.getOutputStream();
@@ -332,11 +333,10 @@ public class TestCloseContainerHandlingByClient {
containerIdList.add(info.getContainerID());
}
Assert.assertTrue(!containerIdList.isEmpty());
- waitForContainerClose(type, containerIdList.toArray(new Long[0]));
+ waitForContainerClose(containerIdList.toArray(new Long[0]));
}
- private void waitForContainerClose(HddsProtos.ReplicationType type,
- Long... containerIdList)
+ private void waitForContainerClose(Long... containerIdList)
throws ContainerNotFoundException, PipelineNotFoundException,
TimeoutException, InterruptedException {
List<Pipeline> pipelineList = new ArrayList<>();
@@ -358,7 +358,7 @@ public class TestCloseContainerHandlingByClient {
// send the order to close the container
cluster.getStorageContainerManager().getScmNodeManager()
.addDatanodeCommand(details.getUuid(),
- new CloseContainerCommand(containerID, type, pipeline.getId()));
+ new CloseContainerCommand(containerID, pipeline.getId()));
}
}
int index = 0;
@@ -413,7 +413,7 @@ public class TestCloseContainerHandlingByClient {
.getPipeline(container.getPipelineID());
List<DatanodeDetails> datanodes = pipeline.getNodes();
Assert.assertEquals(1, datanodes.size());
- waitForContainerClose(keyName, key, HddsProtos.ReplicationType.RATIS);
+ waitForContainerClose(keyName, key);
dataString =
ContainerTestHelper.getFixedLengthString(keyString, (1 * blockSize));
data = dataString.getBytes(UTF_8);
@@ -459,7 +459,7 @@ public class TestCloseContainerHandlingByClient {
.build();
Assert.assertTrue(key.getOutputStream() instanceof ChunkGroupOutputStream);
- waitForContainerClose(keyName, key, HddsProtos.ReplicationType.RATIS);
+ waitForContainerClose(keyName, key);
// Again Write the Data. This will throw an exception which will be handled
// and new blocks will be allocated
key.write(data);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
index 30e3536..695b3f1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
@@ -22,9 +22,11 @@ import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.client.ObjectStore;
@@ -32,7 +34,6 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
@@ -63,8 +64,9 @@ public class TestCloseContainerByPipeline {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, "1");
cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3).build();
+ .setNumDatanodes(9).build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
client = OzoneClientFactory.getClient(conf);
@@ -86,7 +88,7 @@ public class TestCloseContainerByPipeline {
@Test
public void testIfCloseContainerCommandHandlerIsInvoked() throws Exception {
OzoneOutputStream key = objectStore.getVolume("test").getBucket("test")
- .createKey("testCloseContainer", 1024, ReplicationType.STAND_ALONE,
+ .createKey("standalone", 1024, ReplicationType.RATIS,
ReplicationFactor.ONE);
key.write("standalone".getBytes());
key.close();
@@ -94,10 +96,9 @@ public class TestCloseContainerByPipeline {
//get the name of a valid container
OmKeyArgs keyArgs =
new OmKeyArgs.Builder().setVolumeName("test").setBucketName("test")
- .setType(HddsProtos.ReplicationType.STAND_ALONE)
+ .setType(HddsProtos.ReplicationType.RATIS)
.setFactor(HddsProtos.ReplicationFactor.ONE).setDataSize(1024)
- .setKeyName("testCloseContainer").build();
-
+ .setKeyName("standalone").build();
OmKeyLocationInfo omKeyLocationInfo =
cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions()
.get(0).getBlocksLatestVersionOnly().get(0);
@@ -127,8 +128,7 @@ public class TestCloseContainerByPipeline {
//send the order to close the container
cluster.getStorageContainerManager().getScmNodeManager()
.addDatanodeCommand(datanodeDetails.getUuid(),
- new CloseContainerCommand(containerID,
- HddsProtos.ReplicationType.STAND_ALONE, pipeline.getId()));
+ new CloseContainerCommand(containerID, pipeline.getId()));
GenericTestUtils
.waitFor(() -> isContainerClosed(cluster, containerID, datanodeDetails),
500, 5 * 1000);
@@ -142,7 +142,7 @@ public class TestCloseContainerByPipeline {
throws IOException, TimeoutException, InterruptedException {
OzoneOutputStream key = objectStore.getVolume("test").getBucket("test")
- .createKey("standalone", 1024, ReplicationType.STAND_ALONE,
+ .createKey("standalone", 1024, ReplicationType.RATIS,
ReplicationFactor.ONE);
key.write("standalone".getBytes());
key.close();
@@ -150,7 +150,7 @@ public class TestCloseContainerByPipeline {
//get the name of a valid container
OmKeyArgs keyArgs =
new OmKeyArgs.Builder().setVolumeName("test").setBucketName("test")
- .setType(HddsProtos.ReplicationType.STAND_ALONE)
+ .setType(HddsProtos.ReplicationType.RATIS)
.setFactor(HddsProtos.ReplicationFactor.ONE).setDataSize(1024)
.setKeyName("standalone").build();
@@ -170,30 +170,20 @@ public class TestCloseContainerByPipeline {
Assert
.assertFalse(isContainerClosed(cluster, containerID, datanodeDetails));
- GenericTestUtils.LogCapturer logCapturer =
- GenericTestUtils.LogCapturer.captureLogs(OzoneContainer.LOG);
- //send the order to close the container
+ // Send the order to close the container, give random pipeline id so that
+ // the container will not be closed via RATIS
cluster.getStorageContainerManager().getScmNodeManager()
.addDatanodeCommand(datanodeDetails.getUuid(),
- new CloseContainerCommand(containerID,
- HddsProtos.ReplicationType.STAND_ALONE, pipeline.getId()));
-
- // The log will appear after the state changed to closed in standalone,
- // wait for the log to ensure the operation has been done.
- GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains(
- "submitting CloseContainer request over STAND_ALONE server for"
- + " container " + containerID), 500, 5 * 1000);
+ new CloseContainerCommand(containerID, PipelineID.randomId()));
//double check if it's really closed (waitFor also throws an exception)
- Assert.assertTrue(isContainerClosed(cluster, containerID, datanodeDetails));
- Assert.assertTrue(logCapturer.getOutput().contains(
- "submitting CloseContainer request over STAND_ALONE server for"
- + " container " + containerID));
- // Make sure it was really closed via StandAlone not Ratis server
- Assert.assertFalse((logCapturer.getOutput().contains(
- "submitting CloseContainer request over RATIS server for container "
- + containerID)));
- logCapturer.stopCapturing();
+ // TODO: change the below line after implementing QUASI_CLOSED to CLOSED
+ // logic. The container will be QUASI closed as of now
+ GenericTestUtils
+ .waitFor(() -> isContainerQuasiClosed(
+ cluster, containerID, datanodeDetails), 500, 5 * 1000);
+ Assert.assertTrue(
+ isContainerQuasiClosed(cluster, containerID, datanodeDetails));
}
@Test
@@ -224,18 +214,14 @@ public class TestCloseContainerByPipeline {
List<DatanodeDetails> datanodes = pipeline.getNodes();
Assert.assertEquals(3, datanodes.size());
- GenericTestUtils.LogCapturer logCapturer =
- GenericTestUtils.LogCapturer.captureLogs(OzoneContainer.LOG);
-
for (DatanodeDetails details : datanodes) {
Assert.assertFalse(isContainerClosed(cluster, containerID, details));
//send the order to close the container
cluster.getStorageContainerManager().getScmNodeManager()
.addDatanodeCommand(details.getUuid(),
- new CloseContainerCommand(containerID,
- HddsProtos.ReplicationType.RATIS, pipeline.getId()));
+ new CloseContainerCommand(containerID, pipeline.getId()));
}
-
+ // Make sure that it is CLOSED
for (DatanodeDetails datanodeDetails : datanodes) {
GenericTestUtils.waitFor(
() -> isContainerClosed(cluster, containerID, datanodeDetails), 500,
@@ -244,14 +230,6 @@ public class TestCloseContainerByPipeline {
Assert.assertTrue(isContainerClosed(cluster,
containerID, datanodeDetails));
}
- // Make sure it was really closed via Ratis not STAND_ALONE server
- Assert.assertFalse(logCapturer.getOutput().contains(
- "submitting CloseContainer request over STAND_ALONE "
- + "server for container " + containerID));
- Assert.assertTrue((logCapturer.getOutput().contains(
- "submitting CloseContainer request over RATIS server for container "
- + containerID)));
- logCapturer.stopCapturing();
}
private Boolean isContainerClosed(MiniOzoneCluster cluster, long containerID,
@@ -267,4 +245,18 @@ public class TestCloseContainerByPipeline {
}
return false;
}
+
+ private Boolean isContainerQuasiClosed(MiniOzoneCluster miniCluster,
+ long containerID, DatanodeDetails datanode) {
+ ContainerData containerData;
+ for (HddsDatanodeService datanodeService : miniCluster.getHddsDatanodes()) {
+ if (datanode.equals(datanodeService.getDatanodeDetails())) {
+ containerData =
+ datanodeService.getDatanodeStateMachine().getContainer()
+ .getContainerSet().getContainer(containerID).getContainerData();
+ return containerData.isQuasiClosed();
+ }
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
index 9cf51d1..588a301 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
@@ -95,8 +95,7 @@ public class TestCloseContainerHandler {
//send the order to close the container
cluster.getStorageContainerManager().getScmNodeManager()
.addDatanodeCommand(datanodeDetails.getUuid(),
- new CloseContainerCommand(containerId.getId(),
- HddsProtos.ReplicationType.STAND_ALONE, pipeline.getId()));
+ new CloseContainerCommand(containerId.getId(), pipeline.getId()));
GenericTestUtils.waitFor(() ->
isContainerClosed(cluster, containerId.getId()),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
index d4f7ae5..527ab45 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
@@ -21,6 +21,7 @@ import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import com.google.common.collect.Maps;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -36,12 +37,15 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.replication.GrpcReplicationService;
import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
import org.apache.hadoop.test.GenericTestUtils;
@@ -49,6 +53,7 @@ import org.junit.Assert;
import org.junit.Test;
import java.io.File;
+import java.util.Map;
import java.util.UUID;
/**
@@ -57,9 +62,9 @@ import java.util.UUID;
public class TestContainerMetrics {
private GrpcReplicationService createReplicationService(
- ContainerSet containerSet) {
+ ContainerController controller) {
return new GrpcReplicationService(
- new OnDemandContainerReplicationSource(containerSet));
+ new OnDemandContainerReplicationSource(controller));
}
@Test
@@ -85,12 +90,21 @@ public class TestContainerMetrics {
VolumeSet volumeSet = new VolumeSet(
datanodeDetails.getUuidString(), conf);
ContainerSet containerSet = new ContainerSet();
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerProtos.ContainerType containerType :
+ ContainerProtos.ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(
+ containerType, conf, null, containerSet, volumeSet, metrics));
+ }
HddsDispatcher dispatcher = new HddsDispatcher(conf, containerSet,
- volumeSet, null);
+ volumeSet, handlers, null, metrics);
dispatcher.setScmId(UUID.randomUUID().toString());
server = new XceiverServerGrpc(datanodeDetails, conf, dispatcher,
- createReplicationService(containerSet));
+ createReplicationService(new ContainerController(
+ containerSet, handlers)));
client = new XceiverClientGrpc(pipeline, conf);
server.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
index 9a3fa1b..3e98594 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
@@ -18,11 +18,14 @@
package org.apache.hadoop.ozone.container.server;
+import com.google.common.collect.Maps;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.replication.GrpcReplicationService;
import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -57,6 +60,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import static org.apache.ratis.rpc.SupportedRpcType.GRPC;
import static org.apache.ratis.rpc.SupportedRpcType.NETTY;
@@ -71,15 +75,17 @@ public class TestContainerServer {
= GenericTestUtils.getTestDir("dfs").getAbsolutePath() + File.separator;
private GrpcReplicationService createReplicationService(
- ContainerSet containerSet) {
+ ContainerController containerController) {
return new GrpcReplicationService(
- new OnDemandContainerReplicationSource(containerSet));
+ new OnDemandContainerReplicationSource(containerController));
}
@Test
public void testClientServer() throws Exception {
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
ContainerSet containerSet = new ContainerSet();
+ ContainerController controller = new ContainerController(
+ containerSet, null);
runTestClientServer(1, (pipeline, conf) -> conf
.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
pipeline.getFirstNode()
@@ -87,7 +93,7 @@ public class TestContainerServer {
XceiverClientGrpc::new,
(dn, conf) -> new XceiverServerGrpc(datanodeDetails, conf,
new TestContainerDispatcher(),
- createReplicationService(containerSet)), (dn, p) -> {
+ createReplicationService(controller)), (dn, p) -> {
});
}
@@ -185,12 +191,22 @@ public class TestContainerServer {
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
ContainerSet containerSet = new ContainerSet();
+ VolumeSet volumeSet = mock(VolumeSet.class);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerProtos.ContainerType containerType :
+ ContainerProtos.ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(
+ containerType, conf, null, containerSet, volumeSet, metrics));
+ }
HddsDispatcher dispatcher = new HddsDispatcher(
- conf, mock(ContainerSet.class), mock(VolumeSet.class), null);
+ conf, containerSet, volumeSet, handlers, null, metrics);
dispatcher.init();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
server = new XceiverServerGrpc(datanodeDetails, conf, dispatcher,
- createReplicationService(containerSet));
+ createReplicationService(
+ new ContainerController(containerSet, null)));
client = new XceiverClientGrpc(pipeline, conf);
server.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index f9e5753..01b51fa 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -17,9 +17,12 @@
*/
package org.apache.hadoop.ozone.genesis;
+import com.google.common.collect.Maps;
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.statemachine
.DatanodeStateMachine.DatanodeStates;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
@@ -44,6 +47,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
@@ -100,9 +104,18 @@ public class BenchMarkDatanodeDispatcher {
ContainerSet containerSet = new ContainerSet();
VolumeSet volumeSet = new VolumeSet(datanodeUuid, conf);
-
- dispatcher = new HddsDispatcher(conf, containerSet, volumeSet,
- new StateContext(conf, DatanodeStates.RUNNING, null));
+ StateContext context = new StateContext(
+ conf, DatanodeStates.RUNNING, null);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerProtos.ContainerType containerType :
+ ContainerProtos.ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(
+ containerType, conf, context, containerSet, volumeSet, metrics));
+ }
+ dispatcher = new HddsDispatcher(conf, containerSet, volumeSet, handlers,
+ context, metrics);
dispatcher.init();
containerCount = new AtomicInteger();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeFastRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeFastRestart.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeFastRestart.java
index a91e190..4a0cb61 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeFastRestart.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeFastRestart.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.ozone.freon;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.container.common.transport
.server.XceiverServerSpi;
@@ -128,7 +127,7 @@ public class TestFreonWithDatanodeFastRestart {
private StateMachine getStateMachine() throws Exception {
XceiverServerSpi server =
cluster.getHddsDatanodes().get(0).getDatanodeStateMachine().
- getContainer().getServer(HddsProtos.ReplicationType.RATIS);
+ getContainer().getWriteChannel();
RaftServerProxy proxy =
(RaftServerProxy)(((XceiverServerRatis)server).getServer());
RaftGroupId groupId = proxy.getGroupIds().iterator().next();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDFS-13906. RBF: Add multiple
paths for dfsrouteradmin 'rm' and 'clrquota' commands. Contributed by Ayush
Saxena.
Posted by br...@apache.org.
HDFS-13906. RBF: Add multiple paths for dfsrouteradmin 'rm' and 'clrquota' commands. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/507ab988
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/507ab988
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/507ab988
Branch: refs/heads/HDFS-13891
Commit: 507ab98846864bbcea7042a786e38db78e9bb2b3
Parents: 892b33e
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri Oct 12 17:19:55 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:34:52 2018 +0530
----------------------------------------------------------------------
.../hdfs/tools/federation/RouterAdmin.java | 102 ++++++++++---------
.../federation/router/TestRouterAdminCLI.java | 82 ++++++++++++---
2 files changed, 122 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/507ab988/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
index 1aefe4f..4a9cc7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -151,17 +151,7 @@ public class RouterAdmin extends Configured implements Tool {
* @param arg List of of command line parameters.
*/
private void validateMax(String[] arg) {
- if (arg[0].equals("-rm")) {
- if (arg.length > 2) {
- throw new IllegalArgumentException(
- "Too many arguments, Max=1 argument allowed");
- }
- } else if (arg[0].equals("-ls")) {
- if (arg.length > 2) {
- throw new IllegalArgumentException(
- "Too many arguments, Max=1 argument allowed");
- }
- } else if (arg[0].equals("-clrQuota")) {
+ if (arg[0].equals("-ls")) {
if (arg.length > 2) {
throw new IllegalArgumentException(
"Too many arguments, Max=1 argument allowed");
@@ -183,63 +173,63 @@ public class RouterAdmin extends Configured implements Tool {
}
}
- @Override
- public int run(String[] argv) throws Exception {
- if (argv.length < 1) {
- System.err.println("Not enough parameters specified");
- printUsage();
- return -1;
- }
-
- int exitCode = -1;
- int i = 0;
- String cmd = argv[i++];
-
- // Verify that we have enough command line parameters
+ /**
+ * Usage: validates the minimum number of arguments for a command.
+ * @param argv List of of command line parameters.
+ * @return true if number of arguments are valid for the command else false.
+ */
+ private boolean validateMin(String[] argv) {
+ String cmd = argv[0];
if ("-add".equals(cmd)) {
if (argv.length < 4) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-update".equals(cmd)) {
if (argv.length < 4) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-rm".equals(cmd)) {
if (argv.length < 2) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-setQuota".equals(cmd)) {
if (argv.length < 4) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-clrQuota".equals(cmd)) {
if (argv.length < 2) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-safemode".equals(cmd)) {
if (argv.length < 2) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-nameservice".equals(cmd)) {
if (argv.length < 3) {
- System.err.println("Not enough parameters specificed for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
}
+ return true;
+ }
+
+ @Override
+ public int run(String[] argv) throws Exception {
+ if (argv.length < 1) {
+ System.err.println("Not enough parameters specified");
+ printUsage();
+ return -1;
+ }
+
+ int exitCode = -1;
+ int i = 0;
+ String cmd = argv[i++];
+ // Verify that we have enough command line parameters
+ if (!validateMin(argv)) {
+ System.err.println("Not enough parameters specificed for cmd " + cmd);
+ printUsage(cmd);
+ return exitCode;
+ }
// Initialize RouterClient
try {
String address = getConf().getTrimmed(
@@ -273,8 +263,17 @@ public class RouterAdmin extends Configured implements Tool {
exitCode = -1;
}
} else if ("-rm".equals(cmd)) {
- if (removeMount(argv[i])) {
- System.out.println("Successfully removed mount point " + argv[i]);
+ while (i < argv.length) {
+ try {
+ if (removeMount(argv[i])) {
+ System.out.println("Successfully removed mount point " + argv[i]);
+ }
+ } catch (IOException e) {
+ exitCode = -1;
+ System.err
+ .println(cmd.substring(1) + ": " + e.getLocalizedMessage());
+ }
+ i++;
}
} else if ("-ls".equals(cmd)) {
if (argv.length > 1) {
@@ -288,9 +287,12 @@ public class RouterAdmin extends Configured implements Tool {
"Successfully set quota for mount point " + argv[i]);
}
} else if ("-clrQuota".equals(cmd)) {
- if (clrQuota(argv[i])) {
- System.out.println(
- "Successfully clear quota for mount point " + argv[i]);
+ while (i < argv.length) {
+ if (clrQuota(argv[i])) {
+ System.out
+ .println("Successfully clear quota for mount point " + argv[i]);
+ i++;
+ }
}
} else if ("-safemode".equals(cmd)) {
manageSafeMode(argv[i]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/507ab988/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
index d968c60..65fe054 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
@@ -342,13 +342,43 @@ public class TestRouterAdminCLI {
assertEquals(0, ToolRunner.run(admin, argv));
assertTrue(out.toString().contains(
"Cannot remove mount point " + invalidPath));
+ }
- // test wrong number of arguments
- System.setErr(new PrintStream(err));
- argv = new String[] {"-rm", src, "check" };
- ToolRunner.run(admin, argv);
- assertTrue(err.toString()
- .contains("Too many arguments, Max=1 argument allowed"));
+ @Test
+ public void testMultiArgsRemoveMountTable() throws Exception {
+ String nsId = "ns0";
+ String src1 = "/test-rmmounttable1";
+ String src2 = "/test-rmmounttable2";
+ String dest1 = "/rmmounttable1";
+ String dest2 = "/rmmounttable2";
+ // Adding mount table entries
+ String[] argv = new String[] {"-add", src1, nsId, dest1};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ argv = new String[] {"-add", src2, nsId, dest2};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ // Ensure mount table entries added successfully
+ GetMountTableEntriesRequest getRequest =
+ GetMountTableEntriesRequest.newInstance(src1);
+ GetMountTableEntriesResponse getResponse =
+ client.getMountTableManager().getMountTableEntries(getRequest);
+ MountTable mountTable = getResponse.getEntries().get(0);
+ getRequest = GetMountTableEntriesRequest.newInstance(src2);
+ getResponse =
+ client.getMountTableManager().getMountTableEntries(getRequest);
+ assertEquals(src1, mountTable.getSourcePath());
+ mountTable = getResponse.getEntries().get(0);
+ assertEquals(src2, mountTable.getSourcePath());
+ // Remove multiple mount table entries
+ argv = new String[] {"-rm", src1, src2};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ // Verify successful deletion of mount table entries
+ getResponse =
+ client.getMountTableManager().getMountTableEntries(getRequest);
+ assertEquals(0, getResponse.getEntries().size());
}
@Test
@@ -540,6 +570,7 @@ public class TestRouterAdminCLI {
public void testSetAndClearQuota() throws Exception {
String nsId = "ns0";
String src = "/test-QuotaMounttable";
+ String src1 = "/test-QuotaMounttable1";
String dest = "/QuotaMounttable";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
@@ -605,15 +636,42 @@ public class TestRouterAdminCLI {
assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getQuota());
assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getSpaceQuota());
+ // verify multi args ClrQuota
+ String dest1 = "/QuotaMounttable1";
+ // Add mount table entries.
+ argv = new String[] {"-add", src, nsId, dest};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ argv = new String[] {"-add", src1, nsId, dest1};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ // SetQuota for the added entries
+ argv = new String[] {"-setQuota", src, "-nsQuota", String.valueOf(nsQuota),
+ "-ssQuota", String.valueOf(ssQuota)};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ argv = new String[] {"-setQuota", src1, "-nsQuota",
+ String.valueOf(nsQuota), "-ssQuota", String.valueOf(ssQuota)};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ // Clear quota for the added entries
+ argv = new String[] {"-clrQuota", src, src1};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ getResponse =
+ client.getMountTableManager().getMountTableEntries(getRequest);
+
+ // Verify clear quota for the entries
+ for (int i = 0; i < 2; i++) {
+ mountTable = getResponse.getEntries().get(i);
+ quotaUsage = mountTable.getQuota();
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getSpaceQuota());
+ }
+
// verify wrong arguments
System.setErr(new PrintStream(err));
- argv = new String[] {"-clrQuota", src, "check"};
- ToolRunner.run(admin, argv);
- assertTrue(err.toString(),
- err.toString().contains("Too many arguments, Max=1 argument allowed"));
-
argv = new String[] {"-setQuota", src, "check", "check2"};
- err.reset();
ToolRunner.run(admin, argv);
assertTrue(err.toString().contains("Invalid argument : check"));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HADOOP-15919. AliyunOSS: Enable
Yarn to use OSS. Contributed by wujinhu.
Posted by br...@apache.org.
HADOOP-15919. AliyunOSS: Enable Yarn to use OSS. Contributed by wujinhu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be0708c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be0708c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be0708c6
Branch: refs/heads/HDFS-13891
Commit: be0708c6eb3c759e43639e0f8df7c9910308fb97
Parents: d027a24
Author: Weiwei Yang <ww...@apache.org>
Authored: Mon Nov 19 14:00:54 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Mon Nov 19 14:00:54 2018 +0800
----------------------------------------------------------------------
hadoop-tools/hadoop-aliyun/pom.xml | 6 +
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 7 +-
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 2 -
.../apache/hadoop/fs/aliyun/oss/Constants.java | 2 +
.../org/apache/hadoop/fs/aliyun/oss/OSS.java | 43 +++++
.../site/markdown/tools/hadoop-aliyun/index.md | 10 ++
.../fs/aliyun/oss/AliyunOSSTestUtils.java | 19 ++-
.../oss/TestAliyunOSSBlockOutputStream.java | 3 +
.../aliyun/oss/contract/AliyunOSSContract.java | 9 ++
.../oss/fileContext/TestOSSFileContext.java | 28 ++++
.../TestOSSFileContextCreateMkdir.java | 40 +++++
.../TestOSSFileContextMainOperations.java | 71 ++++++++
.../TestOSSFileContextStatistics.java | 69 ++++++++
.../oss/fileContext/TestOSSFileContextURI.java | 51 ++++++
.../oss/fileContext/TestOSSFileContextUtil.java | 40 +++++
.../hadoop/fs/aliyun/oss/yarn/TestOSS.java | 83 ++++++++++
.../aliyun/oss/yarn/TestOSSMiniYarnCluster.java | 160 +++++++++++++++++++
17 files changed, 635 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
index 13e1296..eefdbf9 100644
--- a/hadoop-tools/hadoop-aliyun/pom.xml
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -142,5 +142,11 @@
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-examples</artifactId>
+ <scope>test</scope>
+ <type>jar</type>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 9c4435c..3def6a7 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -136,7 +136,7 @@ public class AliyunOSSFileSystem extends FileSystem {
key,
uploadPartSize,
new SemaphoredDelegatingExecutor(boundedThreadPool,
- blockOutputActiveBlocks, true)), (Statistics)(null));
+ blockOutputActiveBlocks, true)), statistics);
}
/**
@@ -298,6 +298,11 @@ public class AliyunOSSFileSystem extends FileSystem {
}
@Override
+ public int getDefaultPort() {
+ return Constants.OSS_DEFAULT_PORT;
+ }
+
+ @Override
public Path getWorkingDirectory() {
return workingDir;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index 4fc1325..c0caee8 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -416,7 +416,6 @@ public class AliyunOSSFileSystemStore {
PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta);
LOG.debug(result.getETag());
statistics.incrementWriteOps(1);
- statistics.incrementBytesWritten(file.length());
} finally {
fis.close();
}
@@ -617,7 +616,6 @@ public class AliyunOSSFileSystemStore {
uploadRequest.setPartNumber(idx);
UploadPartResult uploadResult = ossClient.uploadPart(uploadRequest);
statistics.incrementWriteOps(1);
- statistics.incrementBytesWritten(file.length());
return uploadResult.getPartETag();
} catch (Exception e) {
LOG.debug("Failed to upload "+ file.getPath() +", " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index 24e35d9..43886d6 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -38,6 +38,8 @@ public final class Constants {
public static final String CREDENTIALS_PROVIDER_KEY =
"fs.oss.credentials.provider";
+ public static final int OSS_DEFAULT_PORT = -1;
+
// OSS access verification
public static final String ACCESS_KEY_ID = "fs.oss.accessKeyId";
public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSS.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSS.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSS.java
new file mode 100644
index 0000000..d545138
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSS.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * OSS implementation of AbstractFileSystem.
+ * This impl delegates to the AliyunOSSFileSystem.
+ */
+public class OSS extends DelegateToFileSystem {
+
+ public OSS(URI theUri, Configuration conf)
+ throws IOException, URISyntaxException {
+ super(theUri, new AliyunOSSFileSystem(), conf, "oss", false);
+ }
+
+ @Override
+ public int getUriDefaultPort() {
+ return Constants.OSS_DEFAULT_PORT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
index 87aa90b..425fee5 100644
--- a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
+++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
@@ -111,6 +111,16 @@ please raise your issues with them.
### Other properties
<property>
+ <name>fs.AbstractFileSystem.oss.impl</name>
+ <value>org.apache.hadoop.fs.aliyun.oss.OSS</value>
+ <description>The implementation class of the OSS AbstractFileSystem.
+ If you want to use OSS as YARN’s resource storage dir via the
+ fs.defaultFS configuration property in Hadoop’s core-site.xml,
+ you should add this configuration to Hadoop's core-site.xml
+ </description>
+ </property>
+
+ <property>
<name>fs.oss.endpoint</name>
<description>Aliyun OSS endpoint to connect to. An up-to-date list is
provided in the Aliyun OSS Documentation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
index 79e0de3..cdf4971 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.aliyun.oss;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
import org.junit.internal.AssumptionViolatedException;
import java.io.IOException;
@@ -45,10 +46,21 @@ public final class AliyunOSSTestUtils {
*/
public static AliyunOSSFileSystem createTestFileSystem(Configuration conf)
throws IOException {
+ AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem();
+ ossfs.initialize(getURI(conf), conf);
+ return ossfs;
+ }
+
+ public static FileContext createTestFileContext(Configuration conf) throws
+ IOException {
+ return FileContext.getFileContext(getURI(conf), conf);
+ }
+
+ private static URI getURI(Configuration conf) {
String fsname = conf.getTrimmed(
TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME, "");
- boolean liveTest = StringUtils.isNotEmpty(fsname);
+ boolean liveTest = !StringUtils.isEmpty(fsname);
URI testURI = null;
if (liveTest) {
testURI = URI.create(fsname);
@@ -59,11 +71,8 @@ public final class AliyunOSSTestUtils {
throw new AssumptionViolatedException("No test filesystem in "
+ TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME);
}
- AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem();
- ossfs.initialize(testURI, conf);
- return ossfs;
+ return testURI;
}
-
/**
* Generate unique test path for multiple user tests.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
index c3387a3..47a2494 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
@@ -75,6 +75,7 @@ public class TestAliyunOSSBlockOutputStream {
@Test
public void testRegularUpload() throws IOException {
+ FileSystem.clearStatistics();
long size = 1024 * 1024;
FileSystem.Statistics statistics =
FileSystem.getStatistics("oss", AliyunOSSFileSystem.class);
@@ -111,6 +112,7 @@ public class TestAliyunOSSBlockOutputStream {
@Test
public void testMultiPartUpload() throws IOException {
long size = 6 * 1024 * 1024;
+ FileSystem.clearStatistics();
FileSystem.Statistics statistics =
FileSystem.getStatistics("oss", AliyunOSSFileSystem.class);
ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size - 1);
@@ -134,6 +136,7 @@ public class TestAliyunOSSBlockOutputStream {
@Test
public void testMultiPartUploadConcurrent() throws IOException {
+ FileSystem.clearStatistics();
long size = 50 * 1024 * 1024 - 1;
ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size);
FileSystem.Statistics statistics =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
index 624c606..fd0afca 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
@@ -19,9 +19,13 @@
package org.apache.hadoop.fs.aliyun.oss.contract;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+import java.io.IOException;
+
/**
* The contract of Aliyun OSS: only enabled if the test bucket is provided.
*/
@@ -41,6 +45,11 @@ public class AliyunOSSContract extends AbstractBondedFSContract {
}
@Override
+ public FileSystem getTestFileSystem() throws IOException {
+ return AliyunOSSTestUtils.createTestFileSystem(new Configuration());
+ }
+
+ @Override
public Path getTestPath() {
String testUniqueForkId = System.getProperty("test.unique.fork.id");
return testUniqueForkId == null ? super.getTestPath() :
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContext.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContext.java
new file mode 100644
index 0000000..530797b
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContext.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.fs.TestFileContext;
+
+/**
+ * Implementation of TestFileContext for OSS.
+ */
+public class TestOSSFileContext extends TestFileContext {
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextCreateMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextCreateMkdir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextCreateMkdir.java
new file mode 100644
index 0000000..7bddbd0
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextCreateMkdir.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.Before;
+
+import java.io.IOException;
+
+/**
+ * OSS implementation of FileContextCreateMkdirBaseTest.
+ */
+public class TestOSSFileContextCreateMkdir
+ extends FileContextCreateMkdirBaseTest {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = AliyunOSSTestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextMainOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextMainOperations.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextMainOperations.java
new file mode 100644
index 0000000..36d02df
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextMainOperations.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextMainOperationsBaseTest;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * OSS implementation of FileContextMainOperationsBaseTest.
+ */
+public class TestOSSFileContextMainOperations
+ extends FileContextMainOperationsBaseTest {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = AliyunOSSTestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+ @Override
+ protected boolean listCorruptedBlocksSupported() {
+ return false;
+ }
+
+ @Test
+ @Ignore
+ public void testCreateFlagAppendExistingFile() throws IOException {
+ // append not supported, so test removed
+ }
+
+ @Test
+ @Ignore
+ public void testCreateFlagCreateAppendExistingFile() throws IOException {
+ // append not supported, so test removed
+ }
+
+ @Test
+ @Ignore
+ public void testSetVerifyChecksum() throws IOException {
+ // checksums ignored, so test ignored
+ }
+
+ @Test
+ @Ignore
+ public void testBuilderCreateAppendExistingFile() throws IOException {
+ // append not supported, so test removed
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextStatistics.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextStatistics.java
new file mode 100644
index 0000000..cbb7d85
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextStatistics.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FCStatisticsBaseTest;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+import java.net.URI;
+
+/**
+ * OSS implementation of FCStatisticsBaseTest.
+ */
+public class TestOSSFileContextStatistics extends FCStatisticsBaseTest {
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fc = AliyunOSSTestUtils.createTestFileContext(conf);
+ fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"),
+ FileContext.DEFAULT_PERM, true);
+ FileContext.clearStatistics();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fc != null) {
+ fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true);
+ }
+ }
+
+ @Override
+ protected void verifyReadBytes(FileSystem.Statistics stats) {
+ // one blockSize for read, one for pread
+ Assert.assertEquals(2 * blockSize, stats.getBytesRead());
+ }
+
+ @Override
+ protected void verifyWrittenBytes(FileSystem.Statistics stats) {
+ // no extra bytes are written
+ Assert.assertEquals(blockSize, stats.getBytesWritten());
+ }
+
+ @Override
+ protected URI getFsUri() {
+ return fc.getHomeDirectory().toUri();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextURI.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextURI.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextURI.java
new file mode 100644
index 0000000..cde708e
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextURI.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextURIBase;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * OSS implementation of FileContextURIBase.
+ */
+public class TestOSSFileContextURI extends FileContextURIBase {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc1 = AliyunOSSTestUtils.createTestFileContext(conf);
+ // different object, same FS
+ fc2 = AliyunOSSTestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+ @Test
+ @Ignore
+ public void testFileStatus() throws IOException {
+ // test ignored
+ // (the statistics tested with this method are not relevant for an OSSFS)
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextUtil.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextUtil.java
new file mode 100644
index 0000000..ce55c54
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextUtil.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextUtilBase;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.Before;
+
+import java.io.IOException;
+
+/**
+ * OSS implementation of FileContextUtilBase.
+ */
+public class TestOSSFileContextUtil extends FileContextUtilBase {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = AliyunOSSTestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSS.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSS.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSS.java
new file mode 100644
index 0000000..08ef60f
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSS.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.yarn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FsStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.EnumSet;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * OSS tests through the {@link FileContext} API.
+ */
+public class TestOSS {
+ private FileContext fc;
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fc = AliyunOSSTestUtils.createTestFileContext(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fc != null) {
+ fc.delete(getTestPath(), true);
+ }
+ }
+
+ protected Path getTestPath() {
+ return new Path(AliyunOSSTestUtils.generateUniqueTestPath());
+ }
+
+ @Test
+ public void testOSSStatus() throws Exception {
+ FsStatus fsStatus = fc.getFsStatus(null);
+ assertNotNull(fsStatus);
+ assertTrue("Used capacity should be positive: " + fsStatus.getUsed(),
+ fsStatus.getUsed() >= 0);
+ assertTrue("Remaining capacity should be positive: " + fsStatus
+ .getRemaining(),
+ fsStatus.getRemaining() >= 0);
+ assertTrue("Capacity should be positive: " + fsStatus.getCapacity(),
+ fsStatus.getCapacity() >= 0);
+ }
+
+ @Test(timeout = 90000L)
+ public void testOSSCreateFileInSubDir() throws Exception {
+ Path dirPath = getTestPath();
+ fc.mkdir(dirPath, FileContext.DIR_DEFAULT_PERM, true);
+ Path filePath = new Path(dirPath, "file");
+ try (FSDataOutputStream file = fc.create(filePath, EnumSet.of(CreateFlag
+ .CREATE))) {
+ file.write(666);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be0708c6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSSMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSSMiniYarnCluster.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSSMiniYarnCluster.java
new file mode 100644
index 0000000..439aa0c
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSSMiniYarnCluster.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.yarn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.examples.WordCount;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Tests that OSS is usable through a YARN application.
+ */
+public class TestOSSMiniYarnCluster extends AbstractFSContractTestBase {
+
+ private Configuration conf;
+ private FileSystem fs;
+ private MiniYARNCluster yarnCluster;
+ private Path rootPath;
+
+ @Override
+ protected AbstractFSContract createContract(Configuration configuration) {
+ this.conf = configuration;
+ return new AliyunOSSContract(conf);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ fs = getFileSystem();
+ rootPath = path("MiniClusterWordCount");
+ Path workingDir = path("working");
+ fs.setWorkingDirectory(workingDir);
+ fs.mkdirs(new Path(rootPath, "input/"));
+
+ yarnCluster = new MiniYARNCluster("MiniClusterWordCount", // testName
+ 1, // number of node managers
+ 1, // number of local log dirs per node manager
+ 1); // number of hdfs dirs per node manager
+ yarnCluster.init(conf);
+ yarnCluster.start();
+ }
+
+ @Test
+ public void testWithMiniCluster() throws Exception {
+ Path input = new Path(rootPath, "input/in");
+ input = input.makeQualified(fs.getUri(), fs.getWorkingDirectory());
+ Path output = new Path(rootPath, "output/");
+ output = output.makeQualified(fs.getUri(), fs.getWorkingDirectory());
+
+ writeStringToFile(input, "first line\nsecond line\nthird line");
+
+ Job job = Job.getInstance(conf, "word count");
+ job.setJarByClass(WordCount.class);
+ job.setMapperClass(WordCount.TokenizerMapper.class);
+ job.setCombinerClass(WordCount.IntSumReducer.class);
+ job.setReducerClass(WordCount.IntSumReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+ FileInputFormat.addInputPath(job, input);
+ FileOutputFormat.setOutputPath(job, output);
+
+ int exitCode = (job.waitForCompletion(true) ? 0 : 1);
+ assertEquals("Returned error code.", 0, exitCode);
+
+ assertTrue(fs.exists(new Path(output, "_SUCCESS")));
+ String outputAsStr = readStringFromFile(new Path(output, "part-r-00000"));
+ Map<String, Integer> resAsMap = getResultAsMap(outputAsStr);
+
+ assertEquals(4, resAsMap.size());
+ assertEquals(1, (int) resAsMap.get("first"));
+ assertEquals(1, (int) resAsMap.get("second"));
+ assertEquals(1, (int) resAsMap.get("third"));
+ assertEquals(3, (int) resAsMap.get("line"));
+ }
+
+
+ /**
+ * helper method.
+ */
+ private Map<String, Integer> getResultAsMap(String outputAsStr) {
+ Map<String, Integer> result = new HashMap<>();
+ for (String line : outputAsStr.split("\n")) {
+ String[] tokens = line.split("\t");
+ assertTrue("Not enough tokens in in string \" "
+ + line + "\" from output \"" + outputAsStr + "\"",
+ tokens.length > 1);
+ result.put(tokens[0], Integer.parseInt(tokens[1]));
+ }
+ return result;
+ }
+
+ /**
+ * helper method.
+ */
+ private void writeStringToFile(Path path, String string) throws IOException {
+ FileContext fc = AliyunOSSTestUtils.createTestFileContext(conf);
+ try (FSDataOutputStream file = fc.create(path,
+ EnumSet.of(CreateFlag.CREATE))) {
+ file.write(string.getBytes());
+ }
+ }
+
+ /**
+ * helper method.
+ */
+ private String readStringFromFile(Path path) throws IOException {
+ try (FSDataInputStream in = fs.open(path)) {
+ long bytesLen = fs.getFileStatus(path).getLen();
+ byte[] buffer = new byte[(int) bytesLen];
+ IOUtils.readFully(in, buffer, 0, buffer.length);
+ return new String(buffer);
+ }
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ if (yarnCluster != null) {
+ yarnCluster.stop();
+ }
+ super.teardown();
+ IOUtils.closeStream(getFileSystem());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HDFS-12284. RBF: Support for
Kerberos authentication. Contributed by Sherwood Zheng and Inigo Goiri.
Posted by br...@apache.org.
HDFS-12284. RBF: Support for Kerberos authentication. Contributed by Sherwood Zheng and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb40d229
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb40d229
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb40d229
Branch: refs/heads/HDFS-13891
Commit: bb40d2293df63f2220264f680728a431d7234621
Parents: 10ea633
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Wed Nov 7 07:33:37 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:34:52 2018 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 10 ++
.../server/federation/router/RBFConfigKeys.java | 11 ++
.../hdfs/server/federation/router/Router.java | 28 ++++
.../federation/router/RouterAdminServer.java | 7 +
.../federation/router/RouterHttpServer.java | 5 +-
.../federation/router/RouterRpcClient.java | 9 +-
.../federation/router/RouterRpcServer.java | 12 ++
.../src/main/resources/hdfs-rbf-default.xml | 47 ++++++
.../fs/contract/router/RouterHDFSContract.java | 9 +-
.../fs/contract/router/SecurityConfUtil.java | 156 +++++++++++++++++++
.../TestRouterHDFSContractAppendSecure.java | 46 ++++++
.../TestRouterHDFSContractConcatSecure.java | 51 ++++++
.../TestRouterHDFSContractCreateSecure.java | 48 ++++++
.../TestRouterHDFSContractDeleteSecure.java | 46 ++++++
...stRouterHDFSContractGetFileStatusSecure.java | 47 ++++++
.../TestRouterHDFSContractMkdirSecure.java | 48 ++++++
.../TestRouterHDFSContractOpenSecure.java | 47 ++++++
.../TestRouterHDFSContractRenameSecure.java | 48 ++++++
...stRouterHDFSContractRootDirectorySecure.java | 63 ++++++++
.../TestRouterHDFSContractSeekSecure.java | 48 ++++++
.../TestRouterHDFSContractSetTimesSecure.java | 48 ++++++
.../server/federation/MiniRouterDFSCluster.java | 58 ++++++-
22 files changed, 879 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
index 386eb41..9f515bc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
@@ -35,6 +35,16 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependencies>
<dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk16</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
index bbd4250..fa474f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
@@ -242,4 +242,15 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
FEDERATION_ROUTER_PREFIX + "quota-cache.update.interval";
public static final long DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL_DEFAULT =
60000;
+
+ // HDFS Router security
+ public static final String DFS_ROUTER_KEYTAB_FILE_KEY =
+ FEDERATION_ROUTER_PREFIX + "keytab.file";
+ public static final String DFS_ROUTER_KERBEROS_PRINCIPAL_KEY =
+ FEDERATION_ROUTER_PREFIX + "kerberos.principal";
+ public static final String DFS_ROUTER_KERBEROS_PRINCIPAL_HOSTNAME_KEY =
+ FEDERATION_ROUTER_PREFIX + "kerberos.principal.hostname";
+
+ public static final String DFS_ROUTER_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY =
+ FEDERATION_ROUTER_PREFIX + "kerberos.internal.spnego.principal";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index 5ddc129..3288273 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_KERBEROS_PRINCIPAL_HOSTNAME_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_KEYTAB_FILE_KEY;
+
import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.newActiveNamenodeResolver;
import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.newFileSubclusterResolver;
@@ -41,6 +45,8 @@ import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.JvmPauseMonitor;
import org.apache.hadoop.util.Time;
@@ -145,6 +151,11 @@ public class Router extends CompositeService {
this.conf = configuration;
updateRouterState(RouterServiceState.INITIALIZING);
+ // Enable the security for the Router
+ UserGroupInformation.setConfiguration(conf);
+ SecurityUtil.login(conf, DFS_ROUTER_KEYTAB_FILE_KEY,
+ DFS_ROUTER_KERBEROS_PRINCIPAL_KEY, getHostName(conf));
+
if (conf.getBoolean(
RBFConfigKeys.DFS_ROUTER_STORE_ENABLE,
RBFConfigKeys.DFS_ROUTER_STORE_ENABLE_DEFAULT)) {
@@ -246,6 +257,23 @@ public class Router extends CompositeService {
super.serviceInit(conf);
}
+ /**
+ * Returns the hostname for this Router. If the hostname is not
+ * explicitly configured in the given config, then it is determined.
+ *
+ * @param config configuration
+ * @return the hostname (NB: may not be a FQDN)
+ * @throws UnknownHostException if the hostname cannot be determined
+ */
+ private static String getHostName(Configuration config)
+ throws UnknownHostException {
+ String name = config.get(DFS_ROUTER_KERBEROS_PRINCIPAL_HOSTNAME_KEY);
+ if (name == null) {
+ name = InetAddress.getLocalHost().getHostName();
+ }
+ return name;
+ }
+
@Override
protected void serviceStart() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
index e7fec9e..f34dc41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY;
@@ -27,6 +28,7 @@ import java.util.Set;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HDFSPolicyProvider;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService;
@@ -142,6 +144,11 @@ public class RouterAdminServer extends AbstractService
.setVerbose(false)
.build();
+ // Set service-level authorization security policy
+ if (conf.getBoolean(HADOOP_SECURITY_AUTHORIZATION, false)) {
+ this.adminServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
+ }
+
// The RPC-server port can be ephemeral... ensure we have the correct info
InetSocketAddress listenAddress = this.adminServer.getListenerAddress();
this.adminAddress = new InetSocketAddress(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
index d223e2a..d6a5146 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.federation.router;
import java.net.InetSocketAddress;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
@@ -84,8 +83,8 @@ public class RouterHttpServer extends AbstractService {
String webApp = "router";
HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(
this.conf, this.httpAddress, this.httpsAddress, webApp,
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY);
+ RBFConfigKeys.DFS_ROUTER_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
+ RBFConfigKeys.DFS_ROUTER_KEYTAB_FILE_KEY);
this.httpServer = builder.build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index 34f51ec..a21e980 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -255,7 +255,14 @@ public class RouterRpcClient {
// for each individual request.
// TODO Add tokens from the federated UGI
- connection = this.connectionManager.getConnection(ugi, rpcAddress, proto);
+ UserGroupInformation connUGI = ugi;
+ if (UserGroupInformation.isSecurityEnabled()) {
+ UserGroupInformation routerUser = UserGroupInformation.getLoginUser();
+ connUGI = UserGroupInformation.createProxyUser(
+ ugi.getUserName(), routerUser);
+ }
+ connection = this.connectionManager.getConnection(
+ connUGI, rpcAddress, proto);
LOG.debug("User {} NN {} is using connection {}",
ugi.getUserName(), rpcAddress, connection);
} catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 525d6d5..0f27efa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT;
@@ -60,6 +61,7 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.AddBlockFlag;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.HDFSPolicyProvider;
import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
@@ -174,6 +176,9 @@ public class RouterRpcServer extends AbstractService
/** Monitor metrics for the RPC calls. */
private final RouterRpcMonitor rpcMonitor;
+ /** If we use authentication for the connections. */
+ private final boolean serviceAuthEnabled;
+
/** Interface to identify the active NN for a nameservice or blockpool ID. */
private final ActiveNamenodeResolver namenodeResolver;
@@ -265,6 +270,13 @@ public class RouterRpcServer extends AbstractService
DFSUtil.addPBProtocol(
conf, NamenodeProtocolPB.class, nnPbService, this.rpcServer);
+ // Set service-level authorization security policy
+ this.serviceAuthEnabled = conf.getBoolean(
+ HADOOP_SECURITY_AUTHORIZATION, false);
+ if (this.serviceAuthEnabled) {
+ rpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
+ }
+
// We don't want the server to log the full stack trace for some exceptions
this.rpcServer.addTerseExceptions(
RemoteException.class,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
index 3f56043..29c3093 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
@@ -465,4 +465,51 @@
</description>
</property>
+ <property>
+ <name>dfs.federation.router.keytab.file</name>
+ <value></value>
+ <description>
+ The keytab file used by router to login as its
+ service principal. The principal name is configured with
+ dfs.federation.router.kerberos.principal.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.kerberos.principal</name>
+ <value></value>
+ <description>
+ The Router service principal. This is typically set to
+ router/_HOST@REALM.TLD. Each Router will substitute _HOST with its
+ own fully qualified hostname at startup. The _HOST placeholder
+ allows using the same configuration setting on both Router
+ in an HA setup.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.kerberos.principal.hostname</name>
+ <value></value>
+ <description>
+ Optional. The hostname for the Router containing this
+ configuration file. Will be different for each machine.
+ Defaults to current hostname.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.kerberos.internal.spnego.principal</name>
+ <value>${dfs.web.authentication.kerberos.principal}</value>
+ <description>
+ The server principal used by the Router for web UI SPNEGO
+ authentication when Kerberos security is enabled. This is
+ typically set to HTTP/_HOST@REALM.TLD The SPNEGO server principal
+ begins with the prefix HTTP/ by convention.
+
+ If the value is '*', the web server will attempt to login with
+ every principal specified in the keytab file
+ dfs.web.authentication.kerberos.keytab.
+ </description>
+ </property>
+
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/RouterHDFSContract.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/RouterHDFSContract.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/RouterHDFSContract.java
index 97a426e..510cb95 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/RouterHDFSContract.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/RouterHDFSContract.java
@@ -43,12 +43,17 @@ public class RouterHDFSContract extends HDFSContract {
}
public static void createCluster() throws IOException {
+ createCluster(null);
+ }
+
+ public static void createCluster(Configuration conf) throws IOException {
try {
- cluster = new MiniRouterDFSCluster(true, 2);
+ cluster = new MiniRouterDFSCluster(true, 2, conf);
// Start NNs and DNs and wait until ready
- cluster.startCluster();
+ cluster.startCluster(conf);
+ cluster.addRouterOverrides(conf);
// Start routers with only an RPC service
cluster.startRouters();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/SecurityConfUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/SecurityConfUtil.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/SecurityConfUtil.java
new file mode 100644
index 0000000..deb6ace
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/SecurityConfUtil.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SERVICE_USER_NAME_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+
+/**
+ * Test utility to provide a standard routine to initialize the configuration
+ * for secure RBF HDFS cluster.
+ */
+public final class SecurityConfUtil {
+
+ // SSL keystore
+ private static String keystoresDir;
+ private static String sslConfDir;
+
+ // State string for mini dfs
+ private static final String SPNEGO_USER_NAME = "HTTP";
+ private static final String ROUTER_USER_NAME = "router";
+
+ private static String spnegoPrincipal;
+ private static String routerPrincipal;
+
+ private SecurityConfUtil() {
+ // Utility Class
+ }
+
+ public static Configuration initSecurity() throws Exception {
+ // delete old test dir
+ File baseDir = GenericTestUtils.getTestDir(
+ SecurityConfUtil.class.getSimpleName());
+ FileUtil.fullyDelete(baseDir);
+ assertTrue(baseDir.mkdirs());
+
+ // start a mini kdc with default conf
+ Properties kdcConf = MiniKdc.createConf();
+ MiniKdc kdc = new MiniKdc(kdcConf, baseDir);
+ kdc.start();
+
+ Configuration conf = new HdfsConfiguration();
+ SecurityUtil.setAuthenticationMethod(
+ UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
+
+ UserGroupInformation.setConfiguration(conf);
+ assertTrue("Expected configuration to enable security",
+ UserGroupInformation.isSecurityEnabled());
+
+ // Setup the keytab
+ File keytabFile = new File(baseDir, "test.keytab");
+ String keytab = keytabFile.getAbsolutePath();
+
+ // Windows will not reverse name lookup "127.0.0.1" to "localhost".
+ String krbInstance = Path.WINDOWS ? "127.0.0.1" : "localhost";
+
+ kdc.createPrincipal(keytabFile,
+ SPNEGO_USER_NAME + "/" + krbInstance,
+ ROUTER_USER_NAME + "/" + krbInstance);
+
+ routerPrincipal =
+ ROUTER_USER_NAME + "/" + krbInstance + "@" + kdc.getRealm();
+ spnegoPrincipal =
+ SPNEGO_USER_NAME + "/" + krbInstance + "@" + kdc.getRealm();
+
+ // Set auth configuration for mini DFS
+ conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ conf.set(HADOOP_SECURITY_SERVICE_USER_NAME_KEY, routerPrincipal);
+
+ // Setup principles and keytabs for dfs
+ conf.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, routerPrincipal);
+ conf.set(DFS_NAMENODE_KEYTAB_FILE_KEY, keytab);
+ conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, routerPrincipal);
+ conf.set(DFS_DATANODE_KEYTAB_FILE_KEY, keytab);
+ conf.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, spnegoPrincipal);
+ conf.set(DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY, keytab);
+
+ conf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
+ conf.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
+
+ conf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
+ conf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, "authentication");
+ conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
+
+ // Setup SSL configuration
+ keystoresDir = baseDir.getAbsolutePath();
+ sslConfDir = KeyStoreTestUtil.getClasspathDir(
+ SecurityConfUtil.class);
+ KeyStoreTestUtil.setupSSLConfig(
+ keystoresDir, sslConfDir, conf, false);
+ conf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY,
+ KeyStoreTestUtil.getClientSSLConfigFileName());
+ conf.set(DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY,
+ KeyStoreTestUtil.getServerSSLConfigFileName());
+
+ // Setup principals and keytabs for router
+ conf.set(DFS_ROUTER_KEYTAB_FILE_KEY, keytab);
+ conf.set(DFS_ROUTER_KERBEROS_PRINCIPAL_KEY, routerPrincipal);
+ conf.set(DFS_ROUTER_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY, "*");
+
+ // Setup basic state store
+ conf.setClass(RBFConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
+ StateStoreFileImpl.class, StateStoreDriver.class);
+
+ // We need to specify the host to prevent 0.0.0.0 as the host address
+ conf.set(DFS_ROUTER_RPC_BIND_HOST_KEY, "localhost");
+
+ return conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractAppendSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractAppendSecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractAppendSecure.java
new file mode 100644
index 0000000..fe4951d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractAppendSecure.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractAppendTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+/**
+ * Test secure append operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractAppendSecure
+ extends AbstractContractAppendTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractConcatSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractConcatSecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractConcatSecure.java
new file mode 100644
index 0000000..c9a0cc8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractConcatSecure.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractContractConcatTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+
+/**
+ * Test secure concat operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractConcatSecure
+ extends AbstractContractConcatTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ // perform a simple operation on the cluster to verify it is up
+ RouterHDFSContract.getFileSystem().getDefaultBlockSize(new Path("/"));
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractCreateSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractCreateSecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractCreateSecure.java
new file mode 100644
index 0000000..dc264b0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractCreateSecure.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+
+/**
+ * Test secure create operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractCreateSecure
+ extends AbstractContractCreateTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractDeleteSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractDeleteSecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractDeleteSecure.java
new file mode 100644
index 0000000..57cc138
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractDeleteSecure.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+/**
+ * Test secure delete operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractDeleteSecure
+ extends AbstractContractDeleteTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractGetFileStatusSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractGetFileStatusSecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractGetFileStatusSecure.java
new file mode 100644
index 0000000..13e4e96
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractGetFileStatusSecure.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+
+/**
+ * Test secure get file status operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractGetFileStatusSecure
+ extends AbstractContractGetFileStatusTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractMkdirSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractMkdirSecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractMkdirSecure.java
new file mode 100644
index 0000000..7c223a6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractMkdirSecure.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+
+/**
+ * Test secure dir operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractMkdirSecure
+ extends AbstractContractMkdirTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractOpenSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractOpenSecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractOpenSecure.java
new file mode 100644
index 0000000..434402c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractOpenSecure.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+
+/**
+ * Test secure open operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractOpenSecure extends AbstractContractOpenTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRenameSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRenameSecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRenameSecure.java
new file mode 100644
index 0000000..29d7398
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRenameSecure.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+
+/**
+ * Test secure rename operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractRenameSecure
+ extends AbstractContractRenameTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRootDirectorySecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRootDirectorySecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRootDirectorySecure.java
new file mode 100644
index 0000000..32ec161
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRootDirectorySecure.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+
+/**
+ * Test secure root dir operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractRootDirectorySecure
+ extends AbstractContractRootDirectoryTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+
+ @Override
+ public void testListEmptyRootDirectory() throws IOException {
+ // It doesn't apply because we still have the mount points here
+ }
+
+ @Override
+ public void testRmEmptyRootDirNonRecursive() throws IOException {
+ // It doesn't apply because we still have the mount points here
+ }
+
+ @Override
+ public void testRecursiveRootListing() throws IOException {
+ // It doesn't apply because we still have the mount points here
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSeekSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSeekSecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSeekSecure.java
new file mode 100644
index 0000000..f281b47
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSeekSecure.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+
+/**
+ * Test secure seek operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractSeekSecure extends AbstractContractSeekTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSetTimesSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSetTimesSecure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSetTimesSecure.java
new file mode 100644
index 0000000..8f86b95
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSetTimesSecure.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.fs.contract.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSetTimesTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.router.SecurityConfUtil.initSecurity;
+
+
+/**
+ * Test secure set times operations on the Router-based FS.
+ */
+public class TestRouterHDFSContractSetTimesSecure
+ extends AbstractContractSetTimesTest {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ RouterHDFSContract.createCluster(initSecurity());
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ RouterHDFSContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RouterHDFSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40d229/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
index e34713d..a5693a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
@@ -28,6 +28,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDR
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.waitNamenodeRegistered;
@@ -85,6 +87,7 @@ import org.apache.hadoop.hdfs.server.federation.router.RouterClient;
import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service.STATE;
@@ -270,6 +273,7 @@ public class MiniRouterDFSCluster {
private int servicePort;
private int lifelinePort;
private int httpPort;
+ private int httpsPort;
private URI fileSystemUri;
private int index;
private DFSClient client;
@@ -305,7 +309,12 @@ public class MiniRouterDFSCluster {
this.rpcPort = nn.getNameNodeAddress().getPort();
this.servicePort = nn.getServiceRpcAddress().getPort();
this.lifelinePort = nn.getServiceRpcAddress().getPort();
- this.httpPort = nn.getHttpAddress().getPort();
+ if (nn.getHttpAddress() != null) {
+ this.httpPort = nn.getHttpAddress().getPort();
+ }
+ if (nn.getHttpsAddress() != null) {
+ this.httpsPort = nn.getHttpsAddress().getPort();
+ }
this.fileSystemUri = new URI("hdfs://" + namenode.getHostAndPort());
DistributedFileSystem.setDefaultUri(this.conf, this.fileSystemUri);
@@ -328,10 +337,22 @@ public class MiniRouterDFSCluster {
return namenode.getServiceRpcAddress().getHostName() + ":" + lifelinePort;
}
+ public String getWebAddress() {
+ if (conf.get(DFS_HTTP_POLICY_KEY)
+ .equals(HttpConfig.Policy.HTTPS_ONLY.name())) {
+ return getHttpsAddress();
+ }
+ return getHttpAddress();
+ }
+
public String getHttpAddress() {
return namenode.getHttpAddress().getHostName() + ":" + httpPort;
}
+ public String getHttpsAddress() {
+ return namenode.getHttpsAddress().getHostName() + ":" + httpsPort;
+ }
+
public FileSystem getFileSystem() throws IOException {
return DistributedFileSystem.get(conf);
}
@@ -375,22 +396,38 @@ public class MiniRouterDFSCluster {
public MiniRouterDFSCluster(
boolean ha, int numNameservices, int numNamenodes,
- long heartbeatInterval, long cacheFlushInterval) {
+ long heartbeatInterval, long cacheFlushInterval,
+ Configuration overrideConf) {
this.highAvailability = ha;
this.heartbeatInterval = heartbeatInterval;
this.cacheFlushInterval = cacheFlushInterval;
- configureNameservices(numNameservices, numNamenodes);
+ configureNameservices(numNameservices, numNamenodes, overrideConf);
+ }
+
+ public MiniRouterDFSCluster(
+ boolean ha, int numNameservices, int numNamenodes,
+ long heartbeatInterval, long cacheFlushInterval) {
+ this(ha, numNameservices, numNamenodes,
+ heartbeatInterval, cacheFlushInterval, null);
}
public MiniRouterDFSCluster(boolean ha, int numNameservices) {
this(ha, numNameservices, 2,
- DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS,
+ null);
}
public MiniRouterDFSCluster(
boolean ha, int numNameservices, int numNamenodes) {
this(ha, numNameservices, numNamenodes,
- DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS,
+ null);
+ }
+
+ public MiniRouterDFSCluster(boolean ha, int numNameservices,
+ Configuration overrideConf) {
+ this(ha, numNameservices, 2,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS, overrideConf);
}
/**
@@ -447,6 +484,8 @@ public class MiniRouterDFSCluster {
"127.0.0.1:" + context.httpPort);
conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + suffix,
"0.0.0.0");
+ conf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY + "." + suffix,
+ "127.0.0.1:" + context.httpsPort);
// If the service port is enabled by default, we need to set them up
boolean servicePortEnabled = false;
@@ -543,7 +582,8 @@ public class MiniRouterDFSCluster {
return conf;
}
- public void configureNameservices(int numNameservices, int numNamenodes) {
+ public void configureNameservices(int numNameservices, int numNamenodes,
+ Configuration overrideConf) {
this.nameservices = new ArrayList<>();
this.namenodes = new ArrayList<>();
@@ -554,6 +594,10 @@ public class MiniRouterDFSCluster {
this.nameservices.add("ns" + i);
Configuration nnConf = generateNamenodeConfiguration(ns);
+ if (overrideConf != null) {
+ nnConf.addResource(overrideConf);
+ }
+
if (!highAvailability) {
context = new NamenodeContext(nnConf, ns, null, nnIndex++);
this.namenodes.add(context);
@@ -788,7 +832,7 @@ public class MiniRouterDFSCluster {
NamenodeStatusReport report = new NamenodeStatusReport(
nn.nameserviceId, nn.namenodeId,
nn.getRpcAddress(), nn.getServiceAddress(),
- nn.getLifelineAddress(), nn.getHttpAddress());
+ nn.getLifelineAddress(), nn.getWebAddress());
FSImage fsImage = nn.namenode.getNamesystem().getFSImage();
NamespaceInfo nsInfo = fsImage.getStorage().getNamespaceInfo();
report.setNamespaceInfo(nsInfo);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDFS-13834. RBF: Connection
creator thread should catch Throwable. Contributed by CR Hota.
Posted by br...@apache.org.
HDFS-13834. RBF: Connection creator thread should catch Throwable. Contributed by CR Hota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b262941f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b262941f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b262941f
Branch: refs/heads/HDFS-13891
Commit: b262941fd97abe9ab23dc174929daee82e480fce
Parents: cab0430
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Nov 14 18:35:12 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:34:52 2018 +0530
----------------------------------------------------------------------
.../federation/router/ConnectionManager.java | 4 +-
.../router/TestConnectionManager.java | 43 ++++++++++++++++++++
2 files changed, 46 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b262941f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
index 9fb83e4..fa2bf94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
@@ -393,7 +393,7 @@ public class ConnectionManager {
/**
* Thread that creates connections asynchronously.
*/
- private static class ConnectionCreator extends Thread {
+ static class ConnectionCreator extends Thread {
/** If the creator is running. */
private boolean running = true;
/** Queue to push work to. */
@@ -426,6 +426,8 @@ public class ConnectionManager {
} catch (InterruptedException e) {
LOG.error("The connection creator was interrupted");
this.running = false;
+ } catch (Throwable e) {
+ LOG.error("Fatal error caught by connection creator ", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b262941f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
index 0e1eb40..765f6c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
@@ -22,12 +22,17 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -49,6 +54,7 @@ public class TestConnectionManager {
private static final UserGroupInformation TEST_USER3 =
UserGroupInformation.createUserForTesting("user3", TEST_GROUP);
private static final String TEST_NN_ADDRESS = "nn1:8080";
+ private static final String UNRESOLVED_TEST_NN_ADDRESS = "unknownhost:8080";
@Before
public void setup() throws Exception {
@@ -59,6 +65,9 @@ public class TestConnectionManager {
connManager.start();
}
+ @Rule
+ public ExpectedException exceptionRule = ExpectedException.none();
+
@After
public void shutdown() {
if (connManager != null) {
@@ -122,6 +131,40 @@ public class TestConnectionManager {
}
@Test
+ public void testConnectionCreatorWithException() throws Exception {
+ // Create a bad connection pool pointing to unresolvable namenode address.
+ ConnectionPool badPool = new ConnectionPool(
+ conf, UNRESOLVED_TEST_NN_ADDRESS, TEST_USER1, 0, 10,
+ ClientProtocol.class);
+ BlockingQueue<ConnectionPool> queue = new ArrayBlockingQueue<>(1);
+ queue.add(badPool);
+ ConnectionManager.ConnectionCreator connectionCreator =
+ new ConnectionManager.ConnectionCreator(queue);
+ connectionCreator.setDaemon(true);
+ connectionCreator.start();
+ // Wait to make sure async thread is scheduled and picks
+ GenericTestUtils.waitFor(()->queue.isEmpty(), 50, 5000);
+ // At this point connection creation task should be definitely picked up.
+ assertTrue(queue.isEmpty());
+ // At this point connection thread should still be alive.
+ assertTrue(connectionCreator.isAlive());
+ // Stop the thread as test is successful at this point
+ connectionCreator.interrupt();
+ }
+
+ @Test
+ public void testGetConnectionWithException() throws Exception {
+ String exceptionCause = "java.net.UnknownHostException: unknownhost";
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage(exceptionCause);
+
+ // Create a bad connection pool pointing to unresolvable namenode address.
+ ConnectionPool badPool = new ConnectionPool(
+ conf, UNRESOLVED_TEST_NN_ADDRESS, TEST_USER1, 1, 10,
+ ClientProtocol.class);
+ }
+
+ @Test
public void testGetConnection() throws Exception {
Map<ConnectionPoolId, ConnectionPool> poolMap = connManager.getPools();
final int totalConns = 10;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: Add 2.9.2 release notes and
changes documents.
Posted by br...@apache.org.
Add 2.9.2 release notes and changes documents.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a00b4e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a00b4e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a00b4e3
Branch: refs/heads/HDFS-13891
Commit: 1a00b4e325146988375c9ce5b11016c45f059a4e
Parents: 5fb14e0
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 20 12:55:40 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 20 14:25:18 2018 +0900
----------------------------------------------------------------------
.../markdown/release/2.9.2/CHANGELOG.2.9.2.md | 80 ++++-
.../release/2.9.2/RELEASENOTES.2.9.2.md | 12 +-
.../jdiff/Apache_Hadoop_HDFS_2.9.2.xml | 312 +++++++++++++++++++
3 files changed, 399 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a00b4e3/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/CHANGELOG.2.9.2.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/CHANGELOG.2.9.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/CHANGELOG.2.9.2.md
index e0a167d..cf5e59b 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/CHANGELOG.2.9.2.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/CHANGELOG.2.9.2.md
@@ -16,10 +16,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-->
-# Apache Hadoop Changelog
+# "Apache Hadoop" Changelog
-## Release 2.9.2 - Unreleased (as of 2018-09-02)
+## Release 2.9.2 - 2018-11-19
+### INCOMPATIBLE CHANGES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+
+
+### IMPORTANT ISSUES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
### NEW FEATURES:
@@ -34,23 +44,28 @@
| JIRA | Summary | Priority | Component | Reporter | Contributor |
|:---- |:---- | :--- |:---- |:---- |:---- |
| [HADOOP-14987](https://issues.apache.org/jira/browse/HADOOP-14987) | Improve KMSClientProvider log around delegation token checking | Major | . | Xiaoyu Yao | Xiaoyu Yao |
+| [YARN-7274](https://issues.apache.org/jira/browse/YARN-7274) | Ability to disable elasticity at leaf queue level | Major | capacityscheduler | Scott Brokaw | Zian Chen |
| [HADOOP-15394](https://issues.apache.org/jira/browse/HADOOP-15394) | Backport PowerShell NodeFencer HADOOP-14309 to branch-2 | Minor | . | Íñigo Goiri | Íñigo Goiri |
| [HDFS-13462](https://issues.apache.org/jira/browse/HDFS-13462) | Add BIND\_HOST configuration for JournalNode's HTTP and RPC Servers | Major | hdfs, journal-node | Lukas Majercak | Lukas Majercak |
| [HADOOP-14841](https://issues.apache.org/jira/browse/HADOOP-14841) | Kms client should disconnect if unable to get output stream from connection. | Major | kms | Xiao Chen | Rushabh S Shah |
| [HDFS-13272](https://issues.apache.org/jira/browse/HDFS-13272) | DataNodeHttpServer to have configurable HttpServer2 threads | Major | datanode | Erik Krogen | Erik Krogen |
| [HADOOP-15441](https://issues.apache.org/jira/browse/HADOOP-15441) | Log kms url and token service at debug level. | Minor | . | Wei-Chiu Chuang | Gabor Bota |
-| [HDFS-13544](https://issues.apache.org/jira/browse/HDFS-13544) | Improve logging for JournalNode in federated cluster | Major | federation, hdfs | Hanisha Koneru | Hanisha Koneru |
| [HADOOP-15486](https://issues.apache.org/jira/browse/HADOOP-15486) | Make NetworkTopology#netLock fair | Major | net | Nanda kumar | Nanda kumar |
| [HADOOP-15449](https://issues.apache.org/jira/browse/HADOOP-15449) | Increase default timeout of ZK session to avoid frequent NameNode failover | Critical | common | Karthik Palanisamy | Karthik Palanisamy |
| [HDFS-13602](https://issues.apache.org/jira/browse/HDFS-13602) | Add checkOperation(WRITE) checks in FSNamesystem | Major | ha, namenode | Erik Krogen | Chao Sun |
| [HDFS-13653](https://issues.apache.org/jira/browse/HDFS-13653) | Make dfs.client.failover.random.order a per nameservice configuration | Major | federation | Ekanth Sethuramalingam | Ekanth Sethuramalingam |
+| [HDFS-13686](https://issues.apache.org/jira/browse/HDFS-13686) | Add overall metrics for FSNamesystemLock | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
| [HDFS-13714](https://issues.apache.org/jira/browse/HDFS-13714) | Fix TestNameNodePrunesMissingStorages test failures on Windows | Major | hdfs, namenode, test | Lukas Majercak | Lukas Majercak |
+| [HDFS-13719](https://issues.apache.org/jira/browse/HDFS-13719) | Docs around dfs.image.transfer.timeout are misleading | Major | documentation | Kitti Nanasi | Kitti Nanasi |
| [HDFS-11060](https://issues.apache.org/jira/browse/HDFS-11060) | make DEFAULT\_MAX\_CORRUPT\_FILEBLOCKS\_RETURNED configurable | Minor | hdfs | Lantao Jin | Lantao Jin |
| [HDFS-13813](https://issues.apache.org/jira/browse/HDFS-13813) | Exit NameNode if dangling child inode is detected when saving FsImage | Major | hdfs, namenode | Siyao Meng | Siyao Meng |
| [HDFS-13821](https://issues.apache.org/jira/browse/HDFS-13821) | RBF: Add dfs.federation.router.mount-table.cache.enable so that users can disable cache | Major | hdfs | Fei Hui | Fei Hui |
| [HADOOP-15689](https://issues.apache.org/jira/browse/HADOOP-15689) | Add "\*.patch" into .gitignore file of branch-2 | Major | . | Rui Gao | Rui Gao |
| [HDFS-13854](https://issues.apache.org/jira/browse/HDFS-13854) | RBF: The ProcessingAvgTime and ProxyAvgTime should display by JMX with ms unit. | Major | federation, hdfs | yanghuafeng | yanghuafeng |
| [YARN-8051](https://issues.apache.org/jira/browse/YARN-8051) | TestRMEmbeddedElector#testCallbackSynchronization is flakey | Major | test | Robert Kanter | Robert Kanter |
+| [HDFS-13857](https://issues.apache.org/jira/browse/HDFS-13857) | RBF: Choose to enable the default nameservice to read/write files | Major | federation, hdfs | yanghuafeng | yanghuafeng |
+| [HDFS-13812](https://issues.apache.org/jira/browse/HDFS-13812) | Fix the inconsistent default refresh interval on Caching documentation | Trivial | documentation | BELUGA BEHR | Hrishikesh Gadre |
+| [HDFS-13902](https://issues.apache.org/jira/browse/HDFS-13902) | Add JMX, conf and stacks menus to the datanode page | Minor | datanode | fengchuang | fengchuang |
### BUG FIXES:
@@ -58,6 +73,8 @@
| JIRA | Summary | Priority | Component | Reporter | Contributor |
|:---- |:---- | :--- |:---- |:---- |:---- |
| [HADOOP-15121](https://issues.apache.org/jira/browse/HADOOP-15121) | Encounter NullPointerException when using DecayRpcScheduler | Major | . | Tao Jie | Tao Jie |
+| [YARN-7765](https://issues.apache.org/jira/browse/YARN-7765) | [Atsv2] GSSException: No valid credentials provided - Failed to find any Kerberos tgt thrown by Timelinev2Client & HBaseClient in NM | Blocker | . | Sumana Sathish | Rohith Sharma K S |
+| [MAPREDUCE-7027](https://issues.apache.org/jira/browse/MAPREDUCE-7027) | HadoopArchiveLogs shouldn't delete the original logs if the HAR creation fails | Critical | harchive | Gergely Novák | Gergely Novák |
| [HDFS-10803](https://issues.apache.org/jira/browse/HDFS-10803) | TestBalancerWithMultipleNameNodes#testBalancing2OutOf3Blockpools fails intermittently due to no free space available | Major | . | Yiqun Lin | Yiqun Lin |
| [YARN-8068](https://issues.apache.org/jira/browse/YARN-8068) | Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ | Blocker | yarn | Sunil Govindan | Sunil Govindan |
| [HADOOP-15317](https://issues.apache.org/jira/browse/HADOOP-15317) | Improve NetworkTopology chooseRandom's loop | Major | . | Xiao Chen | Xiao Chen |
@@ -77,8 +94,10 @@
| [HDFS-13336](https://issues.apache.org/jira/browse/HDFS-13336) | Test cases of TestWriteToReplica failed in windows | Major | . | Xiao Liang | Xiao Liang |
| [HADOOP-15385](https://issues.apache.org/jira/browse/HADOOP-15385) | Many tests are failing in hadoop-distcp project in branch-2 | Critical | tools/distcp | Rushabh S Shah | Jason Lowe |
| [HDFS-13509](https://issues.apache.org/jira/browse/HDFS-13509) | Bug fix for breakHardlinks() of ReplicaInfo/LocalReplica, and fix TestFileAppend failures on Windows | Major | . | Xiao Liang | Xiao Liang |
+| [MAPREDUCE-7073](https://issues.apache.org/jira/browse/MAPREDUCE-7073) | Optimize TokenCache#obtainTokensForNamenodesInternal | Major | . | Bibin A Chundatt | Bibin A Chundatt |
| [YARN-8232](https://issues.apache.org/jira/browse/YARN-8232) | RMContainer lost queue name when RM HA happens | Major | resourcemanager | Hu Ziqian | Hu Ziqian |
| [HDFS-13537](https://issues.apache.org/jira/browse/HDFS-13537) | TestHdfsHelper does not generate jceks path properly for relative path in Windows | Major | . | Xiao Liang | Xiao Liang |
+| [HADOOP-15446](https://issues.apache.org/jira/browse/HADOOP-15446) | WASB: PageBlobInputStream.skip breaks HBASE replication | Major | fs/azure | Thomas Marquardt | Thomas Marquardt |
| [YARN-7003](https://issues.apache.org/jira/browse/YARN-7003) | DRAINING state of queues is not recovered after RM restart | Major | capacityscheduler | Tao Yang | Tao Yang |
| [YARN-8244](https://issues.apache.org/jira/browse/YARN-8244) | TestContainerSchedulerQueuing.testStartMultipleContainers failed | Major | . | Miklos Szegedi | Jim Brennan |
| [HDFS-13581](https://issues.apache.org/jira/browse/HDFS-13581) | DN UI logs link is broken when https is enabled | Minor | datanode | Namit Maheshwari | Shashikant Banerjee |
@@ -97,6 +116,7 @@
| [HDFS-13667](https://issues.apache.org/jira/browse/HDFS-13667) | Typo: Marking all "datandoes" as stale | Trivial | namenode | Wei-Chiu Chuang | Nanda kumar |
| [YARN-8405](https://issues.apache.org/jira/browse/YARN-8405) | RM zk-state-store.parent-path ACLs has been changed since HADOOP-14773 | Major | . | Rohith Sharma K S | Íñigo Goiri |
| [MAPREDUCE-7108](https://issues.apache.org/jira/browse/MAPREDUCE-7108) | TestFileOutputCommitter fails on Windows | Minor | test | Zuoming Zhang | Zuoming Zhang |
+| [YARN-8404](https://issues.apache.org/jira/browse/YARN-8404) | Timeline event publish need to be async to avoid Dispatcher thread leak in case ATS is down | Blocker | . | Rohith Sharma K S | Rohith Sharma K S |
| [HDFS-13675](https://issues.apache.org/jira/browse/HDFS-13675) | Speed up TestDFSAdminWithHA | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
| [HDFS-13673](https://issues.apache.org/jira/browse/HDFS-13673) | TestNameNodeMetrics fails on Windows | Minor | test | Zuoming Zhang | Zuoming Zhang |
| [HDFS-13676](https://issues.apache.org/jira/browse/HDFS-13676) | TestEditLogRace fails on Windows | Minor | test | Zuoming Zhang | Zuoming Zhang |
@@ -116,6 +136,7 @@
| [YARN-8577](https://issues.apache.org/jira/browse/YARN-8577) | Fix the broken anchor in SLS site-doc | Minor | documentation | Weiwei Yang | Weiwei Yang |
| [YARN-4606](https://issues.apache.org/jira/browse/YARN-4606) | CapacityScheduler: applications could get starved because computation of #activeUsers considers pending apps | Critical | capacity scheduler, capacityscheduler | Karam Singh | Manikandan R |
| [HADOOP-15637](https://issues.apache.org/jira/browse/HADOOP-15637) | LocalFs#listLocatedStatus does not filter out hidden .crc files | Minor | fs | Erik Krogen | Erik Krogen |
+| [HADOOP-15644](https://issues.apache.org/jira/browse/HADOOP-15644) | Hadoop Docker Image Pip Install Fails on branch-2 | Critical | build | Haibo Chen | Haibo Chen |
| [YARN-8331](https://issues.apache.org/jira/browse/YARN-8331) | Race condition in NM container launched after done | Major | . | Yang Wang | Pradeep Ambati |
| [HDFS-13758](https://issues.apache.org/jira/browse/HDFS-13758) | DatanodeManager should throw exception if it has BlockRecoveryCommand but the block is not under construction | Major | namenode | Wei-Chiu Chuang | chencan |
| [YARN-8612](https://issues.apache.org/jira/browse/YARN-8612) | Fix NM Collector Service Port issue in YarnConfiguration | Major | ATSv2 | Prabha Manepalli | Prabha Manepalli |
@@ -123,6 +144,40 @@
| [YARN-8640](https://issues.apache.org/jira/browse/YARN-8640) | Restore previous state in container-executor after failure | Major | . | Jim Brennan | Jim Brennan |
| [HADOOP-14314](https://issues.apache.org/jira/browse/HADOOP-14314) | The OpenSolaris taxonomy link is dead in InterfaceClassification.md | Major | documentation | Daniel Templeton | Rui Gao |
| [YARN-8649](https://issues.apache.org/jira/browse/YARN-8649) | NPE in localizer hearbeat processing if a container is killed while localizing | Major | . | lujie | lujie |
+| [HADOOP-10219](https://issues.apache.org/jira/browse/HADOOP-10219) | ipc.Client.setupIOstreams() needs to check for ClientCache.stopClient requested shutdowns | Major | ipc | Steve Loughran | Kihwal Lee |
+| [MAPREDUCE-7131](https://issues.apache.org/jira/browse/MAPREDUCE-7131) | Job History Server has race condition where it moves files from intermediate to finished but thinks file is in intermediate | Major | . | Anthony Hsu | Anthony Hsu |
+| [HDFS-13836](https://issues.apache.org/jira/browse/HDFS-13836) | RBF: Handle mount table znode with null value | Major | federation, hdfs | yanghuafeng | yanghuafeng |
+| [YARN-8709](https://issues.apache.org/jira/browse/YARN-8709) | CS preemption monitor always fails since one under-served queue was deleted | Major | capacityscheduler, scheduler preemption | Tao Yang | Tao Yang |
+| [HDFS-13051](https://issues.apache.org/jira/browse/HDFS-13051) | Fix dead lock during async editlog rolling if edit queue is full | Major | namenode | zhangwei | Daryn Sharp |
+| [YARN-8729](https://issues.apache.org/jira/browse/YARN-8729) | Node status updater thread could be lost after it is restarted | Critical | nodemanager | Tao Yang | Tao Yang |
+| [HDFS-13914](https://issues.apache.org/jira/browse/HDFS-13914) | Fix DN UI logs link broken when https is enabled after HDFS-13902 | Minor | datanode | Jianfei Jiang | Jianfei Jiang |
+| [MAPREDUCE-7133](https://issues.apache.org/jira/browse/MAPREDUCE-7133) | History Server task attempts REST API returns invalid data | Major | jobhistoryserver | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [YARN-8720](https://issues.apache.org/jira/browse/YARN-8720) | CapacityScheduler does not enforce max resource allocation check at queue level | Major | capacity scheduler, capacityscheduler, resourcemanager | Tarun Parimi | Tarun Parimi |
+| [HDFS-13844](https://issues.apache.org/jira/browse/HDFS-13844) | Fix the fmt\_bytes function in the dfs-dust.js | Minor | hdfs, ui | yanghuafeng | yanghuafeng |
+| [HADOOP-15755](https://issues.apache.org/jira/browse/HADOOP-15755) | StringUtils#createStartupShutdownMessage throws NPE when args is null | Major | . | Lokesh Jain | Dinesh Chitlangia |
+| [MAPREDUCE-3801](https://issues.apache.org/jira/browse/MAPREDUCE-3801) | org.apache.hadoop.mapreduce.v2.app.TestRuntimeEstimators.testExponentialEstimator fails intermittently | Major | mrv2 | Robert Joseph Evans | Jason Lowe |
+| [MAPREDUCE-7137](https://issues.apache.org/jira/browse/MAPREDUCE-7137) | MRAppBenchmark.benchmark1() fails with NullPointerException | Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [MAPREDUCE-7138](https://issues.apache.org/jira/browse/MAPREDUCE-7138) | ThrottledContainerAllocator in MRAppBenchmark should implement RMHeartbeatHandler | Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [HDFS-13908](https://issues.apache.org/jira/browse/HDFS-13908) | TestDataNodeMultipleRegistrations is flaky | Major | . | Íñigo Goiri | Ayush Saxena |
+| [YARN-8804](https://issues.apache.org/jira/browse/YARN-8804) | resourceLimits may be wrongly calculated when leaf-queue is blocked in cluster with 3+ level queues | Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-8774](https://issues.apache.org/jira/browse/YARN-8774) | Memory leak when CapacityScheduler allocates from reserved container with non-default label | Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [HADOOP-15817](https://issues.apache.org/jira/browse/HADOOP-15817) | Reuse Object Mapper in KMSJSONReader | Major | kms | Jonathan Eagles | Jonathan Eagles |
+| [HADOOP-15820](https://issues.apache.org/jira/browse/HADOOP-15820) | ZStandardDecompressor native code sets an integer field as a long | Blocker | . | Jason Lowe | Jason Lowe |
+| [HDFS-13964](https://issues.apache.org/jira/browse/HDFS-13964) | RBF: TestRouterWebHDFSContractAppend fails with No Active Namenode under nameservice | Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-15835](https://issues.apache.org/jira/browse/HADOOP-15835) | Reuse Object Mapper in KMSJSONWriter | Major | . | Jonathan Eagles | Jonathan Eagles |
+| [HDFS-13976](https://issues.apache.org/jira/browse/HDFS-13976) | Backport HDFS-12813 to branch-2.9 | Major | hdfs, hdfs-client | Lukas Majercak | Lukas Majercak |
+| [HADOOP-15679](https://issues.apache.org/jira/browse/HADOOP-15679) | ShutdownHookManager shutdown time needs to be configurable & extended | Major | util | Steve Loughran | Steve Loughran |
+| [HDFS-13802](https://issues.apache.org/jira/browse/HDFS-13802) | RBF: Remove FSCK from Router Web UI | Major | . | Fei Hui | Fei Hui |
+| [HADOOP-15859](https://issues.apache.org/jira/browse/HADOOP-15859) | ZStandardDecompressor.c mistakes a class for an instance | Blocker | . | Ben Lau | Jason Lowe |
+| [HADOOP-15850](https://issues.apache.org/jira/browse/HADOOP-15850) | CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0 | Critical | tools/distcp | Ted Yu | Ted Yu |
+| [YARN-7502](https://issues.apache.org/jira/browse/YARN-7502) | Nodemanager restart docs should describe nodemanager supervised property | Major | documentation | Jason Lowe | Suma Shivaprasad |
+| [HADOOP-15866](https://issues.apache.org/jira/browse/HADOOP-15866) | Renamed HADOOP\_SECURITY\_GROUP\_SHELL\_COMMAND\_TIMEOUT keys break compatibility | Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-15822](https://issues.apache.org/jira/browse/HADOOP-15822) | zstd compressor can fail with a small output buffer | Major | . | Jason Lowe | Jason Lowe |
+| [HADOOP-15899](https://issues.apache.org/jira/browse/HADOOP-15899) | Update AWS Java SDK versions in NOTICE.txt | Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15900](https://issues.apache.org/jira/browse/HADOOP-15900) | Update JSch versions in LICENSE.txt | Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-8858](https://issues.apache.org/jira/browse/YARN-8858) | CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. | Major | . | Sumana Sathish | Wangda Tan |
+| [YARN-8233](https://issues.apache.org/jira/browse/YARN-8233) | NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null | Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [HADOOP-15923](https://issues.apache.org/jira/browse/HADOOP-15923) | create-release script should set max-cache-ttl as well as default-cache-ttl for gpg-agent | Blocker | build | Akira Ajisaka | Akira Ajisaka |
### TESTS:
@@ -163,6 +218,7 @@
| JIRA | Summary | Priority | Component | Reporter | Contributor |
|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-13299](https://issues.apache.org/jira/browse/HDFS-13299) | RBF : Fix compilation error in branch-2 (TestMultipleDestinationResolver) | Blocker | . | Brahma Reddy Battula | Brahma Reddy Battula |
| [HDFS-13353](https://issues.apache.org/jira/browse/HDFS-13353) | RBF: TestRouterWebHDFSContractCreate failed | Major | test | Takanobu Asanuma | Takanobu Asanuma |
| [YARN-8110](https://issues.apache.org/jira/browse/YARN-8110) | AMRMProxy recover should catch for all throwable to avoid premature exit | Major | . | Botong Huang | Botong Huang |
| [HDFS-13402](https://issues.apache.org/jira/browse/HDFS-13402) | RBF: Fix java doc for StateStoreFileSystemImpl | Minor | hdfs | Yiran Wu | Yiran Wu |
@@ -171,7 +227,7 @@
| [HDFS-13045](https://issues.apache.org/jira/browse/HDFS-13045) | RBF: Improve error message returned from subcluster | Minor | . | Wei Yan | Íñigo Goiri |
| [HDFS-13428](https://issues.apache.org/jira/browse/HDFS-13428) | RBF: Remove LinkedList From StateStoreFileImpl.java | Trivial | federation | BELUGA BEHR | BELUGA BEHR |
| [HDFS-13386](https://issues.apache.org/jira/browse/HDFS-13386) | RBF: Wrong date information in list file(-ls) result | Minor | . | Dibyendu Karmakar | Dibyendu Karmakar |
-| [HADOOP-14999](https://issues.apache.org/jira/browse/HADOOP-14999) | AliyunOSS: provide one asynchronous multi-part based uploading mechanism | Major | fs/oss | Genmao Yu | Genmao Yu |
+| [YARN-7810](https://issues.apache.org/jira/browse/YARN-7810) | TestDockerContainerRuntime test failures due to UID lookup of a non-existent user | Major | . | Shane Kumpf | Shane Kumpf |
| [HDFS-13435](https://issues.apache.org/jira/browse/HDFS-13435) | RBF: Improve the error loggings for printing the stack trace | Major | . | Yiqun Lin | Yiqun Lin |
| [YARN-7189](https://issues.apache.org/jira/browse/YARN-7189) | Container-executor doesn't remove Docker containers that error out early | Major | yarn | Eric Badger | Eric Badger |
| [HDFS-13466](https://issues.apache.org/jira/browse/HDFS-13466) | RBF: Add more router-related information to the UI | Minor | . | Wei Yan | Wei Yan |
@@ -188,15 +244,31 @@
| [YARN-8253](https://issues.apache.org/jira/browse/YARN-8253) | HTTPS Ats v2 api call fails with "bad HTTP parsed" | Critical | ATSv2 | Yesha Vora | Charan Hebri |
| [HADOOP-15454](https://issues.apache.org/jira/browse/HADOOP-15454) | TestRollingFileSystemSinkWithLocal fails on Windows | Major | test | Xiao Liang | Xiao Liang |
| [HADOOP-15498](https://issues.apache.org/jira/browse/HADOOP-15498) | TestHadoopArchiveLogs (#testGenerateScript, #testPrepareWorkingDir) fails on Windows | Minor | . | Anbang Hu | Anbang Hu |
+| [HADOOP-15497](https://issues.apache.org/jira/browse/HADOOP-15497) | TestTrash should use proper test path to avoid failing on Windows | Minor | . | Anbang Hu | Anbang Hu |
| [HDFS-13637](https://issues.apache.org/jira/browse/HDFS-13637) | RBF: Router fails when threadIndex (in ConnectionPool) wraps around Integer.MIN\_VALUE | Critical | federation | CR Hota | CR Hota |
| [YARN-4781](https://issues.apache.org/jira/browse/YARN-4781) | Support intra-queue preemption for fairness ordering policy. | Major | scheduler | Wangda Tan | Eric Payne |
| [HDFS-13281](https://issues.apache.org/jira/browse/HDFS-13281) | Namenode#createFile should be /.reserved/raw/ aware. | Critical | encryption | Rushabh S Shah | Rushabh S Shah |
| [YARN-4677](https://issues.apache.org/jira/browse/YARN-4677) | RMNodeResourceUpdateEvent update from scheduler can lead to race condition | Major | graceful, resourcemanager, scheduler | Brook Zhou | Wilfred Spiegelenburg |
| [HADOOP-15529](https://issues.apache.org/jira/browse/HADOOP-15529) | ContainerLaunch#testInvalidEnvVariableSubstitutionType is not supported in Windows | Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
| [HADOOP-15458](https://issues.apache.org/jira/browse/HADOOP-15458) | TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows | Minor | test | Xiao Liang | Xiao Liang |
+| [YARN-8481](https://issues.apache.org/jira/browse/YARN-8481) | AMRMProxyPolicies should accept heartbeat response from new/unknown subclusters | Minor | amrmproxy, federation | Botong Huang | Botong Huang |
| [HDFS-13475](https://issues.apache.org/jira/browse/HDFS-13475) | RBF: Admin cannot enforce Router enter SafeMode | Major | . | Wei Yan | Chao Sun |
| [HDFS-13733](https://issues.apache.org/jira/browse/HDFS-13733) | RBF: Add Web UI configurations and descriptions to RBF document | Minor | documentation | Takanobu Asanuma | Takanobu Asanuma |
| [HDFS-13743](https://issues.apache.org/jira/browse/HDFS-13743) | RBF: Router throws NullPointerException due to the invalid initialization of MountTableResolver | Major | . | Takanobu Asanuma | Takanobu Asanuma |
| [HDFS-13750](https://issues.apache.org/jira/browse/HDFS-13750) | RBF: Router ID in RouterRpcClient is always null | Major | . | Takanobu Asanuma | Takanobu Asanuma |
| [HDFS-13848](https://issues.apache.org/jira/browse/HDFS-13848) | Refactor NameNode failover proxy providers | Major | ha, hdfs-client | Konstantin Shvachko | Konstantin Shvachko |
| [HADOOP-15699](https://issues.apache.org/jira/browse/HADOOP-15699) | Fix some of testContainerManager failures in Windows | Major | . | Botong Huang | Botong Huang |
+| [HADOOP-15731](https://issues.apache.org/jira/browse/HADOOP-15731) | TestDistributedShell fails on Windows | Major | . | Botong Huang | Botong Huang |
+| [HADOOP-15759](https://issues.apache.org/jira/browse/HADOOP-15759) | AliyunOSS: update oss-sdk version to 3.0.0 | Major | fs/oss | wujinhu | wujinhu |
+| [HADOOP-15671](https://issues.apache.org/jira/browse/HADOOP-15671) | AliyunOSS: Support Assume Roles in AliyunOSS | Major | fs/oss | wujinhu | wujinhu |
+| [HDFS-13790](https://issues.apache.org/jira/browse/HDFS-13790) | RBF: Move ClientProtocol APIs to its own module | Major | . | Íñigo Goiri | Chao Sun |
+| [HADOOP-15607](https://issues.apache.org/jira/browse/HADOOP-15607) | AliyunOSS: fix duplicated partNumber issue in AliyunOSSBlockOutputStream | Critical | . | wujinhu | wujinhu |
+| [HADOOP-15868](https://issues.apache.org/jira/browse/HADOOP-15868) | AliyunOSS: update document for properties of multiple part download, multiple part upload and directory copy | Major | fs/oss | wujinhu | wujinhu |
+
+
+### OTHER:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a00b4e3/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/RELEASENOTES.2.9.2.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/RELEASENOTES.2.9.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/RELEASENOTES.2.9.2.md
index 439933e..5faa4cf 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/RELEASENOTES.2.9.2.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/RELEASENOTES.2.9.2.md
@@ -16,6 +16,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-->
-# Apache Hadoop 2.9.2 Release Notes
+# "Apache Hadoop" 2.9.2 Release Notes
These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HADOOP-15446](https://issues.apache.org/jira/browse/HADOOP-15446) | *Major* | **WASB: PageBlobInputStream.skip breaks HBASE replication**
+
+WASB: Bug fix to support non-sequential page blob reads. Required for HBASE replication.
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a00b4e3/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.9.2.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.9.2.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.9.2.xml
new file mode 100644
index 0000000..8ca51fa
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.9.2.xml
@@ -0,0 +1,312 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Nov 13 16:20:19 UTC 2018 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="Apache Hadoop HDFS 2.9.2"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/classes:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.9.2.jar:/usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.9.2.jar:/maven/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar:/maven/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar:/maven/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/4.41.1/nimbus-jose-jwt-4.41.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/1.3.1/json-smart-1.3.1.jar:/maven/org/apache/directory/serv
er/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/maven/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/maven/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/maven/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/maven/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/maven/jline/jline/0.9.94/jline-0.9.94.jar:/maven/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-2.9.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/commons-net/commons-net/3.1/commons-net-3.1.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/org/mortbay/jetty/jetty-sslengine/6.1.26/jetty-sslengine-6.1.26.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/maven/org/codehaus
/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/maven/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/maven/javax/activation/activation/1.1/activation-1.1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/maven/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/maven/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/maven/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/maven/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/maven/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/maven/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoug
htworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/maven/com/jcraft/jsch/0.1.54/jsch-0.1.54.jar:/maven/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar:/maven/org/apache/curator/curator-recipes/2.7.1/curator-recipes-2.7.1.jar:/maven/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar:/maven/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/maven/org/tukaani/xz/1.0/xz-1.0.jar:/maven/org/codehaus/woodstox/stax2-api/3.1.4/stax2-api-3.1.4.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.0.3/woodstox-core-5.0.3.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-2.9.2.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/maven/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/maven/org/mortb
ay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/maven/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/maven/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/maven/asm/asm/3.2/asm-3.2.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/maven/commons-io/commons-io/2.4/commons-io-2.4.jar:/maven/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/log4j/log4j/1.2.17/log4j-1.2.17.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/maven/org/slf4j/slf4j-log4j12/1.7.25/slf4j-log4j12-1.7.25.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/xmlenc/xmlenc/0.52/xmlenc
-0.52.jar:/maven/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/maven/io/netty/netty-all/4.0.23.Final/netty-all-4.0.23.Final.jar:/maven/xerces/xercesImpl/2.9.1/xercesImpl-2.9.1.jar:/maven/xml-apis/xml-apis/1.3.04/xml-apis-1.3.04.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.7.8/jackson-databind-2.7.8.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.7.8/jackson-annotations-2.7.8.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.7.8/jackson-core-2.7.8.jar -sourcepath /build/source/hadoop-hdfs-project/hadoop-hdfs/src/main/java -apidir /build/source/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname Apache Hadoop HDFS 2.9.2 -->
+<package name="org.apache.hadoop.hdfs">
+ <doc>
+ <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}. This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files
+have strictly one writer at any one time. Bytes are always appended
+to the end of the writer's stream. There is no notion of "record appends"
+or "mutations" that are then checked or reordered. Writers simply emit
+a byte stream. That byte stream is guaranteed to be stored in the
+order written.</p>]]>
+ </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.net">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer.sasl">
+</package>
+<package name="org.apache.hadoop.hdfs.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.client">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.server">
+ <!-- start interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+ <interface name="JournalNodeMXBean" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getJournalsStatus" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get status information (e.g., whether formatted) of JournalNode's journals.
+
+ @return A string presenting status for each journal]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[This is the JMX management interface for JournalNode information]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.block">
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.delegation">
+</package>
+<package name="org.apache.hadoop.hdfs.server.balancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.blockmanagement">
+</package>
+<package name="org.apache.hadoop.hdfs.server.common">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset.impl">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web.webhdfs">
+</package>
+<package name="org.apache.hadoop.hdfs.server.mover">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode">
+ <!-- start interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+ <interface name="AuditLogger" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="initialize"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Called during initialization of the logger.
+
+ @param conf The configuration object.]]>
+ </doc>
+ </method>
+ <method name="logAuditEvent"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="succeeded" type="boolean"/>
+ <param name="userName" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetAddress"/>
+ <param name="cmd" type="java.lang.String"/>
+ <param name="src" type="java.lang.String"/>
+ <param name="dst" type="java.lang.String"/>
+ <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+ <doc>
+ <![CDATA[Called to log an audit event.
+ <p>
+ This method must return as quickly as possible, since it's called
+ in a critical section of the NameNode's operation.
+
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's
+ metadata (permissions, owner, times, etc).]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Interface defining an audit logger.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+ <!-- start class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+ <class name="HdfsAuditLogger" extends="java.lang.Object"
+ abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.hdfs.server.namenode.AuditLogger"/>
+ <constructor name="HdfsAuditLogger"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="logAuditEvent"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="succeeded" type="boolean"/>
+ <param name="userName" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetAddress"/>
+ <param name="cmd" type="java.lang.String"/>
+ <param name="src" type="java.lang.String"/>
+ <param name="dst" type="java.lang.String"/>
+ <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+ </method>
+ <method name="logAuditEvent"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="succeeded" type="boolean"/>
+ <param name="userName" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetAddress"/>
+ <param name="cmd" type="java.lang.String"/>
+ <param name="src" type="java.lang.String"/>
+ <param name="dst" type="java.lang.String"/>
+ <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+ <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+ <doc>
+ <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String, String,
+ FileStatus)} with additional parameters related to logging delegation token
+ tracking IDs.
+
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's metadata
+ (permissions, owner, times, etc).
+ @param callerContext Context information of the caller
+ @param ugi UserGroupInformation of the current user, or null if not logging
+ token tracking information
+ @param dtSecretManager The token secret manager, or null if not logging
+ token tracking information]]>
+ </doc>
+ </method>
+ <method name="logAuditEvent"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="succeeded" type="boolean"/>
+ <param name="userName" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetAddress"/>
+ <param name="cmd" type="java.lang.String"/>
+ <param name="src" type="java.lang.String"/>
+ <param name="dst" type="java.lang.String"/>
+ <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+ <doc>
+ <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String,
+ String, FileStatus, CallerContext, UserGroupInformation,
+ DelegationTokenSecretManager)} without {@link CallerContext} information.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Extension of {@link AuditLogger}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+ <!-- start class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+ <class name="INodeAttributeProvider" extends="java.lang.Object"
+ abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="INodeAttributeProvider"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="start"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Initialize the provider. This method is called at NameNode startup
+ time.]]>
+ </doc>
+ </method>
+ <method name="stop"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Shutdown the provider. This method is called at NameNode shutdown time.]]>
+ </doc>
+ </method>
+ <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="fullPath" type="java.lang.String"/>
+ <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+ </method>
+ <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="pathElements" type="java.lang.String[]"/>
+ <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+ </method>
+ <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="components" type="byte[][]"/>
+ <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+ </method>
+ <method name="getExternalAccessControlEnforcer" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="defaultEnforcer" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"/>
+ <doc>
+ <![CDATA[Can be over-ridden by implementations to provide a custom Access Control
+ Enforcer that can provide an alternate implementation of the
+ default permission checking logic.
+ @param defaultEnforcer The Default AccessControlEnforcer
+ @return The AccessControlEnforcer to use]]>
+ </doc>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.ha">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.window">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.web.resources">
+</package>
+<package name="org.apache.hadoop.hdfs.server.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.tools">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineEditsViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineImageViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.util">
+</package>
+<package name="org.apache.hadoop.hdfs.web">
+</package>
+<package name="org.apache.hadoop.hdfs.web.resources">
+</package>
+
+</api>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HDFS-14082. RBF: Add option to
fail operations when a subcluster is unavailable. Contributed by Inigo Goiri.
Posted by br...@apache.org.
HDFS-14082. RBF: Add option to fail operations when a subcluster is unavailable. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e34074b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e34074b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e34074b
Branch: refs/heads/HDFS-13891
Commit: 7e34074bae5c678d137923d7d103229192fa1c76
Parents: b262941
Author: Yiqun Lin <yq...@apache.org>
Authored: Wed Nov 21 10:40:26 2018 +0800
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:34:52 2018 +0530
----------------------------------------------------------------------
.../server/federation/router/RBFConfigKeys.java | 4 ++
.../federation/router/RouterClientProtocol.java | 15 +++--
.../federation/router/RouterRpcServer.java | 9 +++
.../src/main/resources/hdfs-rbf-default.xml | 10 ++++
.../router/TestRouterRpcMultiDestination.java | 59 ++++++++++++++++++++
5 files changed, 93 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e34074b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
index dd72e36..10018fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
@@ -125,6 +125,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
public static final String DFS_ROUTER_CLIENT_REJECT_OVERLOAD =
FEDERATION_ROUTER_PREFIX + "client.reject.overload";
public static final boolean DFS_ROUTER_CLIENT_REJECT_OVERLOAD_DEFAULT = false;
+ public static final String DFS_ROUTER_ALLOW_PARTIAL_LIST =
+ FEDERATION_ROUTER_PREFIX + "client.allow-partial-listing";
+ public static final boolean DFS_ROUTER_ALLOW_PARTIAL_LIST_DEFAULT = true;
+
// HDFS Router State Store connection
public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e34074b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
index de94eaf..2fc5358 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
@@ -111,6 +111,9 @@ public class RouterClientProtocol implements ClientProtocol {
private final FileSubclusterResolver subclusterResolver;
private final ActiveNamenodeResolver namenodeResolver;
+ /** If it requires response from all subclusters. */
+ private final boolean allowPartialList;
+
/** Identifier for the super user. */
private final String superUser;
/** Identifier for the super group. */
@@ -124,6 +127,10 @@ public class RouterClientProtocol implements ClientProtocol {
this.subclusterResolver = rpcServer.getSubclusterResolver();
this.namenodeResolver = rpcServer.getNamenodeResolver();
+ this.allowPartialList = conf.getBoolean(
+ RBFConfigKeys.DFS_ROUTER_ALLOW_PARTIAL_LIST,
+ RBFConfigKeys.DFS_ROUTER_ALLOW_PARTIAL_LIST_DEFAULT);
+
// User and group for reporting
this.superUser = System.getProperty("user.name");
this.superGroup = conf.get(
@@ -606,8 +613,8 @@ public class RouterClientProtocol implements ClientProtocol {
new Class<?>[] {String.class, startAfter.getClass(), boolean.class},
new RemoteParam(), startAfter, needLocation);
Map<RemoteLocation, DirectoryListing> listings =
- rpcClient.invokeConcurrent(
- locations, method, false, false, DirectoryListing.class);
+ rpcClient.invokeConcurrent(locations, method,
+ !this.allowPartialList, false, DirectoryListing.class);
Map<String, HdfsFileStatus> nnListing = new TreeMap<>();
int totalRemainingEntries = 0;
@@ -996,8 +1003,8 @@ public class RouterClientProtocol implements ClientProtocol {
RemoteMethod method = new RemoteMethod("getContentSummary",
new Class<?>[] {String.class}, new RemoteParam());
Map<RemoteLocation, ContentSummary> results =
- rpcClient.invokeConcurrent(
- locations, method, false, false, ContentSummary.class);
+ rpcClient.invokeConcurrent(locations, method,
+ !this.allowPartialList, false, ContentSummary.class);
summaries.addAll(results.values());
} catch (FileNotFoundException e) {
notFoundException = e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e34074b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 0f27efa..a30d0cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -1471,6 +1471,15 @@ public class RouterRpcServer extends AbstractService
}
/**
+ * Get ClientProtocol module implementation.
+ * @return ClientProtocol implementation
+ */
+ @VisibleForTesting
+ public RouterClientProtocol getClientProtocolModule() {
+ return this.clientProto;
+ }
+
+ /**
* Get RPC metrics info.
* @return The instance of FederationRPCMetrics.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e34074b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
index 53bf53a..09050bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
@@ -483,6 +483,16 @@
</property>
<property>
+ <name>dfs.federation.router.client.allow-partial-listing</name>
+ <value>true</value>
+ <description>
+ If the Router can return a partial list of files in a multi-destination mount point when one of the subclusters is unavailable.
+ True may return a partial list of files if a subcluster is down.
+ False will fail the request if one is unavailable.
+ </description>
+ </property>
+
+ <property>
<name>dfs.federation.router.keytab.file</name>
<value></value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e34074b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
index 94b712f..3101748 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
@@ -20,6 +20,13 @@ package org.apache.hadoop.hdfs.server.federation.router;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.internal.util.reflection.Whitebox.getInternalState;
+import static org.mockito.internal.util.reflection.Whitebox.setInternalState;
import java.io.IOException;
import java.lang.reflect.Method;
@@ -44,6 +51,13 @@ import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterConte
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Test;
/**
* The the RPC interface of the {@link getRouter()} implemented by
@@ -214,4 +228,49 @@ public class TestRouterRpcMultiDestination extends TestRouterRpc {
testRename(getRouterContext(), filename1, renamedFile, false);
testRename2(getRouterContext(), filename1, renamedFile, false);
}
+
+ @Test
+ public void testSubclusterDown() throws Exception {
+ final int totalFiles = 6;
+
+ List<RouterContext> routers = getCluster().getRouters();
+
+ // Test the behavior when everything is fine
+ FileSystem fs = getRouterFileSystem();
+ FileStatus[] files = fs.listStatus(new Path("/"));
+ assertEquals(totalFiles, files.length);
+
+ // Simulate one of the subclusters is in standby
+ NameNode nn0 = getCluster().getNamenode("ns0", null).getNamenode();
+ FSNamesystem ns0 = nn0.getNamesystem();
+ HAContext nn0haCtx = (HAContext)getInternalState(ns0, "haContext");
+ HAContext mockCtx = mock(HAContext.class);
+ doThrow(new StandbyException("Mock")).when(mockCtx).checkOperation(any());
+ setInternalState(ns0, "haContext", mockCtx);
+
+ // router0 should throw an exception
+ RouterContext router0 = routers.get(0);
+ RouterRpcServer router0RPCServer = router0.getRouter().getRpcServer();
+ RouterClientProtocol router0ClientProtocol =
+ router0RPCServer.getClientProtocolModule();
+ setInternalState(router0ClientProtocol, "allowPartialList", false);
+ try {
+ router0.getFileSystem().listStatus(new Path("/"));
+ fail("I should throw an exception");
+ } catch (RemoteException re) {
+ GenericTestUtils.assertExceptionContains(
+ "No namenode available to invoke getListing", re);
+ }
+
+ // router1 should report partial results
+ RouterContext router1 = routers.get(1);
+ files = router1.getFileSystem().listStatus(new Path("/"));
+ assertTrue("Found " + files.length + " items, we should have less",
+ files.length < totalFiles);
+
+
+ // Restore the HA context and the Router
+ setInternalState(ns0, "haContext", nn0haCtx);
+ setInternalState(router0ClientProtocol, "allowPartialList", true);
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HDDS-835. Use storageSize instead
of Long for buffer size configs in Ozone Client. Contributed by Shashikant
Banerjee.
Posted by br...@apache.org.
HDDS-835. Use storageSize instead of Long for buffer size configs in Ozone Client. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7478303
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7478303
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7478303
Branch: refs/heads/HDFS-13891
Commit: c74783036d8a2a1d2ce071a888d14e0ccff9263e
Parents: 10b5da8
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Tue Nov 20 23:01:02 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Tue Nov 20 23:01:02 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 3 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 9 +++---
.../apache/hadoop/ozone/OzoneConfigKeys.java | 21 ++++++-------
.../org/apache/hadoop/ozone/OzoneConsts.java | 5 ++++
.../main/java/org/apache/ratis/RatisHelper.java | 3 +-
.../common/src/main/resources/ozone-default.xml | 6 ++--
.../transport/server/XceiverServerGrpc.java | 5 ++--
.../server/ratis/XceiverServerRatis.java | 14 +++++----
.../replication/GrpcReplicationClient.java | 4 +--
.../ozone/client/io/ChunkGroupOutputStream.java | 7 ++---
.../hadoop/ozone/client/rpc/RpcClient.java | 31 +++++++++++---------
.../hadoop/ozone/MiniOzoneClusterImpl.java | 17 ++++++-----
.../rpc/TestCloseContainerHandlingByClient.java | 4 ++-
.../client/rpc/TestFailureHandlingByClient.java | 5 ----
.../om/TestMultipleContainerReadWrite.java | 5 ++--
.../web/storage/DistributedStorageHandler.java | 29 ++++++++++--------
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 9 +++---
.../ozone/web/ozShell/keys/GetKeyHandler.java | 7 +++--
.../ozone/web/ozShell/keys/PutKeyHandler.java | 6 ++--
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 4 ++-
20 files changed, 110 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index 5592c1d..a824c29 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.util.Time;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
@@ -103,7 +104,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
LOG.debug("Connecting to server Port : " + dn.getIpAddress());
ManagedChannel channel =
NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.build();
XceiverClientProtocolServiceStub asyncStub =
XceiverClientProtocolServiceGrpc.newStub(channel);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 896caed..f04d12f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -73,12 +73,12 @@ public final class ScmConfigKeys {
= 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
- public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
- 16 * 1024;
+ public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
+ "16KB";
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"dfs.container.ratis.segment.preallocated.size";
- public static final int
- DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
+ public static final String
+ DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "128MB";
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"dfs.container.ratis.statemachinedata.sync.timeout";
@@ -137,7 +137,6 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
// 16 MB by default
public static final int OZONE_SCM_CHUNK_SIZE_DEFAULT = 16 * 1024 * 1024;
- public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
public static final String OZONE_SCM_CLIENT_PORT_KEY =
"ozone.scm.client.port";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 8a5762f..8d5c180 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -95,9 +95,9 @@ public final class OzoneConfigKeys {
"ozone.container.cache.size";
public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024;
- public static final String OZONE_SCM_BLOCK_SIZE_IN_MB =
- "ozone.scm.block.size.in.mb";
- public static final long OZONE_SCM_BLOCK_SIZE_DEFAULT = 256;
+ public static final String OZONE_SCM_BLOCK_SIZE =
+ "ozone.scm.block.size";
+ public static final String OZONE_SCM_BLOCK_SIZE_DEFAULT = "256MB";
/**
* Ozone administrator users delimited by comma.
@@ -115,12 +115,14 @@ public final class OzoneConfigKeys {
public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE =
"ozone.client.stream.buffer.flush.size";
- public static final long OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT = 64;
+ public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT =
+ "64MB";
public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE =
"ozone.client.stream.buffer.max.size";
- public static final long OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT = 128;
+ public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT =
+ "128MB";
public static final String OZONE_CLIENT_WATCH_REQUEST_TIMEOUT =
"ozone.client.watch.request.timeout";
@@ -231,12 +233,13 @@ public final class OzoneConfigKeys {
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
- public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
+ public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
- public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
- = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
+ public static final String
+ DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
// config settings to enable stateMachineData write timeout
public static final String
@@ -246,8 +249,6 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
- public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
- = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index f8f31c8..7dd9fe3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -146,6 +146,11 @@ public final class OzoneConsts {
public static final String OM_S3_PREFIX ="S3:";
public static final String OM_S3_VOLUME_PREFIX = "s3";
+ /**
+ * Max chunk size limit.
+ */
+ public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
+
/**
* Max OM Quota size of 1024 PB.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
index 1ff7695..7e454d6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
@@ -155,7 +156,7 @@ public interface RatisHelper {
RaftConfigKeys.Rpc.setType(properties, rpcType);
GrpcConfigKeys.setMessageSizeMax(properties,
- SizeInBytes.valueOf(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE));
+ SizeInBytes.valueOf(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE));
return RaftClient.newBuilder()
.setRaftGroup(group)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 865335c..94e134d 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -623,11 +623,11 @@
</description>
</property>
<property>
- <name>ozone.scm.block.size.in.mb</name>
- <value>256</value>
+ <name>ozone.scm.block.size</name>
+ <value>256MB</value>
<tag>OZONE, SCM</tag>
<description>
- The default size of a scm block in bytes. This is maps to the default
+ The default size of a scm block. This is maps to the default
Ozone block size.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 992f3cb..be00c8a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.common.helpers.
StorageContainerException;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.ratis.thirdparty.io.grpc.BindableService;
@@ -90,12 +91,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
server = ((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher))
.build();
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher));
for (BindableService service : additionalServices) {
nettyServerBuilder.addService(service);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 434d330..3e54700 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
@@ -33,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server
@@ -154,23 +156,25 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftConfigKeys.Rpc.setType(properties, rpc);
// set raft segment size
- final int raftSegmentSize = conf.getInt(
+ final int raftSegmentSize = (int)conf.getStorageSize(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT,
+ StorageUnit.BYTES);
RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
SizeInBytes.valueOf(raftSegmentSize));
// set raft segment pre-allocated size
- final int raftSegmentPreallocatedSize = conf.getInt(
+ final int raftSegmentPreallocatedSize = (int) conf.getStorageSize(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT,
+ StorageUnit.BYTES);
RaftServerConfigKeys.Log.Appender.setBufferCapacity(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
RaftServerConfigKeys.Log.setPreallocatedSize(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
// Set max write buffer size, which is the scm chunk size
- final int maxChunkSize = OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE;
+ final int maxChunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
RaftServerConfigKeys.Log.setWriteBufferSize(properties,
SizeInBytes.valueOf(maxChunkSize));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 8149e2c..768d266 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -33,9 +33,9 @@ import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc;
import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc.IntraDatanodeProtocolServiceStub;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
@@ -61,7 +61,7 @@ public class GrpcReplicationClient {
channel = NettyChannelBuilder.forAddress(host, port)
.usePlaintext()
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.build();
client = IntraDatanodeProtocolServiceGrpc.newStub(channel);
this.workingDirectory = workingDir;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
index 5dbe9f6..d2a4443 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -159,9 +158,9 @@ public class ChunkGroupOutputStream extends OutputStream {
this.xceiverClientManager = xceiverClientManager;
this.chunkSize = chunkSize;
this.requestID = requestId;
- this.streamBufferFlushSize = bufferFlushSize * OzoneConsts.MB;
- this.streamBufferMaxSize = bufferMaxSize * OzoneConsts.MB;
- this.blockSize = size * OzoneConsts.MB;
+ this.streamBufferFlushSize = bufferFlushSize;
+ this.streamBufferMaxSize = bufferMaxSize;
+ this.blockSize = size;
this.watchTimeout = watchTimeout;
Preconditions.checkState(chunkSize > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 98481e1..3a0f475 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.client.rpc;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
@@ -139,26 +140,28 @@ public class RpcClient implements ClientProtocol {
this.xceiverClientManager = new XceiverClientManager(conf);
- int configuredChunkSize = conf.getInt(
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
- if(configuredChunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
+ int configuredChunkSize = (int) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
+ if(configuredChunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- configuredChunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
+ configuredChunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
} else {
chunkSize = configuredChunkSize;
}
- streamBufferFlushSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
- streamBufferMaxSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
- blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
+ streamBufferFlushSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ streamBufferMaxSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 324e17b..ceea3ee 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -24,6 +24,7 @@ import java.util.Optional;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -392,6 +393,7 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
Files.createDirectories(metaDir);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
if (!chunkSize.isPresent()) {
+ //set it to 1MB by default in tests
chunkSize = Optional.of(1);
}
if (!streamBufferFlushSize.isPresent()) {
@@ -403,13 +405,14 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
if (!blockSize.isPresent()) {
blockSize = Optional.of(2 * streamBufferMaxSize.get());
}
- conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- (int) (chunkSize.get() * OzoneConsts.MB));
- conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- streamBufferFlushSize.get());
- conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- streamBufferMaxSize.get());
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, blockSize.get());
+ conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ chunkSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ streamBufferFlushSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ streamBufferMaxSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, blockSize.get(),
+ StorageUnit.MB);
configureTrace();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index abd60a1..d268aeb 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.client.rpc;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -94,7 +95,8 @@ public class TestCloseContainerHandlingByClient {
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
+ StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(7).build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index dc6747f..923300f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -83,15 +82,11 @@ public class TestFailureHandlingByClient {
maxRetries = 100;
chunkSize = (int) OzoneConsts.MB;
blockSize = 4 * chunkSize;
- conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
- conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE, 1);
- conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE, 2);
conf.setTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT, 5,
TimeUnit.SECONDS);
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(6).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
index 3848cda..d0f4ce2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -69,8 +70,8 @@ public class TestMultipleContainerReadWrite {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- // set to as small as 100 bytes per block.
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 1);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 1,
+ StorageUnit.MB);
conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 5);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
index 6a433b5..fd10055 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.web.storage;
import com.google.common.base.Strings;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.io.IOUtils;
@@ -100,27 +101,29 @@ public final class DistributedStorageHandler implements StorageHandler {
this.storageContainerLocationClient = storageContainerLocation;
this.xceiverClientManager = new XceiverClientManager(conf);
- chunkSize = conf.getInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
+ chunkSize = (int)conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS,
OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT);
- if(chunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
+ if(chunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- chunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
+ chunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
}
- streamBufferFlushSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
- streamBufferMaxSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
- blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
+ streamBufferFlushSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ streamBufferMaxSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 98a27bf..a41dae6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -25,7 +26,6 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -59,7 +59,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVI
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
/**
* Implementation of keyManager.
@@ -87,8 +87,9 @@ public class KeyManagerImpl implements KeyManager {
String omId) {
this.scmBlockClient = scmBlockClient;
this.metadataManager = metadataManager;
- this.scmBlockSize = conf.getLong(OZONE_SCM_BLOCK_SIZE_IN_MB,
- OZONE_SCM_BLOCK_SIZE_DEFAULT) * OzoneConsts.MB;
+ this.scmBlockSize = (long) conf
+ .getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT,
+ StorageUnit.BYTES);
this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY,
DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
this.preallocateMax = conf.getLong(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
index be9342d..4e86699 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
@@ -24,6 +24,7 @@ import java.io.FileOutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -94,8 +95,10 @@ public class GetKeyHandler extends Handler {
OzoneInputStream keyInputStream = bucket.readKey(keyName);
if (dataFilePath != null) {
FileOutputStream outputStream = new FileOutputStream(dataFile);
- IOUtils.copyBytes(keyInputStream, outputStream, new OzoneConfiguration()
- .getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
+ IOUtils.copyBytes(keyInputStream, outputStream,
+ (int) new OzoneConfiguration()
+ .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
+ OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES));
outputStream.close();
} else {
throw new OzoneClientException(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
index b2ab68f..f86af3f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileInputStream;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -106,8 +107,9 @@ public class PutKeyHandler extends Handler {
.createKey(keyName, dataFile.length(), replicationType,
replicationFactor);
FileInputStream fileInputStream = new FileInputStream(dataFile);
- IOUtils.copyBytes(fileInputStream, outputStream,
- conf.getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
+ IOUtils.copyBytes(fileInputStream, outputStream, (int) conf
+ .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT,
+ StorageUnit.BYTES));
outputStream.close();
fileInputStream.close();
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
index d3bc857..913bb3e 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.ozone;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -64,7 +65,8 @@ public class TestOzoneFSInputStream {
@BeforeClass
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 10,
+ StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(10)
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDDS-718. Introduce new SCM
Commands to list and close Pipelines. Contributed by Lokesh Jain.
Posted by br...@apache.org.
HDDS-718. Introduce new SCM Commands to list and close Pipelines. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5d7b292
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5d7b292
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5d7b292
Branch: refs/heads/HDFS-13891
Commit: b5d7b292c988de6a8555d472a4448275522b7622
Parents: e7438a1
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Mon Nov 19 22:58:25 2018 +0530
Committer: Mukul Kumar Singh <ms...@apache.org>
Committed: Mon Nov 19 22:59:30 2018 +0530
----------------------------------------------------------------------
.../scm/client/ContainerOperationClient.java | 11 ++++
.../hadoop/hdds/scm/client/ScmClient.java | 16 ++++++
.../hadoop/hdds/scm/pipeline/Pipeline.java | 25 +++++----
.../StorageContainerLocationProtocol.java | 17 +++++++
...rLocationProtocolClientSideTranslatorPB.java | 33 ++++++++++++
...rLocationProtocolServerSideTranslatorPB.java | 37 ++++++++++++++
.../StorageContainerLocationProtocol.proto | 25 +++++++++
.../hdds/scm/pipeline/PipelineManager.java | 2 +
.../hdds/scm/pipeline/PipelineStateManager.java | 4 ++
.../hdds/scm/pipeline/PipelineStateMap.java | 8 +++
.../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 10 ++++
.../scm/server/SCMClientProtocolServer.java | 18 +++++++
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 6 ++-
.../cli/pipeline/ClosePipelineSubcommand.java | 53 ++++++++++++++++++++
.../cli/pipeline/ListPipelinesSubcommand.java | 48 ++++++++++++++++++
.../hdds/scm/cli/pipeline/package-info.java | 22 ++++++++
.../scm/pipeline/TestPipelineStateManager.java | 7 +++
18 files changed, 331 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
index ef72e38..85b5d29 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
@@ -225,6 +225,17 @@ public class ContainerOperationClient implements ScmClient {
}
@Override
+ public List<Pipeline> listPipelines() throws IOException {
+ return storageContainerLocationClient.listPipelines();
+ }
+
+ @Override
+ public void closePipeline(HddsProtos.PipelineID pipelineID)
+ throws IOException {
+ storageContainerLocationClient.closePipeline(pipelineID);
+ }
+
+ @Override
public void close() {
try {
xceiverClientManager.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index 6250adf..4f4239f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -173,6 +173,22 @@ public interface ScmClient extends Closeable {
throws IOException;
/**
+ * Returns the list of active Pipelines.
+ *
+ * @return list of Pipeline
+ * @throws IOException in case of any exception
+ */
+ List<Pipeline> listPipelines() throws IOException;
+
+ /**
+ * Closes the pipeline given a pipeline ID.
+ *
+ * @param pipelineID PipelineID to close.
+ * @throws IOException In case of exception while closing the pipeline
+ */
+ void closePipeline(HddsProtos.PipelineID pipelineID) throws IOException;
+
+ /**
* Check if SCM is in chill mode.
*
* @return Returns true if SCM is in chill mode else returns false.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
index 62081f4..a103bd7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@ -188,6 +188,20 @@ public final class Pipeline {
.toHashCode();
}
+ @Override
+ public String toString() {
+ final StringBuilder b =
+ new StringBuilder(getClass().getSimpleName()).append("[");
+ b.append(" Id: ").append(id.getId());
+ b.append(", Nodes: ");
+ nodeStatus.keySet().forEach(b::append);
+ b.append(", Type:").append(getType());
+ b.append(", Factor:").append(getFactor());
+ b.append(", State:").append(getPipelineState());
+ b.append("]");
+ return b.toString();
+ }
+
public static Builder newBuilder() {
return new Builder();
}
@@ -196,17 +210,6 @@ public final class Pipeline {
return new Builder(pipeline);
}
- @Override
- public String toString() {
- return "Pipeline{" +
- "id=" + id +
- ", type=" + type +
- ", factor=" + factor +
- ", state=" + state +
- ", nodeStatus=" + nodeStatus +
- '}';
- }
-
/**
* Builder class for Pipeline.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 14c55c3..c8a9dcf 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -127,6 +127,23 @@ public interface StorageContainerLocationProtocol {
throws IOException;
/**
+ * Returns the list of active Pipelines.
+ *
+ * @return list of Pipeline
+ *
+ * @throws IOException in case of any exception
+ */
+ List<Pipeline> listPipelines() throws IOException;
+
+ /**
+ * Closes a pipeline given the pipelineID.
+ *
+ * @param pipelineID ID of the pipeline to demolish
+ * @throws IOException
+ */
+ void closePipeline(HddsProtos.PipelineID pipelineID) throws IOException;
+
+ /**
* Returns information about SCM.
*
* @return {@link ScmInfo}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 3a4fa46..117e58d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -20,6 +20,9 @@ import com.google.common.base.Preconditions;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ClosePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitChillModeRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitChillModeResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineRequestProto;
@@ -64,6 +67,7 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Collectors;
/**
* This class is the client-side translator to translate the requests made on
@@ -305,6 +309,35 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
}
@Override
+ public List<Pipeline> listPipelines() throws IOException {
+ try {
+ ListPipelineRequestProto request = ListPipelineRequestProto
+ .newBuilder().build();
+ ListPipelineResponseProto response = rpcProxy.listPipelines(
+ NULL_RPC_CONTROLLER, request);
+ return response.getPipelinesList().stream()
+ .map(Pipeline::getFromProtobuf)
+ .collect(Collectors.toList());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public void closePipeline(HddsProtos.PipelineID pipelineID)
+ throws IOException {
+ try {
+ ClosePipelineRequestProto request =
+ ClosePipelineRequestProto.newBuilder()
+ .setPipelineID(pipelineID)
+ .build();
+ rpcProxy.closePipeline(NULL_RPC_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
public ScmInfo getScmInfo() throws IOException {
HddsProtos.GetScmInfoRequestProto request =
HddsProtos.GetScmInfoRequestProto.getDefaultInstance();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
index e2a4ee0..2ae559a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -44,6 +45,14 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerLocationProtocolProtos.ContainerResponseProto;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerLocationProtocolProtos.ClosePipelineRequestProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerLocationProtocolProtos.ClosePipelineResponseProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerLocationProtocolProtos.ListPipelineRequestProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerLocationProtocolProtos.ListPipelineResponseProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerLocationProtocolProtos.GetContainerRequestProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerLocationProtocolProtos.GetContainerResponseProto;
@@ -212,6 +221,34 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
}
@Override
+ public ListPipelineResponseProto listPipelines(
+ RpcController controller, ListPipelineRequestProto request)
+ throws ServiceException {
+ try {
+ ListPipelineResponseProto.Builder builder = ListPipelineResponseProto
+ .newBuilder();
+ List<Pipeline> pipelineIDs = impl.listPipelines();
+ pipelineIDs.stream().map(Pipeline::getProtobufMessage)
+ .forEach(builder::addPipelines);
+ return builder.build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public ClosePipelineResponseProto closePipeline(
+ RpcController controller, ClosePipelineRequestProto request)
+ throws ServiceException {
+ try {
+ impl.closePipeline(request.getPipelineID());
+ return ClosePipelineResponseProto.newBuilder().build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
public HddsProtos.GetScmInfoRespsonseProto getScmInfo(
RpcController controller, HddsProtos.GetScmInfoRequestProto req)
throws ServiceException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
index 71190ac..fe34fc0 100644
--- a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
@@ -149,6 +149,19 @@ message PipelineResponseProto {
optional string errorMessage = 3;
}
+message ListPipelineRequestProto {
+}
+
+message ListPipelineResponseProto {
+ repeated Pipeline pipelines = 1;
+}
+
+message ClosePipelineRequestProto {
+ required PipelineID pipelineID = 1;
+}
+
+message ClosePipelineResponseProto {
+}
message InChillModeRequestProto {
}
@@ -219,6 +232,18 @@ service StorageContainerLocationProtocolService {
returns (PipelineResponseProto);
/**
+ * Returns the list of Pipelines managed by SCM.
+ */
+ rpc listPipelines(ListPipelineRequestProto)
+ returns (ListPipelineResponseProto);
+
+ /**
+ * Closes a pipeline.
+ */
+ rpc closePipeline(ClosePipelineRequestProto)
+ returns (ClosePipelineResponseProto);
+
+ /**
* Returns information about SCM.
*/
rpc getScmInfo(GetScmInfoRequestProto)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
index cce09f3..47a6eac 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
@@ -41,6 +41,8 @@ public interface PipelineManager extends Closeable {
Pipeline getPipeline(PipelineID pipelineID) throws PipelineNotFoundException;
+ List<Pipeline> getPipelines();
+
List<Pipeline> getPipelines(ReplicationType type);
List<Pipeline> getPipelines(ReplicationType type,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
index 9f95378..1edb23a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
@@ -56,6 +56,10 @@ class PipelineStateManager {
return pipelineStateMap.getPipeline(pipelineID);
}
+ public List<Pipeline> getPipelines() {
+ return pipelineStateMap.getPipelines();
+ }
+
List<Pipeline> getPipelines(ReplicationType type) {
return pipelineStateMap.getPipelines(type);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
index 20dfa03..8a0ffbb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
@@ -115,6 +115,14 @@ class PipelineStateMap {
}
/**
+ * Get list of pipelines in SCM.
+ * @return List of pipelines
+ */
+ public List<Pipeline> getPipelines() {
+ return new ArrayList<>(pipelineMap.values());
+ }
+
+ /**
* Get pipeline corresponding to specified replication type.
*
* @param type - ReplicationType
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
index dd79962..6cf3abe 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
@@ -77,7 +77,7 @@ public final class RatisPipelineUtils {
* @param ozoneConf - Ozone configuration
* @throws IOException
*/
- static void destroyPipeline(PipelineManager pipelineManager,
+ public static void destroyPipeline(PipelineManager pipelineManager,
Pipeline pipeline, Configuration ozoneConf) throws IOException {
final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
LOG.debug("destroying pipeline:{} with {}", pipeline.getId(), group);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index cf1955d..382483f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -147,6 +147,16 @@ public class SCMPipelineManager implements PipelineManager {
}
@Override
+ public List<Pipeline> getPipelines() {
+ lock.readLock().lock();
+ try {
+ return stateManager.getPipelines();
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
public List<Pipeline> getPipelines(ReplicationType type) {
lock.readLock().lock();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index b59042e..d80d6e2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -42,6 +42,9 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.RatisPipelineUtils;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -307,6 +310,21 @@ public class SCMClientProtocolServer implements
}
@Override
+ public List<Pipeline> listPipelines() {
+ return scm.getPipelineManager().getPipelines();
+ }
+
+ @Override
+ public void closePipeline(HddsProtos.PipelineID pipelineID)
+ throws IOException {
+ PipelineManager pipelineManager = scm.getPipelineManager();
+ Pipeline pipeline =
+ pipelineManager.getPipeline(PipelineID.getFromProtobuf(pipelineID));
+ RatisPipelineUtils
+ .finalizeAndDestroyPipeline(pipelineManager, pipeline, conf, false);
+ }
+
+ @Override
public ScmInfo getScmInfo() throws IOException {
ScmInfo.Builder builder =
new ScmInfo.Builder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
index 31a36fb..e385f30 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.hdds.scm.cli.container.CreateSubcommand;
import org.apache.hadoop.hdds.scm.cli.container.DeleteSubcommand;
import org.apache.hadoop.hdds.scm.cli.container.InfoSubcommand;
import org.apache.hadoop.hdds.scm.cli.container.ListSubcommand;
+import org.apache.hadoop.hdds.scm.cli.pipeline.ClosePipelineSubcommand;
+import org.apache.hadoop.hdds.scm.cli.pipeline.ListPipelinesSubcommand;
import org.apache.hadoop.hdds.scm.client.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
@@ -78,7 +80,9 @@ import picocli.CommandLine.Option;
InfoSubcommand.class,
DeleteSubcommand.class,
CreateSubcommand.class,
- CloseSubcommand.class
+ CloseSubcommand.class,
+ ListPipelinesSubcommand.class,
+ ClosePipelineSubcommand.class
},
mixinStandardHelpOptions = true)
public class SCMCLI extends GenericCli {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ClosePipelineSubcommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ClosePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ClosePipelineSubcommand.java
new file mode 100644
index 0000000..d99823b
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ClosePipelineSubcommand.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.cli.pipeline;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.SCMCLI;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Handler of closePipeline command.
+ */
+@CommandLine.Command(
+ name = "closePipeline",
+ description = "Close pipeline",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class ClosePipelineSubcommand implements Callable<Void> {
+
+ @CommandLine.ParentCommand
+ private SCMCLI parent;
+
+ @CommandLine.Parameters(description = "ID of the pipeline to close")
+ private String pipelineId;
+
+ @Override
+ public Void call() throws Exception {
+ try (ScmClient scmClient = parent.createScmClient()) {
+ scmClient.closePipeline(
+ HddsProtos.PipelineID.newBuilder().setId(pipelineId).build());
+ return null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java
new file mode 100644
index 0000000..0f8cf28
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.cli.pipeline;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.cli.SCMCLI;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Handler of listPipelines command.
+ */
+@CommandLine.Command(
+ name = "listPipelines",
+ description = "List all active pipelines",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class ListPipelinesSubcommand implements Callable<Void> {
+
+ @CommandLine.ParentCommand
+ private SCMCLI parent;
+
+ @Override
+ public Void call() throws Exception {
+ try (ScmClient scmClient = parent.createScmClient()) {
+ scmClient.listPipelines().forEach(System.out::println);
+ return null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/package-info.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/package-info.java
new file mode 100644
index 0000000..64924d1
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains all of the pipeline related scm commands.
+ */
+package org.apache.hadoop.hdds.scm.cli.pipeline;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d7b292/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
index d404b84..823cd7d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
@@ -100,6 +100,9 @@ public class TestPipelineStateManager {
@Test
public void testGetPipelines() throws IOException {
+ // In start there should be no pipelines
+ Assert.assertTrue(stateManager.getPipelines().isEmpty());
+
Set<Pipeline> pipelines = new HashSet<>();
Pipeline pipeline = createDummyPipeline(1);
stateManager.addPipeline(pipeline);
@@ -113,6 +116,10 @@ public class TestPipelineStateManager {
Set<Pipeline> pipelines1 = new HashSet<>(stateManager.getPipelines(
HddsProtos.ReplicationType.RATIS));
Assert.assertEquals(pipelines1.size(), pipelines.size());
+
+ pipelines1 = new HashSet<>(stateManager.getPipelines());
+ Assert.assertEquals(pipelines1.size(), pipelines.size());
+
// clean up
for (Pipeline pipeline1 : pipelines) {
removePipeline(pipeline1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: Revert "HDDS-692. Use the
ProgressBar class in the RandomKeyGenerator freon test. Contributed by Zsolt
Horvath."
Posted by br...@apache.org.
Revert "HDDS-692. Use the ProgressBar class in the RandomKeyGenerator freon test. Contributed by Zsolt Horvath."
This reverts commit 2a7f4859912e83910f9a418f69ce6d4bd4a37815.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c946f1b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c946f1b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c946f1b1
Branch: refs/heads/HDFS-13891
Commit: c946f1b1211ca29a16828ee6824a9ccefa0b1150
Parents: 2a7f485
Author: Márton Elek <el...@apache.org>
Authored: Tue Nov 20 13:35:40 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Tue Nov 20 13:35:40 2018 +0100
----------------------------------------------------------------------
.../apache/hadoop/ozone/freon/ProgressBar.java | 227 ++++++++++++-------
.../hadoop/ozone/freon/RandomKeyGenerator.java | 87 +++++--
.../hadoop/ozone/freon/TestProgressBar.java | 71 ++++--
3 files changed, 270 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c946f1b1/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
index a987eea..a8d7e73 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
@@ -14,134 +14,197 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
-
package org.apache.hadoop.ozone.freon;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.PrintStream;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
import java.util.function.Supplier;
/**
- * Creates and runs a ProgressBar in new Thread which gets printed on
- * the provided PrintStream.
+ * Run an arbitrary code and print progress on the provided stream. The
+ * progressbar stops when: - the provided currentvalue is less the the maxvalue
+ * - exception thrown
*/
public class ProgressBar {
- private static final Logger LOG = LoggerFactory.getLogger(ProgressBar.class);
private static final long REFRESH_INTERVAL = 1000L;
- private final long maxValue;
- private final Supplier<Long> currentValue;
- private final Thread progressBar;
-
- private volatile boolean running;
-
- private volatile long startTime;
+ private PrintStream stream;
+ private AtomicLong currentValue;
+ private long maxValue;
+ private Thread progressBar;
+ private volatile boolean exception = false;
+ private long startTime;
/**
- * Creates a new ProgressBar instance which prints the progress on the given
- * PrintStream when started.
- *
- * @param stream to display the progress
+ * @param stream Used to display the progress
* @param maxValue Maximum value of the progress
- * @param currentValue Supplier that provides the current value
*/
- public ProgressBar(final PrintStream stream, final Long maxValue,
- final Supplier<Long> currentValue) {
+ ProgressBar(PrintStream stream, long maxValue) {
+ this.stream = stream;
this.maxValue = maxValue;
- this.currentValue = currentValue;
- this.progressBar = new Thread(getProgressBar(stream));
- this.running = false;
+ this.currentValue = new AtomicLong(0);
+ this.progressBar = new Thread(new ProgressBarThread());
}
/**
- * Starts the ProgressBar in a new Thread.
- * This is a non blocking call.
+ * Start a task with a progessbar without any in/out parameters Runnable used
+ * just a task wrapper.
+ *
+ * @param task Runnable
*/
- public synchronized void start() {
- if (!running) {
- running = true;
- startTime = System.nanoTime();
+ public void start(Runnable task) {
+
+ startTime = System.nanoTime();
+
+ try {
+
progressBar.start();
+ task.run();
+
+ } catch (Exception e) {
+ exception = true;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
}
}
/**
- * Graceful shutdown, waits for the progress bar to complete.
- * This is a blocking call.
+ * Start a task with only out parameters.
+ *
+ * @param task Supplier that represents the task
+ * @param <T> Generic return type
+ * @return Whatever the supllier produces
*/
- public synchronized void shutdown() {
- if (running) {
+ public <T> T start(Supplier<T> task) {
+
+ startTime = System.nanoTime();
+ T result = null;
+
+ try {
+
+ progressBar.start();
+ result = task.get();
+
+ } catch (Exception e) {
+ exception = true;
+ } finally {
+
try {
progressBar.join();
- running = false;
} catch (InterruptedException e) {
- LOG.warn("Got interrupted while waiting for the progress bar to " +
- "complete.");
+ e.printStackTrace();
}
+
+ return result;
}
}
/**
- * Terminates the progress bar. This doesn't wait for the progress bar
- * to complete.
+ * Start a task with in/out parameters.
+ *
+ * @param input Input of the function
+ * @param task A Function that does the task
+ * @param <T> type of the input
+ * @param <R> return type
+ * @return Whatever the Function returns
*/
- public synchronized void terminate() {
- if (running) {
+ public <T, R> R start(T input, Function<T, R> task) {
+
+ startTime = System.nanoTime();
+ R result = null;
+
+ try {
+
+ progressBar.start();
+ result = task.apply(input);
+
+ } catch (Exception e) {
+ exception = true;
+ throw e;
+ } finally {
+
try {
- running = false;
progressBar.join();
} catch (InterruptedException e) {
- LOG.warn("Got interrupted while waiting for the progress bar to " +
- "complete.");
+ e.printStackTrace();
}
+
+ return result;
}
}
- private Runnable getProgressBar(final PrintStream stream) {
- return () -> {
- stream.println();
- while (running && currentValue.get() < maxValue) {
- print(stream, currentValue.get());
- try {
+ /**
+ * Increment the progress with one step.
+ */
+ public void incrementProgress() {
+ currentValue.incrementAndGet();
+ }
+
+ private class ProgressBarThread implements Runnable {
+
+ @Override
+ public void run() {
+ try {
+
+ stream.println();
+ long value;
+
+ while ((value = currentValue.get()) < maxValue) {
+ print(value);
+
+ if (exception) {
+ break;
+ }
Thread.sleep(REFRESH_INTERVAL);
- } catch (InterruptedException e) {
- LOG.warn("ProgressBar was interrupted.");
}
- }
- print(stream, maxValue);
- stream.println();
- running = false;
- };
- }
- /**
- * Given current value prints the progress bar.
- *
- * @param value current progress position
- */
- private void print(final PrintStream stream, final long value) {
- stream.print('\r');
- double percent = 100.0 * value / maxValue;
- StringBuilder sb = new StringBuilder();
- sb.append(" " + String.format("%.2f", percent) + "% |");
-
- for (int i = 0; i <= percent; i++) {
- sb.append('█');
+ if (exception) {
+ stream.println();
+ stream.println("Incomplete termination, " + "check log for " +
+ "exception.");
+ } else {
+ print(maxValue);
+ }
+ stream.println();
+ } catch (InterruptedException e) {
+ stream.println(e);
+ }
}
- for (int j = 0; j < 100 - percent; j++) {
- sb.append(' ');
+
+ /**
+ * Given current value prints the progress bar.
+ *
+ * @param value current progress position
+ */
+ private void print(long value) {
+ stream.print('\r');
+ double percent = 100.0 * value / maxValue;
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + String.format("%.2f", percent) + "% |");
+
+ for (int i = 0; i <= percent; i++) {
+ sb.append('█');
+ }
+ for (int j = 0; j < 100 - percent; j++) {
+ sb.append(' ');
+ }
+ sb.append("| ");
+ sb.append(value + "/" + maxValue);
+ long timeInSec = TimeUnit.SECONDS.convert(
+ System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+ (timeInSec % 3600) / 60, timeInSec % 60);
+ sb.append(" Time: " + timeToPrint);
+ stream.print(sb.toString());
}
- sb.append("| ");
- sb.append(value + "/" + maxValue);
- long timeInSec = TimeUnit.SECONDS.convert(
- System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
- String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
- (timeInSec % 3600) / 60, timeInSec % 60);
- sb.append(" Time: " + timeToPrint);
- stream.print(sb.toString());
}
-}
\ No newline at end of file
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c946f1b1/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index ebaece28..184b075 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -185,7 +185,6 @@ public final class RandomKeyGenerator implements Callable<Void> {
private ArrayList<Histogram> histograms = new ArrayList<>();
private OzoneConfiguration ozoneConfiguration;
- private ProgressBar progressbar;
RandomKeyGenerator() {
}
@@ -252,26 +251,13 @@ public final class RandomKeyGenerator implements Callable<Void> {
validator.start();
LOG.info("Data validation is enabled.");
}
-
- Supplier<Long> currentValue;
- long maxValue;
-
- currentValue = () -> numberOfKeysAdded.get();
- maxValue = numOfVolumes *
- numOfBuckets *
- numOfKeys;
-
- progressbar = new ProgressBar(System.out, maxValue, currentValue);
-
+ Thread progressbar = getProgressBarThread();
LOG.info("Starting progress bar Thread.");
-
progressbar.start();
-
processor.shutdown();
processor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
-
- progressbar.shutdown();
-
+ completed = true;
+ progressbar.join();
if (validateWrites) {
validator.join();
}
@@ -910,6 +896,73 @@ public final class RandomKeyGenerator implements Callable<Void> {
}
}
+ private class ProgressBar implements Runnable {
+
+ private static final long REFRESH_INTERVAL = 1000L;
+
+ private PrintStream stream;
+ private Supplier<Long> currentValue;
+ private long maxValue;
+
+ ProgressBar(PrintStream stream, Supplier<Long> currentValue,
+ long maxValue) {
+ this.stream = stream;
+ this.currentValue = currentValue;
+ this.maxValue = maxValue;
+ }
+
+ @Override
+ public void run() {
+ try {
+ stream.println();
+ long value;
+ while ((value = currentValue.get()) < maxValue) {
+ print(value);
+ if (completed) {
+ break;
+ }
+ Thread.sleep(REFRESH_INTERVAL);
+ }
+ if (exception) {
+ stream.println();
+ stream.println("Incomplete termination, " +
+ "check log for exception.");
+ } else {
+ print(maxValue);
+ }
+ stream.println();
+ } catch (InterruptedException e) {
+ }
+ }
+
+ /**
+ * Given current value prints the progress bar.
+ *
+ * @param value
+ */
+ private void print(long value) {
+ stream.print('\r');
+ double percent = 100.0 * value / maxValue;
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + String.format("%.2f", percent) + "% |");
+
+ for (int i = 0; i <= percent; i++) {
+ sb.append('█');
+ }
+ for (int j = 0; j < 100 - percent; j++) {
+ sb.append(' ');
+ }
+ sb.append("| ");
+ sb.append(value + "/" + maxValue);
+ long timeInSec = TimeUnit.SECONDS.convert(
+ System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+ (timeInSec % 3600) / 60, timeInSec % 60);
+ sb.append(" Time: " + timeToPrint);
+ stream.print(sb);
+ }
+ }
+
/**
* Validates the write done in ozone cluster.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c946f1b1/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
index 90366da..cea7be8 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
@@ -22,15 +22,12 @@ import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import java.io.PrintStream;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
import java.util.function.Supplier;
-import java.util.stream.LongStream;
+import java.util.stream.IntStream;
import static org.mockito.Mockito.*;
-/**
- * Using Mockito runner.
- */
@RunWith(MockitoJUnitRunner.class)
/**
* Tests for the Progressbar class for Freon.
@@ -38,36 +35,78 @@ import static org.mockito.Mockito.*;
public class TestProgressBar {
private PrintStream stream;
- private AtomicLong numberOfKeysAdded;
- private Supplier<Long> currentValue;
@Before
public void setupMock() {
- numberOfKeysAdded = new AtomicLong(0L);
- currentValue = () -> numberOfKeysAdded.get();
stream = mock(PrintStream.class);
}
@Test
public void testWithRunnable() {
- Long maxValue = 10L;
+ int maxValue = 10;
- ProgressBar progressbar = new ProgressBar(stream, maxValue, currentValue);
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
Runnable task = () -> {
- LongStream.range(0, maxValue).forEach(
+ IntStream.range(0, maxValue).forEach(
counter -> {
- numberOfKeysAdded.getAndIncrement();
+ progressbar.incrementProgress();
}
);
};
- progressbar.start();
- task.run();
- progressbar.shutdown();
+ progressbar.start(task);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+ @Test
+ public void testWithSupplier() {
+
+ int maxValue = 10;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Supplier<Long> tasks = () -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+
+ return 1L; //return the result of the dummy task
+ };
+
+ progressbar.start(tasks);
verify(stream, atLeastOnce()).print(anyChar());
verify(stream, atLeastOnce()).print(anyString());
}
+
+ @Test
+ public void testWithFunction() {
+
+ int maxValue = 10;
+ Long result;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Function<Long, String> task = (Long l) -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+
+ return "dummy result"; //return the result of the dummy task
+ };
+
+ progressbar.start(1L, task);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-8860. Federation client
intercepter class contains unwanted character. Contributed by Abhishek Modi.
Posted by br...@apache.org.
YARN-8860. Federation client intercepter class contains unwanted character. Contributed by Abhishek Modi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abd6d48c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abd6d48c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abd6d48c
Branch: refs/heads/HDFS-13891
Commit: abd6d48c46342d2694a1cb56c40f208de797794c
Parents: 3420ddd
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Nov 16 10:23:06 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Nov 16 10:23:06 2018 -0800
----------------------------------------------------------------------
.../clientrm/FederationClientInterceptor.java | 60 ++++++++++----------
.../webapp/FederationInterceptorREST.java | 60 ++++++++++----------
2 files changed, 60 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/abd6d48c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
index 797b1aa..59cd266 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
@@ -304,19 +304,19 @@ public class FederationClientInterceptor
*
* Base scenarios:
*
- * The Client submits an application to the Router. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into
- * StateStore with the selected SubCluster (e.g. SC1) and the appId. • The
- * State Store replies with the selected SubCluster (e.g. SC1). • The Router
+ * The Client submits an application to the Router. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into
+ * StateStore with the selected SubCluster (e.g. SC1) and the appId. The
+ * State Store replies with the selected SubCluster (e.g. SC1). The Router
* submits the request to the selected SubCluster.
*
* In case of State Store failure:
*
- * The client submits an application to the Router. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into State
- * Store with the selected SubCluster (e.g. SC1) and the appId. • Due to the
+ * The client submits an application to the Router. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into State
+ * Store with the selected SubCluster (e.g. SC1) and the appId. Due to the
* State Store down the Router times out and it will retry depending on the
- * FederationFacade settings. • The Router replies to the client with an error
+ * FederationFacade settings. The Router replies to the client with an error
* message.
*
* If State Store fails after inserting the tuple: identical behavior as
@@ -326,26 +326,26 @@ public class FederationClientInterceptor
*
* Scenario 1 – Crash before submission to the ResourceManager
*
- * The Client submits an application to the Router. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into State
- * Store with the selected SubCluster (e.g. SC1) and the appId. • The Router
- * crashes. • The Client timeouts and resubmits the application. • The Router
- * selects one SubCluster to forward the request. • The Router inserts a tuple
- * into State Store with the selected SubCluster (e.g. SC2) and the appId. •
+ * The Client submits an application to the Router. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into State
+ * Store with the selected SubCluster (e.g. SC1) and the appId. The Router
+ * crashes. The Client timeouts and resubmits the application. The Router
+ * selects one SubCluster to forward the request. The Router inserts a tuple
+ * into State Store with the selected SubCluster (e.g. SC2) and the appId.
* Because the tuple is already inserted in the State Store, it returns the
- * previous selected SubCluster (e.g. SC1). • The Router submits the request
+ * previous selected SubCluster (e.g. SC1). The Router submits the request
* to the selected SubCluster (e.g. SC1).
*
* Scenario 2 – Crash after submission to the ResourceManager
*
- * • The Client submits an application to the Router. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into State
- * Store with the selected SubCluster (e.g. SC1) and the appId. • The Router
- * submits the request to the selected SubCluster. • The Router crashes. • The
- * Client timeouts and resubmit the application. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into State
- * Store with the selected SubCluster (e.g. SC2) and the appId. • The State
- * Store replies with the selected SubCluster (e.g. SC1). • The Router submits
+ * The Client submits an application to the Router. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into State
+ * Store with the selected SubCluster (e.g. SC1) and the appId. The Router
+ * submits the request to the selected SubCluster. The Router crashes. The
+ * Client timeouts and resubmit the application. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into State
+ * Store with the selected SubCluster (e.g. SC2) and the appId. The State
+ * Store replies with the selected SubCluster (e.g. SC1). The Router submits
* the request to the selected SubCluster (e.g. SC1). When a client re-submits
* the same application to the same RM, it does not raise an exception and
* replies with operation successful message.
@@ -354,14 +354,14 @@ public class FederationClientInterceptor
*
* In case of ResourceManager failure:
*
- * The Client submits an application to the Router. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into State
- * Store with the selected SubCluster (e.g. SC1) and the appId. • The Router
- * submits the request to the selected SubCluster. • The entire SubCluster is
- * down – all the RMs in HA or the master RM is not reachable. • The Router
- * times out. • The Router selects a new SubCluster to forward the request. •
+ * The Client submits an application to the Router. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into State
+ * Store with the selected SubCluster (e.g. SC1) and the appId. The Router
+ * submits the request to the selected SubCluster. The entire SubCluster is
+ * down – all the RMs in HA or the master RM is not reachable. The Router
+ * times out. The Router selects a new SubCluster to forward the request.
* The Router update a tuple into State Store with the selected SubCluster
- * (e.g. SC2) and the appId. • The State Store replies with OK answer. • The
+ * (e.g. SC2) and the appId. The State Store replies with OK answer. The
* Router submits the request to the selected SubCluster (e.g. SC2).
*/
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/abd6d48c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
index 40addc9..8e6b1c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
@@ -325,19 +325,19 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
* <p>
* Base scenarios:
* <p>
- * The Client submits an application to the Router. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into
- * StateStore with the selected SubCluster (e.g. SC1) and the appId. • The
- * State Store replies with the selected SubCluster (e.g. SC1). • The Router
+ * The Client submits an application to the Router. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into
+ * StateStore with the selected SubCluster (e.g. SC1) and the appId. The
+ * State Store replies with the selected SubCluster (e.g. SC1). The Router
* submits the request to the selected SubCluster.
* <p>
* In case of State Store failure:
* <p>
- * The client submits an application to the Router. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into State
- * Store with the selected SubCluster (e.g. SC1) and the appId. • Due to the
+ * The client submits an application to the Router. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into State
+ * Store with the selected SubCluster (e.g. SC1) and the appId. Due to the
* State Store down the Router times out and it will retry depending on the
- * FederationFacade settings. • The Router replies to the client with an error
+ * FederationFacade settings. The Router replies to the client with an error
* message.
* <p>
* If State Store fails after inserting the tuple: identical behavior as
@@ -347,26 +347,26 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
* <p>
* Scenario 1 – Crash before submission to the ResourceManager
* <p>
- * The Client submits an application to the Router. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into State
- * Store with the selected SubCluster (e.g. SC1) and the appId. • The Router
- * crashes. • The Client timeouts and resubmits the application. • The Router
- * selects one SubCluster to forward the request. • The Router inserts a tuple
- * into State Store with the selected SubCluster (e.g. SC2) and the appId. •
+ * The Client submits an application to the Router. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into State
+ * Store with the selected SubCluster (e.g. SC1) and the appId. The Router
+ * crashes. The Client timeouts and resubmits the application. The Router
+ * selects one SubCluster to forward the request. The Router inserts a tuple
+ * into State Store with the selected SubCluster (e.g. SC2) and the appId.
* Because the tuple is already inserted in the State Store, it returns the
- * previous selected SubCluster (e.g. SC1). • The Router submits the request
+ * previous selected SubCluster (e.g. SC1). The Router submits the request
* to the selected SubCluster (e.g. SC1).
* <p>
* Scenario 2 – Crash after submission to the ResourceManager
* <p>
- * • The Client submits an application to the Router. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into State
- * Store with the selected SubCluster (e.g. SC1) and the appId. • The Router
- * submits the request to the selected SubCluster. • The Router crashes. • The
- * Client timeouts and resubmit the application. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into State
- * Store with the selected SubCluster (e.g. SC2) and the appId. • The State
- * Store replies with the selected SubCluster (e.g. SC1). • The Router submits
+ * The Client submits an application to the Router. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into State
+ * Store with the selected SubCluster (e.g. SC1) and the appId. The Router
+ * submits the request to the selected SubCluster. The Router crashes. The
+ * Client timeouts and resubmit the application. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into State
+ * Store with the selected SubCluster (e.g. SC2) and the appId. The State
+ * Store replies with the selected SubCluster (e.g. SC1). The Router submits
* the request to the selected SubCluster (e.g. SC1). When a client re-submits
* the same application to the same RM, it does not raise an exception and
* replies with operation successful message.
@@ -375,14 +375,14 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
* <p>
* In case of ResourceManager failure:
* <p>
- * The Client submits an application to the Router. • The Router selects one
- * SubCluster to forward the request. • The Router inserts a tuple into State
- * Store with the selected SubCluster (e.g. SC1) and the appId. • The Router
- * submits the request to the selected SubCluster. • The entire SubCluster is
- * down – all the RMs in HA or the master RM is not reachable. • The Router
- * times out. • The Router selects a new SubCluster to forward the request. •
+ * The Client submits an application to the Router. The Router selects one
+ * SubCluster to forward the request. The Router inserts a tuple into State
+ * Store with the selected SubCluster (e.g. SC1) and the appId. The Router
+ * submits the request to the selected SubCluster. The entire SubCluster is
+ * down – all the RMs in HA or the master RM is not reachable. The Router
+ * times out. The Router selects a new SubCluster to forward the request.
* The Router update a tuple into State Store with the selected SubCluster
- * (e.g. SC2) and the appId. • The State Store replies with OK answer. • The
+ * (e.g. SC2) and the appId. The State Store replies with OK answer. The
* Router submits the request to the selected SubCluster (e.g. SC2).
*/
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDDS-692. Use the ProgressBar
class in the RandomKeyGenerator freon test. Contributed by Zsolt Horvath.
Posted by br...@apache.org.
HDDS-692. Use the ProgressBar class in the RandomKeyGenerator freon test. Contributed by Zsolt Horvath.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ecdcc96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ecdcc96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ecdcc96
Branch: refs/heads/HDFS-13891
Commit: 4ecdcc9620bd71833cee001aa08d0ceae4b9c59c
Parents: c946f1b
Author: Márton Elek <el...@apache.org>
Authored: Tue Nov 20 12:56:03 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Tue Nov 20 15:24:20 2018 +0100
----------------------------------------------------------------------
.../apache/hadoop/ozone/freon/ProgressBar.java | 227 +++++++------------
.../hadoop/ozone/freon/RandomKeyGenerator.java | 103 ++-------
.../hadoop/ozone/freon/TestProgressBar.java | 71 ++----
3 files changed, 115 insertions(+), 286 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ecdcc96/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
index a8d7e73..a987eea 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
@@ -14,197 +14,134 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
+
package org.apache.hadoop.ozone.freon;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.PrintStream;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Function;
import java.util.function.Supplier;
/**
- * Run an arbitrary code and print progress on the provided stream. The
- * progressbar stops when: - the provided currentvalue is less the the maxvalue
- * - exception thrown
+ * Creates and runs a ProgressBar in new Thread which gets printed on
+ * the provided PrintStream.
*/
public class ProgressBar {
+ private static final Logger LOG = LoggerFactory.getLogger(ProgressBar.class);
private static final long REFRESH_INTERVAL = 1000L;
- private PrintStream stream;
- private AtomicLong currentValue;
- private long maxValue;
- private Thread progressBar;
- private volatile boolean exception = false;
- private long startTime;
+ private final long maxValue;
+ private final Supplier<Long> currentValue;
+ private final Thread progressBar;
+
+ private volatile boolean running;
+
+ private volatile long startTime;
/**
- * @param stream Used to display the progress
+ * Creates a new ProgressBar instance which prints the progress on the given
+ * PrintStream when started.
+ *
+ * @param stream to display the progress
* @param maxValue Maximum value of the progress
+ * @param currentValue Supplier that provides the current value
*/
- ProgressBar(PrintStream stream, long maxValue) {
- this.stream = stream;
+ public ProgressBar(final PrintStream stream, final Long maxValue,
+ final Supplier<Long> currentValue) {
this.maxValue = maxValue;
- this.currentValue = new AtomicLong(0);
- this.progressBar = new Thread(new ProgressBarThread());
+ this.currentValue = currentValue;
+ this.progressBar = new Thread(getProgressBar(stream));
+ this.running = false;
}
/**
- * Start a task with a progessbar without any in/out parameters Runnable used
- * just a task wrapper.
- *
- * @param task Runnable
+ * Starts the ProgressBar in a new Thread.
+ * This is a non blocking call.
*/
- public void start(Runnable task) {
-
- startTime = System.nanoTime();
-
- try {
-
+ public synchronized void start() {
+ if (!running) {
+ running = true;
+ startTime = System.nanoTime();
progressBar.start();
- task.run();
-
- } catch (Exception e) {
- exception = true;
- } finally {
-
- try {
- progressBar.join();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
}
}
/**
- * Start a task with only out parameters.
- *
- * @param task Supplier that represents the task
- * @param <T> Generic return type
- * @return Whatever the supllier produces
+ * Graceful shutdown, waits for the progress bar to complete.
+ * This is a blocking call.
*/
- public <T> T start(Supplier<T> task) {
-
- startTime = System.nanoTime();
- T result = null;
-
- try {
-
- progressBar.start();
- result = task.get();
-
- } catch (Exception e) {
- exception = true;
- } finally {
-
+ public synchronized void shutdown() {
+ if (running) {
try {
progressBar.join();
+ running = false;
} catch (InterruptedException e) {
- e.printStackTrace();
+ LOG.warn("Got interrupted while waiting for the progress bar to " +
+ "complete.");
}
-
- return result;
}
}
/**
- * Start a task with in/out parameters.
- *
- * @param input Input of the function
- * @param task A Function that does the task
- * @param <T> type of the input
- * @param <R> return type
- * @return Whatever the Function returns
+ * Terminates the progress bar. This doesn't wait for the progress bar
+ * to complete.
*/
- public <T, R> R start(T input, Function<T, R> task) {
-
- startTime = System.nanoTime();
- R result = null;
-
- try {
-
- progressBar.start();
- result = task.apply(input);
-
- } catch (Exception e) {
- exception = true;
- throw e;
- } finally {
-
+ public synchronized void terminate() {
+ if (running) {
try {
+ running = false;
progressBar.join();
} catch (InterruptedException e) {
- e.printStackTrace();
+ LOG.warn("Got interrupted while waiting for the progress bar to " +
+ "complete.");
}
-
- return result;
}
}
- /**
- * Increment the progress with one step.
- */
- public void incrementProgress() {
- currentValue.incrementAndGet();
- }
-
- private class ProgressBarThread implements Runnable {
-
- @Override
- public void run() {
- try {
-
- stream.println();
- long value;
-
- while ((value = currentValue.get()) < maxValue) {
- print(value);
-
- if (exception) {
- break;
- }
+ private Runnable getProgressBar(final PrintStream stream) {
+ return () -> {
+ stream.println();
+ while (running && currentValue.get() < maxValue) {
+ print(stream, currentValue.get());
+ try {
Thread.sleep(REFRESH_INTERVAL);
+ } catch (InterruptedException e) {
+ LOG.warn("ProgressBar was interrupted.");
}
-
- if (exception) {
- stream.println();
- stream.println("Incomplete termination, " + "check log for " +
- "exception.");
- } else {
- print(maxValue);
- }
- stream.println();
- } catch (InterruptedException e) {
- stream.println(e);
}
- }
-
- /**
- * Given current value prints the progress bar.
- *
- * @param value current progress position
- */
- private void print(long value) {
- stream.print('\r');
- double percent = 100.0 * value / maxValue;
- StringBuilder sb = new StringBuilder();
- sb.append(" " + String.format("%.2f", percent) + "% |");
+ print(stream, maxValue);
+ stream.println();
+ running = false;
+ };
+ }
- for (int i = 0; i <= percent; i++) {
- sb.append('█');
- }
- for (int j = 0; j < 100 - percent; j++) {
- sb.append(' ');
- }
- sb.append("| ");
- sb.append(value + "/" + maxValue);
- long timeInSec = TimeUnit.SECONDS.convert(
- System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
- String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
- (timeInSec % 3600) / 60, timeInSec % 60);
- sb.append(" Time: " + timeToPrint);
- stream.print(sb.toString());
+ /**
+ * Given current value prints the progress bar.
+ *
+ * @param value current progress position
+ */
+ private void print(final PrintStream stream, final long value) {
+ stream.print('\r');
+ double percent = 100.0 * value / maxValue;
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + String.format("%.2f", percent) + "% |");
+
+ for (int i = 0; i <= percent; i++) {
+ sb.append('█');
+ }
+ for (int j = 0; j < 100 - percent; j++) {
+ sb.append(' ');
}
+ sb.append("| ");
+ sb.append(value + "/" + maxValue);
+ long timeInSec = TimeUnit.SECONDS.convert(
+ System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+ (timeInSec % 3600) / 60, timeInSec % 60);
+ sb.append(" Time: " + timeToPrint);
+ stream.print(sb.toString());
}
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ecdcc96/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index 184b075..5d72f557 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -185,6 +185,7 @@ public final class RandomKeyGenerator implements Callable<Void> {
private ArrayList<Histogram> histograms = new ArrayList<>();
private OzoneConfiguration ozoneConfiguration;
+ private ProgressBar progressbar;
RandomKeyGenerator() {
}
@@ -251,13 +252,26 @@ public final class RandomKeyGenerator implements Callable<Void> {
validator.start();
LOG.info("Data validation is enabled.");
}
- Thread progressbar = getProgressBarThread();
+
+ Supplier<Long> currentValue;
+ long maxValue;
+
+ currentValue = () -> numberOfKeysAdded.get();
+ maxValue = numOfVolumes *
+ numOfBuckets *
+ numOfKeys;
+
+ progressbar = new ProgressBar(System.out, maxValue, currentValue);
+
LOG.info("Starting progress bar Thread.");
+
progressbar.start();
+
processor.shutdown();
processor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
- completed = true;
- progressbar.join();
+
+ progressbar.shutdown();
+
if (validateWrites) {
validator.join();
}
@@ -280,22 +294,6 @@ public final class RandomKeyGenerator implements Callable<Void> {
Runtime.getRuntime().addShutdownHook(
new Thread(() -> printStats(System.out)));
}
-
- private Thread getProgressBarThread() {
- Supplier<Long> currentValue;
- long maxValue;
-
- currentValue = () -> numberOfKeysAdded.get();
- maxValue = (long) numOfVolumes *
- numOfBuckets *
- numOfKeys;
-
- Thread progressBarThread = new Thread(
- new ProgressBar(System.out, currentValue, maxValue));
- progressBarThread.setName("ProgressBar");
- return progressBarThread;
- }
-
/**
* Prints stats of {@link Freon} run to the PrintStream.
*
@@ -896,73 +894,6 @@ public final class RandomKeyGenerator implements Callable<Void> {
}
}
- private class ProgressBar implements Runnable {
-
- private static final long REFRESH_INTERVAL = 1000L;
-
- private PrintStream stream;
- private Supplier<Long> currentValue;
- private long maxValue;
-
- ProgressBar(PrintStream stream, Supplier<Long> currentValue,
- long maxValue) {
- this.stream = stream;
- this.currentValue = currentValue;
- this.maxValue = maxValue;
- }
-
- @Override
- public void run() {
- try {
- stream.println();
- long value;
- while ((value = currentValue.get()) < maxValue) {
- print(value);
- if (completed) {
- break;
- }
- Thread.sleep(REFRESH_INTERVAL);
- }
- if (exception) {
- stream.println();
- stream.println("Incomplete termination, " +
- "check log for exception.");
- } else {
- print(maxValue);
- }
- stream.println();
- } catch (InterruptedException e) {
- }
- }
-
- /**
- * Given current value prints the progress bar.
- *
- * @param value
- */
- private void print(long value) {
- stream.print('\r');
- double percent = 100.0 * value / maxValue;
- StringBuilder sb = new StringBuilder();
- sb.append(" " + String.format("%.2f", percent) + "% |");
-
- for (int i = 0; i <= percent; i++) {
- sb.append('█');
- }
- for (int j = 0; j < 100 - percent; j++) {
- sb.append(' ');
- }
- sb.append("| ");
- sb.append(value + "/" + maxValue);
- long timeInSec = TimeUnit.SECONDS.convert(
- System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
- String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
- (timeInSec % 3600) / 60, timeInSec % 60);
- sb.append(" Time: " + timeToPrint);
- stream.print(sb);
- }
- }
-
/**
* Validates the write done in ozone cluster.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ecdcc96/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
index cea7be8..90366da 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
@@ -22,12 +22,15 @@ import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import java.io.PrintStream;
-import java.util.function.Function;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
-import java.util.stream.IntStream;
+import java.util.stream.LongStream;
import static org.mockito.Mockito.*;
+/**
+ * Using Mockito runner.
+ */
@RunWith(MockitoJUnitRunner.class)
/**
* Tests for the Progressbar class for Freon.
@@ -35,78 +38,36 @@ import static org.mockito.Mockito.*;
public class TestProgressBar {
private PrintStream stream;
+ private AtomicLong numberOfKeysAdded;
+ private Supplier<Long> currentValue;
@Before
public void setupMock() {
+ numberOfKeysAdded = new AtomicLong(0L);
+ currentValue = () -> numberOfKeysAdded.get();
stream = mock(PrintStream.class);
}
@Test
public void testWithRunnable() {
- int maxValue = 10;
+ Long maxValue = 10L;
- ProgressBar progressbar = new ProgressBar(stream, maxValue);
+ ProgressBar progressbar = new ProgressBar(stream, maxValue, currentValue);
Runnable task = () -> {
- IntStream.range(0, maxValue).forEach(
+ LongStream.range(0, maxValue).forEach(
counter -> {
- progressbar.incrementProgress();
+ numberOfKeysAdded.getAndIncrement();
}
);
};
- progressbar.start(task);
-
- verify(stream, atLeastOnce()).print(anyChar());
- verify(stream, atLeastOnce()).print(anyString());
- }
-
- @Test
- public void testWithSupplier() {
-
- int maxValue = 10;
-
- ProgressBar progressbar = new ProgressBar(stream, maxValue);
-
- Supplier<Long> tasks = () -> {
- IntStream.range(0, maxValue).forEach(
- counter -> {
- progressbar.incrementProgress();
- }
- );
-
- return 1L; //return the result of the dummy task
- };
-
- progressbar.start(tasks);
+ progressbar.start();
+ task.run();
+ progressbar.shutdown();
verify(stream, atLeastOnce()).print(anyChar());
verify(stream, atLeastOnce()).print(anyString());
}
-
- @Test
- public void testWithFunction() {
-
- int maxValue = 10;
- Long result;
-
- ProgressBar progressbar = new ProgressBar(stream, maxValue);
-
- Function<Long, String> task = (Long l) -> {
- IntStream.range(0, maxValue).forEach(
- counter -> {
- progressbar.incrementProgress();
- }
- );
-
- return "dummy result"; //return the result of the dummy task
- };
-
- progressbar.start(1L, task);
-
- verify(stream, atLeastOnce()).print(anyChar());
- verify(stream, atLeastOnce()).print(anyString());
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDFS-13852. RBF: The
DN_REPORT_TIME_OUT and DN_REPORT_CACHE_EXPIRE should be configured in
RBFConfigKeys. Contributed by yanghuafeng.
Posted by br...@apache.org.
HDFS-13852. RBF: The DN_REPORT_TIME_OUT and DN_REPORT_CACHE_EXPIRE should be configured in RBFConfigKeys. Contributed by yanghuafeng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cab0430b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cab0430b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cab0430b
Branch: refs/heads/HDFS-13891
Commit: cab0430bfc2f208bc5f399d7d786e071dba26465
Parents: 6647f17
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Nov 13 10:14:35 2018 -0800
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:34:52 2018 +0530
----------------------------------------------------------------------
.../federation/metrics/FederationMetrics.java | 12 +++++++++--
.../federation/metrics/NamenodeBeanMetrics.java | 22 ++++----------------
.../server/federation/router/RBFConfigKeys.java | 7 +++++++
.../src/main/resources/hdfs-rbf-default.xml | 17 +++++++++++++++
.../router/TestRouterRPCClientRetries.java | 2 +-
.../server/federation/router/TestRouterRpc.java | 2 +-
6 files changed, 40 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cab0430b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
index 23f62b6..6a0a46e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
@@ -47,12 +47,14 @@ import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
import javax.management.StandardMBean;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
import org.apache.hadoop.hdfs.server.federation.router.Router;
import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
@@ -95,7 +97,7 @@ public class FederationMetrics implements FederationMBean {
private static final String DATE_FORMAT = "yyyy/MM/dd HH:mm:ss";
/** Prevent holding the page from load too long. */
- private static final long TIME_OUT = TimeUnit.SECONDS.toMillis(1);
+ private final long timeOut;
/** Router interface. */
@@ -143,6 +145,12 @@ public class FederationMetrics implements FederationMBean {
this.routerStore = stateStore.getRegisteredRecordStore(
RouterStore.class);
}
+
+ // Initialize the cache for the DN reports
+ Configuration conf = router.getConfig();
+ this.timeOut = conf.getTimeDuration(RBFConfigKeys.DN_REPORT_TIME_OUT,
+ RBFConfigKeys.DN_REPORT_TIME_OUT_MS_DEFAULT, TimeUnit.MILLISECONDS);
+
}
/**
@@ -434,7 +442,7 @@ public class FederationMetrics implements FederationMBean {
try {
RouterRpcServer rpcServer = this.router.getRpcServer();
DatanodeInfo[] live = rpcServer.getDatanodeReport(
- DatanodeReportType.LIVE, false, TIME_OUT);
+ DatanodeReportType.LIVE, false, timeOut);
if (live.length > 0) {
float totalDfsUsed = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cab0430b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
index e8ebf0d..da9a927 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
@@ -74,21 +74,6 @@ public class NamenodeBeanMetrics
private static final Logger LOG =
LoggerFactory.getLogger(NamenodeBeanMetrics.class);
- /** Prevent holding the page from loading too long. */
- private static final String DN_REPORT_TIME_OUT =
- RBFConfigKeys.FEDERATION_ROUTER_PREFIX + "dn-report.time-out";
- /** We only wait for 1 second. */
- private static final long DN_REPORT_TIME_OUT_DEFAULT =
- TimeUnit.SECONDS.toMillis(1);
-
- /** Time to cache the DN information. */
- public static final String DN_REPORT_CACHE_EXPIRE =
- RBFConfigKeys.FEDERATION_ROUTER_PREFIX + "dn-report.cache-expire";
- /** We cache the DN information for 10 seconds by default. */
- public static final long DN_REPORT_CACHE_EXPIRE_DEFAULT =
- TimeUnit.SECONDS.toMillis(10);
-
-
/** Instance of the Router being monitored. */
private final Router router;
@@ -148,10 +133,11 @@ public class NamenodeBeanMetrics
// Initialize the cache for the DN reports
Configuration conf = router.getConfig();
this.dnReportTimeOut = conf.getTimeDuration(
- DN_REPORT_TIME_OUT, DN_REPORT_TIME_OUT_DEFAULT, TimeUnit.MILLISECONDS);
+ RBFConfigKeys.DN_REPORT_TIME_OUT,
+ RBFConfigKeys.DN_REPORT_TIME_OUT_MS_DEFAULT, TimeUnit.MILLISECONDS);
long dnCacheExpire = conf.getTimeDuration(
- DN_REPORT_CACHE_EXPIRE,
- DN_REPORT_CACHE_EXPIRE_DEFAULT, TimeUnit.MILLISECONDS);
+ RBFConfigKeys.DN_REPORT_CACHE_EXPIRE,
+ RBFConfigKeys.DN_REPORT_CACHE_EXPIRE_MS_DEFAULT, TimeUnit.MILLISECONDS);
this.dnCache = CacheBuilder.newBuilder()
.expireAfterWrite(dnCacheExpire, TimeUnit.MILLISECONDS)
.build(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cab0430b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
index fa474f4..dd72e36 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
@@ -233,6 +233,13 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
FEDERATION_ROUTER_PREFIX + "https-bind-host";
public static final String DFS_ROUTER_HTTPS_ADDRESS_DEFAULT =
"0.0.0.0:" + DFS_ROUTER_HTTPS_PORT_DEFAULT;
+ public static final String DN_REPORT_TIME_OUT =
+ FEDERATION_ROUTER_PREFIX + "dn-report.time-out";
+ public static final long DN_REPORT_TIME_OUT_MS_DEFAULT = 1000;
+ public static final String DN_REPORT_CACHE_EXPIRE =
+ FEDERATION_ROUTER_PREFIX + "dn-report.cache-expire";
+ public static final long DN_REPORT_CACHE_EXPIRE_MS_DEFAULT =
+ TimeUnit.SECONDS.toMillis(10);
// HDFS Router-based federation quota
public static final String DFS_ROUTER_QUOTA_ENABLE =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cab0430b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
index 29c3093..53bf53a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
@@ -143,6 +143,23 @@
</description>
</property>
+
+ <property>
+ <name>dfs.federation.router.dn-report.time-out</name>
+ <value>1000</value>
+ <description>
+ Time out, in milliseconds for getDatanodeReport.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.dn-report.cache-expire</name>
+ <value>10s</value>
+ <description>
+ Expiration time in seconds for datanodereport.
+ </description>
+ </property>
+
<property>
<name>dfs.federation.router.metrics.class</name>
<value>org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cab0430b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
index e5ab3ab..f84e9a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
@@ -81,7 +81,7 @@ public class TestRouterRPCClientRetries {
.rpc()
.build();
routerConf.setTimeDuration(
- NamenodeBeanMetrics.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS);
+ RBFConfigKeys.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS);
// reduce IPC client connection retry times and interval time
Configuration clientConf = new Configuration(false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cab0430b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index a2a7b18..62303c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -175,7 +175,7 @@ public class TestRouterRpc {
.build();
// We decrease the DN cache times to make the test faster
routerConf.setTimeDuration(
- NamenodeBeanMetrics.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS);
+ RBFConfigKeys.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS);
cluster.addRouterOverrides(routerConf);
cluster.startRouters();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDDS-692. Use the ProgressBar
class in the RandomKeyGenerator freon test. Contributed by Zsolt Horvath.
Posted by br...@apache.org.
HDDS-692. Use the ProgressBar class in the RandomKeyGenerator freon test. Contributed by Zsolt Horvath.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a7f4859
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a7f4859
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a7f4859
Branch: refs/heads/HDFS-13891
Commit: 2a7f4859912e83910f9a418f69ce6d4bd4a37815
Parents: d0cc679
Author: Márton Elek <el...@apache.org>
Authored: Tue Nov 20 12:56:03 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Tue Nov 20 12:57:27 2018 +0100
----------------------------------------------------------------------
.../apache/hadoop/ozone/freon/ProgressBar.java | 227 +++++++------------
.../hadoop/ozone/freon/RandomKeyGenerator.java | 87 ++-----
.../hadoop/ozone/freon/TestProgressBar.java | 71 ++----
3 files changed, 115 insertions(+), 270 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a7f4859/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
index a8d7e73..a987eea 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
@@ -14,197 +14,134 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
+
package org.apache.hadoop.ozone.freon;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.PrintStream;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Function;
import java.util.function.Supplier;
/**
- * Run an arbitrary code and print progress on the provided stream. The
- * progressbar stops when: - the provided currentvalue is less the the maxvalue
- * - exception thrown
+ * Creates and runs a ProgressBar in new Thread which gets printed on
+ * the provided PrintStream.
*/
public class ProgressBar {
+ private static final Logger LOG = LoggerFactory.getLogger(ProgressBar.class);
private static final long REFRESH_INTERVAL = 1000L;
- private PrintStream stream;
- private AtomicLong currentValue;
- private long maxValue;
- private Thread progressBar;
- private volatile boolean exception = false;
- private long startTime;
+ private final long maxValue;
+ private final Supplier<Long> currentValue;
+ private final Thread progressBar;
+
+ private volatile boolean running;
+
+ private volatile long startTime;
/**
- * @param stream Used to display the progress
+ * Creates a new ProgressBar instance which prints the progress on the given
+ * PrintStream when started.
+ *
+ * @param stream to display the progress
* @param maxValue Maximum value of the progress
+ * @param currentValue Supplier that provides the current value
*/
- ProgressBar(PrintStream stream, long maxValue) {
- this.stream = stream;
+ public ProgressBar(final PrintStream stream, final Long maxValue,
+ final Supplier<Long> currentValue) {
this.maxValue = maxValue;
- this.currentValue = new AtomicLong(0);
- this.progressBar = new Thread(new ProgressBarThread());
+ this.currentValue = currentValue;
+ this.progressBar = new Thread(getProgressBar(stream));
+ this.running = false;
}
/**
- * Start a task with a progessbar without any in/out parameters Runnable used
- * just a task wrapper.
- *
- * @param task Runnable
+ * Starts the ProgressBar in a new Thread.
+ * This is a non blocking call.
*/
- public void start(Runnable task) {
-
- startTime = System.nanoTime();
-
- try {
-
+ public synchronized void start() {
+ if (!running) {
+ running = true;
+ startTime = System.nanoTime();
progressBar.start();
- task.run();
-
- } catch (Exception e) {
- exception = true;
- } finally {
-
- try {
- progressBar.join();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
}
}
/**
- * Start a task with only out parameters.
- *
- * @param task Supplier that represents the task
- * @param <T> Generic return type
- * @return Whatever the supllier produces
+ * Graceful shutdown, waits for the progress bar to complete.
+ * This is a blocking call.
*/
- public <T> T start(Supplier<T> task) {
-
- startTime = System.nanoTime();
- T result = null;
-
- try {
-
- progressBar.start();
- result = task.get();
-
- } catch (Exception e) {
- exception = true;
- } finally {
-
+ public synchronized void shutdown() {
+ if (running) {
try {
progressBar.join();
+ running = false;
} catch (InterruptedException e) {
- e.printStackTrace();
+ LOG.warn("Got interrupted while waiting for the progress bar to " +
+ "complete.");
}
-
- return result;
}
}
/**
- * Start a task with in/out parameters.
- *
- * @param input Input of the function
- * @param task A Function that does the task
- * @param <T> type of the input
- * @param <R> return type
- * @return Whatever the Function returns
+ * Terminates the progress bar. This doesn't wait for the progress bar
+ * to complete.
*/
- public <T, R> R start(T input, Function<T, R> task) {
-
- startTime = System.nanoTime();
- R result = null;
-
- try {
-
- progressBar.start();
- result = task.apply(input);
-
- } catch (Exception e) {
- exception = true;
- throw e;
- } finally {
-
+ public synchronized void terminate() {
+ if (running) {
try {
+ running = false;
progressBar.join();
} catch (InterruptedException e) {
- e.printStackTrace();
+ LOG.warn("Got interrupted while waiting for the progress bar to " +
+ "complete.");
}
-
- return result;
}
}
- /**
- * Increment the progress with one step.
- */
- public void incrementProgress() {
- currentValue.incrementAndGet();
- }
-
- private class ProgressBarThread implements Runnable {
-
- @Override
- public void run() {
- try {
-
- stream.println();
- long value;
-
- while ((value = currentValue.get()) < maxValue) {
- print(value);
-
- if (exception) {
- break;
- }
+ private Runnable getProgressBar(final PrintStream stream) {
+ return () -> {
+ stream.println();
+ while (running && currentValue.get() < maxValue) {
+ print(stream, currentValue.get());
+ try {
Thread.sleep(REFRESH_INTERVAL);
+ } catch (InterruptedException e) {
+ LOG.warn("ProgressBar was interrupted.");
}
-
- if (exception) {
- stream.println();
- stream.println("Incomplete termination, " + "check log for " +
- "exception.");
- } else {
- print(maxValue);
- }
- stream.println();
- } catch (InterruptedException e) {
- stream.println(e);
}
- }
-
- /**
- * Given current value prints the progress bar.
- *
- * @param value current progress position
- */
- private void print(long value) {
- stream.print('\r');
- double percent = 100.0 * value / maxValue;
- StringBuilder sb = new StringBuilder();
- sb.append(" " + String.format("%.2f", percent) + "% |");
+ print(stream, maxValue);
+ stream.println();
+ running = false;
+ };
+ }
- for (int i = 0; i <= percent; i++) {
- sb.append('█');
- }
- for (int j = 0; j < 100 - percent; j++) {
- sb.append(' ');
- }
- sb.append("| ");
- sb.append(value + "/" + maxValue);
- long timeInSec = TimeUnit.SECONDS.convert(
- System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
- String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
- (timeInSec % 3600) / 60, timeInSec % 60);
- sb.append(" Time: " + timeToPrint);
- stream.print(sb.toString());
+ /**
+ * Given current value prints the progress bar.
+ *
+ * @param value current progress position
+ */
+ private void print(final PrintStream stream, final long value) {
+ stream.print('\r');
+ double percent = 100.0 * value / maxValue;
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + String.format("%.2f", percent) + "% |");
+
+ for (int i = 0; i <= percent; i++) {
+ sb.append('█');
+ }
+ for (int j = 0; j < 100 - percent; j++) {
+ sb.append(' ');
}
+ sb.append("| ");
+ sb.append(value + "/" + maxValue);
+ long timeInSec = TimeUnit.SECONDS.convert(
+ System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+ (timeInSec % 3600) / 60, timeInSec % 60);
+ sb.append(" Time: " + timeToPrint);
+ stream.print(sb.toString());
}
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a7f4859/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index 184b075..ebaece28 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -185,6 +185,7 @@ public final class RandomKeyGenerator implements Callable<Void> {
private ArrayList<Histogram> histograms = new ArrayList<>();
private OzoneConfiguration ozoneConfiguration;
+ private ProgressBar progressbar;
RandomKeyGenerator() {
}
@@ -251,13 +252,26 @@ public final class RandomKeyGenerator implements Callable<Void> {
validator.start();
LOG.info("Data validation is enabled.");
}
- Thread progressbar = getProgressBarThread();
+
+ Supplier<Long> currentValue;
+ long maxValue;
+
+ currentValue = () -> numberOfKeysAdded.get();
+ maxValue = numOfVolumes *
+ numOfBuckets *
+ numOfKeys;
+
+ progressbar = new ProgressBar(System.out, maxValue, currentValue);
+
LOG.info("Starting progress bar Thread.");
+
progressbar.start();
+
processor.shutdown();
processor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
- completed = true;
- progressbar.join();
+
+ progressbar.shutdown();
+
if (validateWrites) {
validator.join();
}
@@ -896,73 +910,6 @@ public final class RandomKeyGenerator implements Callable<Void> {
}
}
- private class ProgressBar implements Runnable {
-
- private static final long REFRESH_INTERVAL = 1000L;
-
- private PrintStream stream;
- private Supplier<Long> currentValue;
- private long maxValue;
-
- ProgressBar(PrintStream stream, Supplier<Long> currentValue,
- long maxValue) {
- this.stream = stream;
- this.currentValue = currentValue;
- this.maxValue = maxValue;
- }
-
- @Override
- public void run() {
- try {
- stream.println();
- long value;
- while ((value = currentValue.get()) < maxValue) {
- print(value);
- if (completed) {
- break;
- }
- Thread.sleep(REFRESH_INTERVAL);
- }
- if (exception) {
- stream.println();
- stream.println("Incomplete termination, " +
- "check log for exception.");
- } else {
- print(maxValue);
- }
- stream.println();
- } catch (InterruptedException e) {
- }
- }
-
- /**
- * Given current value prints the progress bar.
- *
- * @param value
- */
- private void print(long value) {
- stream.print('\r');
- double percent = 100.0 * value / maxValue;
- StringBuilder sb = new StringBuilder();
- sb.append(" " + String.format("%.2f", percent) + "% |");
-
- for (int i = 0; i <= percent; i++) {
- sb.append('█');
- }
- for (int j = 0; j < 100 - percent; j++) {
- sb.append(' ');
- }
- sb.append("| ");
- sb.append(value + "/" + maxValue);
- long timeInSec = TimeUnit.SECONDS.convert(
- System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
- String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
- (timeInSec % 3600) / 60, timeInSec % 60);
- sb.append(" Time: " + timeToPrint);
- stream.print(sb);
- }
- }
-
/**
* Validates the write done in ozone cluster.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a7f4859/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
index cea7be8..90366da 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
@@ -22,12 +22,15 @@ import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import java.io.PrintStream;
-import java.util.function.Function;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
-import java.util.stream.IntStream;
+import java.util.stream.LongStream;
import static org.mockito.Mockito.*;
+/**
+ * Using Mockito runner.
+ */
@RunWith(MockitoJUnitRunner.class)
/**
* Tests for the Progressbar class for Freon.
@@ -35,78 +38,36 @@ import static org.mockito.Mockito.*;
public class TestProgressBar {
private PrintStream stream;
+ private AtomicLong numberOfKeysAdded;
+ private Supplier<Long> currentValue;
@Before
public void setupMock() {
+ numberOfKeysAdded = new AtomicLong(0L);
+ currentValue = () -> numberOfKeysAdded.get();
stream = mock(PrintStream.class);
}
@Test
public void testWithRunnable() {
- int maxValue = 10;
+ Long maxValue = 10L;
- ProgressBar progressbar = new ProgressBar(stream, maxValue);
+ ProgressBar progressbar = new ProgressBar(stream, maxValue, currentValue);
Runnable task = () -> {
- IntStream.range(0, maxValue).forEach(
+ LongStream.range(0, maxValue).forEach(
counter -> {
- progressbar.incrementProgress();
+ numberOfKeysAdded.getAndIncrement();
}
);
};
- progressbar.start(task);
-
- verify(stream, atLeastOnce()).print(anyChar());
- verify(stream, atLeastOnce()).print(anyString());
- }
-
- @Test
- public void testWithSupplier() {
-
- int maxValue = 10;
-
- ProgressBar progressbar = new ProgressBar(stream, maxValue);
-
- Supplier<Long> tasks = () -> {
- IntStream.range(0, maxValue).forEach(
- counter -> {
- progressbar.incrementProgress();
- }
- );
-
- return 1L; //return the result of the dummy task
- };
-
- progressbar.start(tasks);
+ progressbar.start();
+ task.run();
+ progressbar.shutdown();
verify(stream, atLeastOnce()).print(anyChar());
verify(stream, atLeastOnce()).print(anyString());
}
-
- @Test
- public void testWithFunction() {
-
- int maxValue = 10;
- Long result;
-
- ProgressBar progressbar = new ProgressBar(stream, maxValue);
-
- Function<Long, String> task = (Long l) -> {
- IntStream.range(0, maxValue).forEach(
- counter -> {
- progressbar.incrementProgress();
- }
- );
-
- return "dummy result"; //return the result of the dummy task
- };
-
- progressbar.start(1L, task);
-
- verify(stream, atLeastOnce()).print(anyChar());
- verify(stream, atLeastOnce()).print(anyString());
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-8953. [CSI] CSI driver
adaptor module support in NodeManager. Contributed by Weiwei Yang.
Posted by br...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java
new file mode 100644
index 0000000..128240d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.csi.adaptor;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import csi.v0.Csi;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
+import org.apache.hadoop.yarn.api.impl.pb.client.CsiAdaptorProtocolPBClientImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesRequestPBImpl;
+import org.apache.hadoop.yarn.client.NMProxy;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.csi.client.CsiClient;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+
+import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.AccessMode.MULTI_NODE_MULTI_WRITER;
+import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeType.FILE_SYSTEM;
+
+/**
+ * UT for {@link CsiAdaptorProtocolService}.
+ */
+public class TestCsiAdaptorService {
+
+ private static File testRoot = null;
+ private static String domainSocket = null;
+
+ @BeforeClass
+ public static void setUp() throws IOException {
+ testRoot = GenericTestUtils.getTestDir("csi-test");
+ File socketPath = new File(testRoot, "csi.sock");
+ FileUtils.forceMkdirParent(socketPath);
+ domainSocket = "unix://" + socketPath.getAbsolutePath();
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ if (testRoot != null) {
+ FileUtils.deleteDirectory(testRoot);
+ }
+ }
+
+ @Test
+ public void testValidateVolume() throws IOException, YarnException {
+ ServerSocket ss = new ServerSocket(0);
+ ss.close();
+ InetSocketAddress address = new InetSocketAddress(ss.getLocalPort());
+ Configuration conf = new Configuration();
+ conf.setSocketAddr(
+ YarnConfiguration.NM_CSI_ADAPTOR_PREFIX + "test-driver.address",
+ address);
+ CsiAdaptorProtocolService service =
+ new CsiAdaptorProtocolService("test-driver", domainSocket);
+
+ // inject a fake CSI client
+ // this client validates if the ValidateVolumeCapabilitiesRequest
+ // is integrity, and then reply a fake response
+ service.setCsiClient(new CsiClient() {
+ @Override
+ public Csi.GetPluginInfoResponse getPluginInfo() {
+ return Csi.GetPluginInfoResponse.newBuilder()
+ .setName("test-plugin")
+ .setVendorVersion("0.1")
+ .build();
+ }
+
+ @Override
+ public Csi.ValidateVolumeCapabilitiesResponse validateVolumeCapabilities(
+ Csi.ValidateVolumeCapabilitiesRequest request) {
+ // validate we get all info from the request
+ Assert.assertEquals("volume-id-0000123", request.getVolumeId());
+ Assert.assertEquals(1, request.getVolumeCapabilitiesCount());
+ Assert.assertEquals(Csi.VolumeCapability.AccessMode
+ .newBuilder().setModeValue(5).build(),
+ request.getVolumeCapabilities(0).getAccessMode());
+ Assert.assertTrue(request.getVolumeCapabilities(0).hasMount());
+ Assert.assertEquals(2, request.getVolumeCapabilities(0)
+ .getMount().getMountFlagsCount());
+ Assert.assertTrue(request.getVolumeCapabilities(0)
+ .getMount().getMountFlagsList().contains("mountFlag1"));
+ Assert.assertTrue(request.getVolumeCapabilities(0)
+ .getMount().getMountFlagsList().contains("mountFlag2"));
+ Assert.assertEquals(2, request.getVolumeAttributesCount());
+ Assert.assertEquals("v1", request.getVolumeAttributesMap().get("k1"));
+ Assert.assertEquals("v2", request.getVolumeAttributesMap().get("k2"));
+ // return a fake result
+ return Csi.ValidateVolumeCapabilitiesResponse.newBuilder()
+ .setSupported(false)
+ .setMessage("this is a test")
+ .build();
+ }
+ });
+
+ service.init(conf);
+ service.start();
+
+ try (CsiAdaptorProtocolPBClientImpl client =
+ new CsiAdaptorProtocolPBClientImpl(1L, address, new Configuration())) {
+ ValidateVolumeCapabilitiesRequest request =
+ ValidateVolumeCapabilitiesRequestPBImpl
+ .newInstance("volume-id-0000123",
+ ImmutableList.of(
+ new ValidateVolumeCapabilitiesRequest
+ .VolumeCapability(
+ MULTI_NODE_MULTI_WRITER, FILE_SYSTEM,
+ ImmutableList.of("mountFlag1", "mountFlag2"))),
+ ImmutableMap.of("k1", "v1", "k2", "v2"));
+
+ ValidateVolumeCapabilitiesResponse response = client
+ .validateVolumeCapacity(request);
+
+ Assert.assertEquals(false, response.isSupported());
+ Assert.assertEquals("this is a test", response.getResponseMessage());
+ } finally {
+ service.stop();
+ }
+ }
+
+ @Test
+ public void testValidateVolumeWithNMProxy() throws Exception {
+ ServerSocket ss = new ServerSocket(0);
+ ss.close();
+ InetSocketAddress address = new InetSocketAddress(ss.getLocalPort());
+ Configuration conf = new Configuration();
+ conf.setSocketAddr(
+ YarnConfiguration.NM_CSI_ADAPTOR_PREFIX + "test-driver.address",
+ address);
+ CsiAdaptorProtocolService service =
+ new CsiAdaptorProtocolService("test-driver", domainSocket);
+
+ // inject a fake CSI client
+ // this client validates if the ValidateVolumeCapabilitiesRequest
+ // is integrity, and then reply a fake response
+ service.setCsiClient(new CsiClient() {
+ @Override
+ public Csi.GetPluginInfoResponse getPluginInfo() {
+ return Csi.GetPluginInfoResponse.newBuilder()
+ .setName("test-plugin")
+ .setVendorVersion("0.1")
+ .build();
+ }
+
+ @Override
+ public Csi.ValidateVolumeCapabilitiesResponse validateVolumeCapabilities(
+ Csi.ValidateVolumeCapabilitiesRequest request) {
+ // validate we get all info from the request
+ Assert.assertEquals("volume-id-0000123", request.getVolumeId());
+ Assert.assertEquals(1, request.getVolumeCapabilitiesCount());
+ Assert.assertEquals(Csi.VolumeCapability.AccessMode
+ .newBuilder().setModeValue(5).build(),
+ request.getVolumeCapabilities(0).getAccessMode());
+ Assert.assertTrue(request.getVolumeCapabilities(0).hasMount());
+ Assert.assertEquals(2, request.getVolumeCapabilities(0)
+ .getMount().getMountFlagsCount());
+ Assert.assertTrue(request.getVolumeCapabilities(0)
+ .getMount().getMountFlagsList().contains("mountFlag1"));
+ Assert.assertTrue(request.getVolumeCapabilities(0)
+ .getMount().getMountFlagsList().contains("mountFlag2"));
+ Assert.assertEquals(2, request.getVolumeAttributesCount());
+ Assert.assertEquals("v1", request.getVolumeAttributesMap().get("k1"));
+ Assert.assertEquals("v2", request.getVolumeAttributesMap().get("k2"));
+ // return a fake result
+ return Csi.ValidateVolumeCapabilitiesResponse.newBuilder()
+ .setSupported(false)
+ .setMessage("this is a test")
+ .build();
+ }
+ });
+
+ service.init(conf);
+ service.start();
+
+ YarnRPC rpc = YarnRPC.create(conf);
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ CsiAdaptorProtocol adaptorClient = NMProxy
+ .createNMProxy(conf, CsiAdaptorProtocol.class, currentUser, rpc,
+ NetUtils.createSocketAddrForHost("localhost", ss.getLocalPort()));
+ ValidateVolumeCapabilitiesRequest request =
+ ValidateVolumeCapabilitiesRequestPBImpl
+ .newInstance("volume-id-0000123",
+ ImmutableList.of(new ValidateVolumeCapabilitiesRequest
+ .VolumeCapability(
+ MULTI_NODE_MULTI_WRITER, FILE_SYSTEM,
+ ImmutableList.of("mountFlag1", "mountFlag2"))),
+ ImmutableMap.of("k1", "v1", "k2", "v2"));
+
+ ValidateVolumeCapabilitiesResponse response = adaptorClient
+ .validateVolumeCapacity(request);
+ Assert.assertEquals(false, response.isSupported());
+ Assert.assertEquals("this is a test", response.getResponseMessage());
+
+ service.stop();
+ }
+
+ @Test (expected = ServiceStateException.class)
+ public void testMissingConfiguration() {
+ Configuration conf = new Configuration();
+ CsiAdaptorProtocolService service =
+ new CsiAdaptorProtocolService("test-driver", domainSocket);
+ service.init(conf);
+ }
+
+ @Test (expected = ServiceStateException.class)
+ public void testInvalidServicePort() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
+ + "test-driver-0001.address",
+ "0.0.0.0:-100"); // this is an invalid address
+ CsiAdaptorProtocolService service =
+ new CsiAdaptorProtocolService("test-driver-0001", domainSocket);
+ service.init(conf);
+ }
+
+ @Test (expected = ServiceStateException.class)
+ public void testInvalidHost() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
+ + "test-driver-0001.address",
+ "192.0.1:8999"); // this is an invalid ip address
+ CsiAdaptorProtocolService service =
+ new CsiAdaptorProtocolService("test-driver-0001", domainSocket);
+ service.init(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java
new file mode 100644
index 0000000..f1734c8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.csi.adaptor;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoResponsePBImpl;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the integrity of GetPluginInfoRequest and GetPluginInfoResponse.
+ */
+public class TestGetPluginInfoRequestResponse {
+
+ @Test
+ public void testGetPluginInfoRequestPBRecord() {
+ CsiAdaptorProtos.GetPluginInfoRequest requestProto =
+ CsiAdaptorProtos.GetPluginInfoRequest.newBuilder().build();
+ GetPluginInfoRequestPBImpl pbImpl =
+ new GetPluginInfoRequestPBImpl(requestProto);
+ Assert.assertNotNull(pbImpl);
+ Assert.assertEquals(requestProto, pbImpl.getProto());
+ }
+
+ @Test
+ public void testGetPluginInfoResponsePBRecord() {
+ CsiAdaptorProtos.GetPluginInfoResponse responseProto =
+ CsiAdaptorProtos.GetPluginInfoResponse.newBuilder()
+ .setName("test-driver")
+ .setVendorVersion("1.0.1")
+ .build();
+
+ GetPluginInfoResponsePBImpl pbImpl =
+ new GetPluginInfoResponsePBImpl(responseProto);
+ Assert.assertEquals("test-driver", pbImpl.getDriverName());
+ Assert.assertEquals("1.0.1", pbImpl.getVersion());
+ Assert.assertEquals(responseProto, pbImpl.getProto());
+
+ GetPluginInfoResponse pbImpl2 = GetPluginInfoResponsePBImpl
+ .newInstance("test-driver", "1.0.1");
+ Assert.assertEquals("test-driver", pbImpl2.getDriverName());
+ Assert.assertEquals("1.0.1", pbImpl2.getVersion());
+
+ CsiAdaptorProtos.GetPluginInfoResponse proto =
+ ((GetPluginInfoResponsePBImpl) pbImpl2).getProto();
+ Assert.assertEquals("test-driver", proto.getName());
+ Assert.assertEquals("1.0.1", proto.getVendorVersion());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java
new file mode 100644
index 0000000..303cfc4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.csi.adaptor;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeCapability;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesRequestPBImpl;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos.VolumeCapability.AccessMode;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos.VolumeCapability.VolumeType;
+import org.apache.hadoop.yarn.proto.YarnProtos;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.AccessMode.MULTI_NODE_MULTI_WRITER;
+import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeType.FILE_SYSTEM;
+
+/**
+ * UT for message exchanges.
+ */
+public class TestValidateVolumeCapabilityRequest {
+
+ @Test
+ public void testPBRecord() {
+ CsiAdaptorProtos.VolumeCapability vcProto =
+ CsiAdaptorProtos.VolumeCapability.newBuilder()
+ .setAccessMode(AccessMode.MULTI_NODE_MULTI_WRITER)
+ .setVolumeType(VolumeType.FILE_SYSTEM)
+ .addMountFlags("flag0")
+ .addMountFlags("flag1")
+ .build();
+
+ CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest requestProto =
+ CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest.newBuilder()
+ .setVolumeId("volume-id-0000001")
+ .addVolumeCapabilities(vcProto)
+ .addVolumeAttributes(YarnProtos.StringStringMapProto
+ .newBuilder().setKey("attr0")
+ .setValue("value0")
+ .build())
+ .addVolumeAttributes(YarnProtos.StringStringMapProto
+ .newBuilder().setKey("attr1")
+ .setValue("value1")
+ .build())
+ .build();
+
+ ValidateVolumeCapabilitiesRequestPBImpl request =
+ new ValidateVolumeCapabilitiesRequestPBImpl(requestProto);
+
+ Assert.assertEquals("volume-id-0000001", request.getVolumeId());
+ Assert.assertEquals(2, request.getVolumeAttributes().size());
+ Assert.assertEquals("value0", request.getVolumeAttributes().get("attr0"));
+ Assert.assertEquals("value1", request.getVolumeAttributes().get("attr1"));
+ Assert.assertEquals(1, request.getVolumeCapabilities().size());
+ VolumeCapability vc =
+ request.getVolumeCapabilities().get(0);
+ Assert.assertEquals(MULTI_NODE_MULTI_WRITER, vc.getAccessMode());
+ Assert.assertEquals(FILE_SYSTEM, vc.getVolumeType());
+ Assert.assertEquals(2, vc.getMountFlags().size());
+
+ Assert.assertEquals(requestProto, request.getProto());
+ }
+
+ @Test
+ public void testNewInstance() {
+ ValidateVolumeCapabilitiesRequest pbImpl =
+ ValidateVolumeCapabilitiesRequestPBImpl
+ .newInstance("volume-id-0000123",
+ ImmutableList.of(
+ new VolumeCapability(
+ MULTI_NODE_MULTI_WRITER, FILE_SYSTEM,
+ ImmutableList.of("mountFlag1", "mountFlag2"))),
+ ImmutableMap.of("k1", "v1", "k2", "v2"));
+
+ Assert.assertEquals("volume-id-0000123", pbImpl.getVolumeId());
+ Assert.assertEquals(1, pbImpl.getVolumeCapabilities().size());
+ Assert.assertEquals(FILE_SYSTEM,
+ pbImpl.getVolumeCapabilities().get(0).getVolumeType());
+ Assert.assertEquals(MULTI_NODE_MULTI_WRITER,
+ pbImpl.getVolumeCapabilities().get(0).getAccessMode());
+ Assert.assertEquals(2, pbImpl.getVolumeAttributes().size());
+
+ CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest proto =
+ ((ValidateVolumeCapabilitiesRequestPBImpl) pbImpl).getProto();
+ Assert.assertEquals("volume-id-0000123", proto.getVolumeId());
+ Assert.assertEquals(1, proto.getVolumeCapabilitiesCount());
+ Assert.assertEquals(AccessMode.MULTI_NODE_MULTI_WRITER,
+ proto.getVolumeCapabilities(0).getAccessMode());
+ Assert.assertEquals(VolumeType.FILE_SYSTEM,
+ proto.getVolumeCapabilities(0).getVolumeType());
+ Assert.assertEquals(2, proto.getVolumeCapabilities(0)
+ .getMountFlagsCount());
+ Assert.assertEquals(2, proto.getVolumeCapabilities(0)
+ .getMountFlagsList().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java
new file mode 100644
index 0000000..97f116a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.csi.adaptor;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesResponsePBImpl;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * UT for message exchanges.
+ */
+public class TestValidateVolumeCapabilityResponse {
+
+ @Test
+ public void testPBRecord() {
+ CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse proto =
+ CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse.newBuilder()
+ .setSupported(true)
+ .setMessage("capability is supported")
+ .build();
+
+ ValidateVolumeCapabilitiesResponsePBImpl pbImpl =
+ new ValidateVolumeCapabilitiesResponsePBImpl(proto);
+
+ Assert.assertEquals(true, pbImpl.isSupported());
+ Assert.assertEquals("capability is supported", pbImpl.getResponseMessage());
+ Assert.assertEquals(proto, pbImpl.getProto());
+ }
+
+ @Test
+ public void testNewInstance() {
+ ValidateVolumeCapabilitiesResponse pbImpl =
+ ValidateVolumeCapabilitiesResponsePBImpl
+ .newInstance(false, "capability not supported");
+ Assert.assertEquals(false, pbImpl.isSupported());
+ Assert.assertEquals("capability not supported",
+ pbImpl.getResponseMessage());
+
+ CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse proto =
+ ((ValidateVolumeCapabilitiesResponsePBImpl) pbImpl).getProto();
+ Assert.assertEquals(false, proto.getSupported());
+ Assert.assertEquals("capability not supported", proto.getMessage());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java
new file mode 100644
index 0000000..ecc7fc2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains UT classes for CSI adaptor.
+ */
+package org.apache.hadoop.yarn.csi.adaptor;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/volume/csi/CsiAdaptorClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/volume/csi/CsiAdaptorClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/volume/csi/CsiAdaptorClientProtocol.java
deleted file mode 100644
index b894d4e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/volume/csi/CsiAdaptorClientProtocol.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.volume.csi;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.server.volume.csi.exception.VolumeException;
-
-/**
- * Protocol for the CSI adaptor.
- */
-@Private
-@Unstable
-public interface CsiAdaptorClientProtocol {
-
- void validateVolume() throws VolumeException;
-
- void controllerPublishVolume() throws VolumeException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/CsiAdaptorClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/CsiAdaptorClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/CsiAdaptorClient.java
deleted file mode 100644
index 043e7ae..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/CsiAdaptorClient.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.volume.csi;
-
-import org.apache.hadoop.yarn.server.volume.csi.exception.VolumeException;
-import org.apache.hadoop.yarn.server.volume.csi.CsiAdaptorClientProtocol;
-
-/**
- * Client talks to CSI adaptor.
- */
-public class CsiAdaptorClient implements CsiAdaptorClientProtocol {
-
- @Override
- public void validateVolume() throws VolumeException {
- // TODO
- }
-
- @Override public void controllerPublishVolume() throws VolumeException {
- // TODO
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/VolumeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/VolumeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/VolumeManager.java
index 5f2669d..32563cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/VolumeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/VolumeManager.java
@@ -17,13 +17,12 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.volume.csi;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.lifecycle.Volume;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.provisioner.VolumeProvisioningResults;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.provisioner.VolumeProvisioningTask;
-import org.apache.hadoop.yarn.server.volume.csi.CsiAdaptorClientProtocol;
import java.util.concurrent.ScheduledFuture;
@@ -40,12 +39,8 @@ public interface VolumeManager {
/**
* @return all known volumes and their states.
*/
- @VisibleForTesting
VolumeStates getVolumeStates();
- @VisibleForTesting
- void setClient(CsiAdaptorClientProtocol client);
-
/**
* Start to supervise on a volume.
* @param volume
@@ -60,4 +55,20 @@ public interface VolumeManager {
*/
ScheduledFuture<VolumeProvisioningResults> schedule(
VolumeProvisioningTask volumeProvisioningTask, int delaySecond);
+
+ /**
+ * Register a csi-driver-adaptor to the volume manager.
+ * @param driverName
+ * @param client
+ */
+ void registerCsiDriverAdaptor(String driverName, CsiAdaptorProtocol client);
+
+ /**
+ * Returns the csi-driver-adaptor client from cache by the given driver name.
+ * If the client is not found, null is returned.
+ * @param driverName
+ * @return a csi-driver-adaptor client working for given driver or null
+ * if the adaptor could not be found.
+ */
+ CsiAdaptorProtocol getAdaptorByDriverName(String driverName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/VolumeManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/VolumeManagerImpl.java
index 5252f53..839d1bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/VolumeManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/VolumeManagerImpl.java
@@ -18,16 +18,28 @@
package org.apache.hadoop.yarn.server.resourcemanager.volume.csi;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse;
+import org.apache.hadoop.yarn.client.NMProxy;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.lifecycle.Volume;
-import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.lifecycle.VolumeImpl;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.provisioner.VolumeProvisioningResults;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.provisioner.VolumeProvisioningTask;
-import org.apache.hadoop.yarn.server.volume.csi.CsiAdaptorClientProtocol;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
@@ -43,20 +55,84 @@ public class VolumeManagerImpl extends AbstractService
private final VolumeStates volumeStates;
private ScheduledExecutorService provisioningExecutor;
- private CsiAdaptorClientProtocol adaptorClient;
+ private Map<String, CsiAdaptorProtocol> csiAdaptorMap;
private final static int PROVISIONING_TASK_THREAD_POOL_SIZE = 10;
public VolumeManagerImpl() {
super(VolumeManagerImpl.class.getName());
this.volumeStates = new VolumeStates();
+ this.csiAdaptorMap = new ConcurrentHashMap<>();
this.provisioningExecutor = Executors
.newScheduledThreadPool(PROVISIONING_TASK_THREAD_POOL_SIZE);
- this.adaptorClient = new CsiAdaptorClient();
+ }
+
+ // Init the CSI adaptor cache according to the configuration.
+ // user only needs to configure a list of adaptor addresses,
+ // this method extracts each address and init an adaptor client,
+ // then proceed with a hand-shake by calling adaptor's getPluginInfo
+ // method to retrieve the driver info. If the driver can be resolved,
+ // it is then added to the cache. Note, we don't allow two drivers
+ // specified with same driver-name even version is different.
+ private void initCsiAdaptorCache(
+ final Map<String, CsiAdaptorProtocol> adaptorMap, Configuration conf)
+ throws IOException, YarnException {
+ LOG.info("Initializing cache for csi-driver-adaptors");
+ String[] addresses =
+ conf.getStrings(YarnConfiguration.NM_CSI_ADAPTOR_ADDRESSES);
+ if (addresses != null && addresses.length > 0) {
+ for (String addr : addresses) {
+ LOG.info("Found csi-driver-adaptor socket address: " + addr);
+ InetSocketAddress address = NetUtils.createSocketAddr(addr);
+ YarnRPC rpc = YarnRPC.create(conf);
+ UserGroupInformation currentUser =
+ UserGroupInformation.getCurrentUser();
+ CsiAdaptorProtocol adaptorClient = NMProxy
+ .createNMProxy(conf, CsiAdaptorProtocol.class, currentUser, rpc,
+ address);
+ // Attempt to resolve the driver by contacting to
+ // the diver's identity service on the given address.
+ // If the call failed, the initialization is also failed
+ // in order running into inconsistent state.
+ LOG.info("Retrieving info from csi-driver-adaptor on address " + addr);
+ GetPluginInfoResponse response =
+ adaptorClient.getPluginInfo(GetPluginInfoRequest.newInstance());
+ if (!Strings.isNullOrEmpty(response.getDriverName())) {
+ String driverName = response.getDriverName();
+ if (adaptorMap.containsKey(driverName)) {
+ throw new YarnException(
+ "Duplicate driver adaptor found," + " driver name: "
+ + driverName);
+ }
+ adaptorMap.put(driverName, adaptorClient);
+ LOG.info("CSI Adaptor added to the cache, adaptor name: " + driverName
+ + ", driver version: " + response.getVersion());
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns a CsiAdaptorProtocol client by the given driver name,
+ * returns null if no adaptor is found for the driver, that means
+ * the driver has not registered to the volume manager yet enhance not valid.
+ * @param driverName the name of the driver
+ * @return CsiAdaptorProtocol client or null if driver not registered
+ */
+ public CsiAdaptorProtocol getAdaptorByDriverName(String driverName) {
+ return csiAdaptorMap.get(driverName);
+ }
+
+ @VisibleForTesting
+ @Override
+ public void registerCsiDriverAdaptor(String driverName,
+ CsiAdaptorProtocol client) {
+ this.csiAdaptorMap.put(driverName, client);
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
+ initCsiAdaptorCache(csiAdaptorMap, conf);
super.serviceInit(conf);
}
@@ -82,18 +158,11 @@ public class VolumeManagerImpl extends AbstractService
// volume already exists
return volumeStates.getVolume(volume.getVolumeId());
} else {
- // add the volume and set the client
- ((VolumeImpl) volume).setClient(adaptorClient);
this.volumeStates.addVolumeIfAbsent(volume);
return volume;
}
}
- @VisibleForTesting
- public void setClient(CsiAdaptorClientProtocol client) {
- this.adaptorClient = client;
- }
-
@Override
public ScheduledFuture<VolumeProvisioningResults> schedule(
VolumeProvisioningTask volumeProvisioningTask,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/Volume.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/Volume.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/Volume.java
index 68e89b0..83501ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/Volume.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/Volume.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.volume.csi.lifecycle;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.event.VolumeEvent;
import org.apache.hadoop.yarn.server.volume.csi.VolumeId;
+import org.apache.hadoop.yarn.server.volume.csi.VolumeMetaData;
/**
* Major volume interface at RM's view, it maintains the volume states and
@@ -34,4 +36,10 @@ public interface Volume extends EventHandler<VolumeEvent> {
VolumeState getVolumeState();
VolumeId getVolumeId();
+
+ VolumeMetaData getVolumeMeta();
+
+ CsiAdaptorProtocol getClient();
+
+ void setClient(CsiAdaptorProtocol client);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/VolumeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/VolumeImpl.java
index 2515047..82a4acb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/VolumeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/VolumeImpl.java
@@ -18,10 +18,15 @@
package org.apache.hadoop.yarn.server.resourcemanager.volume.csi.lifecycle;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.CsiAdaptorClient;
-import org.apache.hadoop.yarn.server.volume.csi.CsiAdaptorClientProtocol;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeCapability;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.event.VolumeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.event.VolumeEventType;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
@@ -30,13 +35,16 @@ import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.server.volume.csi.VolumeId;
import org.apache.hadoop.yarn.server.volume.csi.VolumeMetaData;
-import org.apache.hadoop.yarn.server.volume.csi.exception.VolumeException;
+import java.io.IOException;
import java.util.EnumSet;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.AccessMode.SINGLE_NODE_READER_ONLY;
+import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeType.FILE_SYSTEM;
+
/**
* This class maintains the volume states and state transition
* according to the CSI volume lifecycle. Volume states are stored in
@@ -54,7 +62,7 @@ public class VolumeImpl implements Volume {
private final VolumeId volumeId;
private final VolumeMetaData volumeMeta;
- private CsiAdaptorClientProtocol client;
+ private CsiAdaptorProtocol adaptorClient;
public VolumeImpl(VolumeMetaData volumeMeta) {
ReadWriteLock lock = new ReentrantReadWriteLock();
@@ -63,16 +71,21 @@ public class VolumeImpl implements Volume {
this.volumeId = volumeMeta.getVolumeId();
this.volumeMeta = volumeMeta;
this.stateMachine = createVolumeStateFactory().make(this);
- this.client = new CsiAdaptorClient();
}
@VisibleForTesting
- public void setClient(CsiAdaptorClientProtocol client) {
- this.client = client;
+ public void setClient(CsiAdaptorProtocol csiAdaptorClient) {
+ this.adaptorClient = csiAdaptorClient;
}
- public CsiAdaptorClientProtocol getClient() {
- return this.client;
+ @Override
+ public CsiAdaptorProtocol getClient() {
+ return this.adaptorClient;
+ }
+
+ @Override
+ public VolumeMetaData getVolumeMeta() {
+ return this.volumeMeta;
}
private StateMachineFactory<VolumeImpl, VolumeState,
@@ -135,9 +148,20 @@ public class VolumeImpl implements Volume {
VolumeEvent volumeEvent) {
try {
// this call could cross node, we should keep the message tight
- volume.getClient().validateVolume();
- return VolumeState.VALIDATED;
- } catch (VolumeException e) {
+ // TODO we should parse the capability from volume resource spec
+ VolumeCapability capability = new VolumeCapability(
+ SINGLE_NODE_READER_ONLY, FILE_SYSTEM,
+ ImmutableList.of());
+ ValidateVolumeCapabilitiesRequest request =
+ ValidateVolumeCapabilitiesRequest
+ .newInstance(volume.getVolumeId().getId(),
+ ImmutableList.of(capability),
+ ImmutableMap.of());
+ ValidateVolumeCapabilitiesResponse response = volume.getClient()
+ .validateVolumeCapacity(request);
+ return response.isSupported() ? VolumeState.VALIDATED
+ : VolumeState.UNAVAILABLE;
+ } catch (YarnException | IOException e) {
LOG.warn("Got exception while calling the CSI adaptor", e);
return VolumeState.UNAVAILABLE;
}
@@ -150,14 +174,8 @@ public class VolumeImpl implements Volume {
@Override
public VolumeState transition(VolumeImpl volume,
VolumeEvent volumeEvent) {
- try {
- // this call could cross node, we should keep the message tight
- volume.getClient().controllerPublishVolume();
- return VolumeState.NODE_READY;
- } catch (VolumeException e) {
- LOG.warn("Got exception while calling the CSI adaptor", e);
- return volume.getVolumeState();
- }
+ // this call could cross node, we should keep the message tight
+ return VolumeState.NODE_READY;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/processor/VolumeAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/processor/VolumeAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/processor/VolumeAMSProcessor.java
index f275768..814634a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/processor/VolumeAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/processor/VolumeAMSProcessor.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.volume.csi.processor;
import org.apache.hadoop.yarn.ams.ApplicationMasterServiceContext;
import org.apache.hadoop.yarn.ams.ApplicationMasterServiceProcessor;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
@@ -29,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.VolumeManager;
@@ -142,8 +144,21 @@ public class VolumeAMSProcessor implements ApplicationMasterServiceProcessor {
* @param metaData
* @return volume
*/
- private Volume checkAndGetVolume(VolumeMetaData metaData) {
+ private Volume checkAndGetVolume(VolumeMetaData metaData)
+ throws InvalidVolumeException {
Volume toAdd = new VolumeImpl(metaData);
+ CsiAdaptorProtocol adaptor = volumeManager
+ .getAdaptorByDriverName(metaData.getDriverName());
+ if (adaptor == null) {
+ throw new InvalidVolumeException("It seems for the driver name"
+ + " specified in the volume " + metaData.getDriverName()
+ + " ,there is no matched driver-adaptor can be found. "
+ + "Is the driver probably registered? Please check if"
+ + " adaptors service addresses defined in "
+ + YarnConfiguration.NM_CSI_ADAPTOR_ADDRESSES
+ + " are correct and services are started.");
+ }
+ toAdd.setClient(adaptor);
return this.volumeManager.addOrGetVolume(toAdd);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeLifecycle.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeLifecycle.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeLifecycle.java
index 18c23e8..a94f508 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeLifecycle.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeLifecycle.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.volume.csi;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.event.ControllerPublishVolumeEvent;
-import org.apache.hadoop.yarn.server.volume.csi.CsiAdaptorClientProtocol;
-import org.apache.hadoop.yarn.server.volume.csi.exception.InvalidVolumeException;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.event.ValidateVolumeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.lifecycle.VolumeImpl;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.lifecycle.VolumeState;
@@ -29,8 +31,8 @@ import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
+import java.io.IOException;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
import static org.mockito.Mockito.*;
@@ -40,7 +42,13 @@ import static org.mockito.Mockito.*;
public class TestVolumeLifecycle {
@Test
- public void testValidation() throws InvalidVolumeException {
+ public void testValidation() throws YarnException, IOException {
+ CsiAdaptorProtocol mockedClient = Mockito
+ .mock(CsiAdaptorProtocol.class);
+ doReturn(ValidateVolumeCapabilitiesResponse.newInstance(true, ""))
+ .when(mockedClient)
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
+
VolumeImpl volume = (VolumeImpl) VolumeBuilder.newBuilder()
.volumeId("test_vol_00000001")
.maxCapability(5L)
@@ -48,6 +56,7 @@ public class TestVolumeLifecycle {
.mountPoint("/path/to/mount")
.driverName("test-driver-name")
.build();
+ volume.setClient(mockedClient);
Assert.assertEquals(VolumeState.NEW, volume.getVolumeState());
volume.handle(new ValidateVolumeEvent(volume));
@@ -55,16 +64,19 @@ public class TestVolumeLifecycle {
}
@Test
- public void testValidationFailure() throws VolumeException {
+ public void testVolumeCapacityNotSupported() throws Exception {
+ CsiAdaptorProtocol mockedClient = Mockito
+ .mock(CsiAdaptorProtocol.class);
+
VolumeImpl volume = (VolumeImpl) VolumeBuilder
.newBuilder().volumeId("test_vol_00000001").build();
- CsiAdaptorClientProtocol mockedClient = Mockito
- .mock(CsiAdaptorClientProtocol.class);
volume.setClient(mockedClient);
// NEW -> UNAVAILABLE
// Simulate a failed API call to the adaptor
- doThrow(new VolumeException("failed")).when(mockedClient).validateVolume();
+ doReturn(ValidateVolumeCapabilitiesResponse.newInstance(false, ""))
+ .when(mockedClient)
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
volume.handle(new ValidateVolumeEvent(volume));
try {
@@ -80,47 +92,62 @@ public class TestVolumeLifecycle {
}
@Test
- public void testValidated() throws InvalidVolumeException {
- AtomicInteger validatedTimes = new AtomicInteger(0);
+ public void testValidationFailure() throws YarnException, IOException {
+ CsiAdaptorProtocol mockedClient = Mockito
+ .mock(CsiAdaptorProtocol.class);
+ doThrow(new VolumeException("fail"))
+ .when(mockedClient)
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
+
+ VolumeImpl volume = (VolumeImpl) VolumeBuilder
+ .newBuilder().volumeId("test_vol_00000001").build();
+ volume.setClient(mockedClient);
+
+ // NEW -> UNAVAILABLE
+ // Simulate a failed API call to the adaptor
+ doThrow(new VolumeException("failed"))
+ .when(mockedClient)
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
+ volume.handle(new ValidateVolumeEvent(volume));
+ }
+
+ @Test
+ public void testValidated() throws YarnException, IOException {
VolumeImpl volume = (VolumeImpl) VolumeBuilder
.newBuilder().volumeId("test_vol_00000001").build();
- CsiAdaptorClientProtocol mockedClient = new CsiAdaptorClientProtocol() {
- @Override
- public void validateVolume() {
- validatedTimes.incrementAndGet();
- }
-
- @Override
- public void controllerPublishVolume() {
- // do nothing
- }
- };
+ CsiAdaptorProtocol mockedClient = Mockito.mock(CsiAdaptorProtocol.class);
// The client has a count to memorize how many times being called
volume.setClient(mockedClient);
// NEW -> VALIDATED
+ doReturn(ValidateVolumeCapabilitiesResponse.newInstance(true, ""))
+ .when(mockedClient)
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
Assert.assertEquals(VolumeState.NEW, volume.getVolumeState());
volume.handle(new ValidateVolumeEvent(volume));
Assert.assertEquals(VolumeState.VALIDATED, volume.getVolumeState());
- Assert.assertEquals(1, validatedTimes.get());
+ verify(mockedClient, times(1))
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
// VALIDATED -> VALIDATED
volume.handle(new ValidateVolumeEvent(volume));
Assert.assertEquals(VolumeState.VALIDATED, volume.getVolumeState());
- Assert.assertEquals(1, validatedTimes.get());
+ verify(mockedClient, times(1))
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
}
@Test
- public void testUnavailableState() throws VolumeException {
+ public void testUnavailableState() throws YarnException, IOException {
VolumeImpl volume = (VolumeImpl) VolumeBuilder
.newBuilder().volumeId("test_vol_00000001").build();
- CsiAdaptorClientProtocol mockedClient = Mockito
- .mock(CsiAdaptorClientProtocol.class);
+ CsiAdaptorProtocol mockedClient = Mockito
+ .mock(CsiAdaptorProtocol.class);
volume.setClient(mockedClient);
// NEW -> UNAVAILABLE
- doThrow(new VolumeException("failed")).when(mockedClient)
- .validateVolume();
+ doThrow(new VolumeException("failed"))
+ .when(mockedClient)
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
Assert.assertEquals(VolumeState.NEW, volume.getVolumeState());
volume.handle(new ValidateVolumeEvent(volume));
Assert.assertEquals(VolumeState.UNAVAILABLE, volume.getVolumeState());
@@ -130,23 +157,26 @@ public class TestVolumeLifecycle {
Assert.assertEquals(VolumeState.UNAVAILABLE, volume.getVolumeState());
// UNAVAILABLE -> VALIDATED
- doNothing().when(mockedClient).validateVolume();
+ doReturn(ValidateVolumeCapabilitiesResponse.newInstance(true, ""))
+ .when(mockedClient)
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
volume.setClient(mockedClient);
volume.handle(new ValidateVolumeEvent(volume));
Assert.assertEquals(VolumeState.VALIDATED, volume.getVolumeState());
}
@Test
- public void testPublishUnavailableVolume() throws VolumeException {
+ public void testPublishUnavailableVolume() throws YarnException, IOException {
VolumeImpl volume = (VolumeImpl) VolumeBuilder
.newBuilder().volumeId("test_vol_00000001").build();
- CsiAdaptorClientProtocol mockedClient = Mockito
- .mock(CsiAdaptorClientProtocol.class);
+ CsiAdaptorProtocol mockedClient = Mockito
+ .mock(CsiAdaptorProtocol.class);
volume.setClient(mockedClient);
// NEW -> UNAVAILABLE (on validateVolume)
- doThrow(new VolumeException("failed")).when(mockedClient)
- .validateVolume();
+ doThrow(new VolumeException("failed"))
+ .when(mockedClient)
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
Assert.assertEquals(VolumeState.NEW, volume.getVolumeState());
volume.handle(new ValidateVolumeEvent(volume));
Assert.assertEquals(VolumeState.UNAVAILABLE, volume.getVolumeState());
@@ -154,7 +184,7 @@ public class TestVolumeLifecycle {
// UNAVAILABLE -> UNAVAILABLE (on publishVolume)
volume.handle(new ControllerPublishVolumeEvent(volume));
// controller publish is not called since the state is UNAVAILABLE
- verify(mockedClient, times(0)).controllerPublishVolume();
+ // verify(mockedClient, times(0)).controllerPublishVolume();
// state remains to UNAVAILABLE
Assert.assertEquals(VolumeState.UNAVAILABLE, volume.getVolumeState());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java
index d6f9d92..cee8fdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java
@@ -20,8 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.volume.csi;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceSizing;
@@ -40,7 +43,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.lifecycle.Volume;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.lifecycle.VolumeState;
import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.processor.VolumeAMSProcessor;
-import org.apache.hadoop.yarn.server.volume.csi.CsiAdaptorClientProtocol;
import org.apache.hadoop.yarn.server.volume.csi.CsiConstants;
import org.apache.hadoop.yarn.server.volume.csi.VolumeId;
import org.apache.hadoop.yarn.server.volume.csi.exception.InvalidVolumeException;
@@ -57,6 +59,10 @@ import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+
/**
* Test cases for volume processor.
*/
@@ -91,6 +97,7 @@ public class TestVolumeProcessor {
conf.set(CapacitySchedulerConfiguration.PREFIX
+ CapacitySchedulerConfiguration.ROOT + ".default.ordering-policy",
"fair");
+ // this is required to enable volume processor
conf.set(YarnConfiguration.RM_APPLICATION_MASTER_SERVICE_PROCESSORS,
VolumeAMSProcessor.class.getName());
mgr = new NullRMNodeLabelsManager();
@@ -155,6 +162,17 @@ public class TestVolumeProcessor {
.schedulingRequests(Arrays.asList(sc))
.build();
+ // inject adaptor client for testing
+ CsiAdaptorProtocol mockedClient = Mockito
+ .mock(CsiAdaptorProtocol.class);
+ rm.getRMContext().getVolumeManager()
+ .registerCsiDriverAdaptor("hostpath", mockedClient);
+
+ // simulate validation succeed
+ doReturn(ValidateVolumeCapabilitiesResponse.newInstance(true, ""))
+ .when(mockedClient)
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
+
am1.allocate(ar);
VolumeStates volumeStates =
rm.getRMContext().getVolumeManager().getVolumeStates();
@@ -212,12 +230,14 @@ public class TestVolumeProcessor {
RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, mockNMS[0]);
- CsiAdaptorClientProtocol mockedClient = Mockito
- .mock(CsiAdaptorClientProtocol.class);
+ CsiAdaptorProtocol mockedClient = Mockito
+ .mock(CsiAdaptorProtocol.class);
// inject adaptor client
- rm.getRMContext().getVolumeManager().setClient(mockedClient);
- Mockito.doThrow(new VolumeException("failed")).when(mockedClient)
- .validateVolume();
+ rm.getRMContext().getVolumeManager()
+ .registerCsiDriverAdaptor("hostpath", mockedClient);
+ doThrow(new VolumeException("failed"))
+ .when(mockedClient)
+ .validateVolumeCapacity(any(ValidateVolumeCapabilitiesRequest.class));
Resource resource = Resource.newInstance(1024, 1);
ResourceInformation volumeResource = ResourceInformation
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDFS-13845. RBF: The default
MountTableResolver should fail resolving multi-destination paths. Contributed
by yanghuafeng.
Posted by br...@apache.org.
HDFS-13845. RBF: The default MountTableResolver should fail resolving multi-destination paths. Contributed by yanghuafeng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e4e0981
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e4e0981
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e4e0981
Branch: refs/heads/HDFS-13891
Commit: 5e4e0981036180555380379f62abb94fc19910f4
Parents: ee21095
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Tue Oct 30 11:21:08 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:34:52 2018 +0530
----------------------------------------------------------------------
.../federation/resolver/MountTableResolver.java | 15 +++++--
.../resolver/TestMountTableResolver.java | 45 ++++++++++++++++----
.../router/TestDisableNameservices.java | 36 ++++++++++------
3 files changed, 70 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e4e0981/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index 121469f..9e69840 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -539,21 +539,28 @@ public class MountTableResolver
* @param entry Mount table entry.
* @return PathLocation containing the namespace, local path.
*/
- private static PathLocation buildLocation(
- final String path, final MountTable entry) {
-
+ private PathLocation buildLocation(
+ final String path, final MountTable entry) throws IOException {
String srcPath = entry.getSourcePath();
if (!path.startsWith(srcPath)) {
LOG.error("Cannot build location, {} not a child of {}", path, srcPath);
return null;
}
+
+ List<RemoteLocation> dests = entry.getDestinations();
+ if (getClass() == MountTableResolver.class && dests.size() > 1) {
+ throw new IOException("Cannnot build location, "
+ + getClass().getSimpleName()
+ + " should not resolve multiple destinations for " + path);
+ }
+
String remainingPath = path.substring(srcPath.length());
if (remainingPath.startsWith(Path.SEPARATOR)) {
remainingPath = remainingPath.substring(1);
}
List<RemoteLocation> locations = new LinkedList<>();
- for (RemoteLocation oneDst : entry.getDestinations()) {
+ for (RemoteLocation oneDst : dests) {
String nsId = oneDst.getNameserviceId();
String dest = oneDst.getDest();
String newPath = dest;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e4e0981/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
index 5e3b861..14ccb61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -79,6 +79,8 @@ public class TestMountTableResolver {
* __usr
* ____bin -> 2:/bin
* __readonly -> 2:/tmp
+ * __multi -> 5:/dest1
+ * 6:/dest2
*
* @throws IOException If it cannot set the mount table.
*/
@@ -126,6 +128,12 @@ public class TestMountTableResolver {
MountTable readOnlyEntry = MountTable.newInstance("/readonly", map);
readOnlyEntry.setReadOnly(true);
mountTable.addEntry(readOnlyEntry);
+
+ // /multi
+ map = getMountTableEntry("5", "/dest1");
+ map.put("6", "/dest2");
+ MountTable multiEntry = MountTable.newInstance("/multi", map);
+ mountTable.addEntry(multiEntry);
}
@Before
@@ -201,6 +209,17 @@ public class TestMountTableResolver {
}
}
+ @Test
+ public void testMuiltipleDestinations() throws IOException {
+ try {
+ mountTable.getDestinationForPath("/multi");
+ fail("The getDestinationForPath call should fail.");
+ } catch (IOException ioe) {
+ GenericTestUtils.assertExceptionContains(
+ "MountTableResolver should not resolve multiple destinations", ioe);
+ }
+ }
+
private void compareLists(List<String> list1, String[] list2) {
assertEquals(list1.size(), list2.length);
for (String item : list2) {
@@ -236,8 +255,9 @@ public class TestMountTableResolver {
// Check getting all mount points (virtual and real) beneath a path
List<String> mounts = mountTable.getMountPoints("/");
- assertEquals(4, mounts.size());
- compareLists(mounts, new String[] {"tmp", "user", "usr", "readonly"});
+ assertEquals(5, mounts.size());
+ compareLists(mounts, new String[] {"tmp", "user", "usr",
+ "readonly", "multi"});
mounts = mountTable.getMountPoints("/user");
assertEquals(2, mounts.size());
@@ -263,6 +283,9 @@ public class TestMountTableResolver {
mounts = mountTable.getMountPoints("/unknownpath");
assertNull(mounts);
+
+ mounts = mountTable.getMountPoints("/multi");
+ assertEquals(0, mounts.size());
}
private void compareRecords(List<MountTable> list1, String[] list2) {
@@ -282,10 +305,10 @@ public class TestMountTableResolver {
// Check listing the mount table records at or beneath a path
List<MountTable> records = mountTable.getMounts("/");
- assertEquals(9, records.size());
+ assertEquals(10, records.size());
compareRecords(records, new String[] {"/", "/tmp", "/user", "/usr/bin",
"user/a", "/user/a/demo/a", "/user/a/demo/b", "/user/b/file1.txt",
- "readonly"});
+ "readonly", "multi"});
records = mountTable.getMounts("/user");
assertEquals(5, records.size());
@@ -305,6 +328,10 @@ public class TestMountTableResolver {
assertEquals(1, records.size());
compareRecords(records, new String[] {"/readonly"});
assertTrue(records.get(0).isReadOnly());
+
+ records = mountTable.getMounts("/multi");
+ assertEquals(1, records.size());
+ compareRecords(records, new String[] {"/multi"});
}
@Test
@@ -313,7 +340,7 @@ public class TestMountTableResolver {
// 3 mount points are present /tmp, /user, /usr
compareLists(mountTable.getMountPoints("/"),
- new String[] {"user", "usr", "tmp", "readonly"});
+ new String[] {"user", "usr", "tmp", "readonly", "multi"});
// /tmp currently points to namespace 2
assertEquals("2", mountTable.getDestinationForPath("/tmp/testfile.txt")
@@ -324,7 +351,7 @@ public class TestMountTableResolver {
// Now 2 mount points are present /user, /usr
compareLists(mountTable.getMountPoints("/"),
- new String[] {"user", "usr", "readonly"});
+ new String[] {"user", "usr", "readonly", "multi"});
// /tmp no longer exists, uses default namespace for mapping /
assertEquals("1", mountTable.getDestinationForPath("/tmp/testfile.txt")
@@ -337,7 +364,7 @@ public class TestMountTableResolver {
// 3 mount points are present /tmp, /user, /usr
compareLists(mountTable.getMountPoints("/"),
- new String[] {"user", "usr", "tmp", "readonly"});
+ new String[] {"user", "usr", "tmp", "readonly", "multi"});
// /usr is virtual, uses namespace 1->/
assertEquals("1", mountTable.getDestinationForPath("/usr/testfile.txt")
@@ -348,7 +375,7 @@ public class TestMountTableResolver {
// Verify the remove failed
compareLists(mountTable.getMountPoints("/"),
- new String[] {"user", "usr", "tmp", "readonly"});
+ new String[] {"user", "usr", "tmp", "readonly", "multi"});
}
@Test
@@ -380,7 +407,7 @@ public class TestMountTableResolver {
// Initial table loaded
testDestination();
- assertEquals(9, mountTable.getMounts("/").size());
+ assertEquals(10, mountTable.getMounts("/").size());
// Replace table with /1 and /2
List<MountTable> records = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e4e0981/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java
index 15b104d..610927d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java
@@ -21,6 +21,7 @@ import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.simul
import static org.apache.hadoop.util.Time.monotonicNow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.Iterator;
@@ -43,13 +44,13 @@ import org.apache.hadoop.hdfs.server.federation.metrics.FederationMetrics;
import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
import org.apache.hadoop.hdfs.server.federation.store.DisabledNameserviceStore;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.DisableNameserviceRequest;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.test.GenericTestUtils;
import org.codehaus.jettison.json.JSONObject;
import org.junit.After;
import org.junit.AfterClass;
@@ -106,14 +107,18 @@ public class TestDisableNameservices {
// Setup a mount table to map to the two namespaces
MountTableManager mountTable = routerAdminClient.getMountTableManager();
Map<String, String> destinations = new TreeMap<>();
- destinations.put("ns0", "/");
- destinations.put("ns1", "/");
- MountTable newEntry = MountTable.newInstance("/", destinations);
- newEntry.setDestOrder(DestinationOrder.RANDOM);
+ destinations.put("ns0", "/dirns0");
+ MountTable newEntry = MountTable.newInstance("/dirns0", destinations);
AddMountTableEntryRequest request =
AddMountTableEntryRequest.newInstance(newEntry);
mountTable.addMountTableEntry(request);
+ destinations = new TreeMap<>();
+ destinations.put("ns1", "/dirns1");
+ newEntry = MountTable.newInstance("/dirns1", destinations);
+ request = AddMountTableEntryRequest.newInstance(newEntry);
+ mountTable.addMountTableEntry(request);
+
// Refresh the cache in the Router
Router router = routerContext.getRouter();
MountTableResolver mountTableResolver =
@@ -122,9 +127,9 @@ public class TestDisableNameservices {
// Add a folder to each namespace
NamenodeContext nn0 = cluster.getNamenode("ns0", null);
- nn0.getFileSystem().mkdirs(new Path("/dirns0"));
+ nn0.getFileSystem().mkdirs(new Path("/dirns0/0"));
NamenodeContext nn1 = cluster.getNamenode("ns1", null);
- nn1.getFileSystem().mkdirs(new Path("/dirns1"));
+ nn1.getFileSystem().mkdirs(new Path("/dirns1/1"));
}
@AfterClass
@@ -153,14 +158,12 @@ public class TestDisableNameservices {
@Test
public void testWithoutDisabling() throws IOException {
-
// ns0 is slow and renewLease should take a long time
long t0 = monotonicNow();
routerProtocol.renewLease("client0");
long t = monotonicNow() - t0;
assertTrue("It took too little: " + t + "ms",
t > TimeUnit.SECONDS.toMillis(1));
-
// Return the results from all subclusters even if slow
FileSystem routerFs = routerContext.getFileSystem();
FileStatus[] filesStatus = routerFs.listStatus(new Path("/"));
@@ -171,7 +174,6 @@ public class TestDisableNameservices {
@Test
public void testDisabling() throws Exception {
-
disableNameservice("ns0");
// renewLease should be fast as we are skipping ns0
@@ -180,12 +182,20 @@ public class TestDisableNameservices {
long t = monotonicNow() - t0;
assertTrue("It took too long: " + t + "ms",
t < TimeUnit.SECONDS.toMillis(1));
-
// We should not report anything from ns0
FileSystem routerFs = routerContext.getFileSystem();
- FileStatus[] filesStatus = routerFs.listStatus(new Path("/"));
+ FileStatus[] filesStatus = null;
+ try {
+ routerFs.listStatus(new Path("/"));
+ fail("The listStatus call should fail.");
+ } catch (IOException ioe) {
+ GenericTestUtils.assertExceptionContains(
+ "No remote locations available", ioe);
+ }
+
+ filesStatus = routerFs.listStatus(new Path("/dirns1"));
assertEquals(1, filesStatus.length);
- assertEquals("dirns1", filesStatus[0].getPath().getName());
+ assertEquals("1", filesStatus[0].getPath().getName());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HADOOP-14739. Update
start-build-env.sh and build instruction for docker for Mac instead of docker
toolbox. Contributed by Dinesh Chitlangia.
Posted by br...@apache.org.
HADOOP-14739. Update start-build-env.sh and build instruction for docker for Mac instead of docker toolbox. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10b5da85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10b5da85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10b5da85
Branch: refs/heads/HDFS-13891
Commit: 10b5da85fab41c68c2c223a3a8749be9719908f6
Parents: 0123e51
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 20 23:59:08 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 20 23:59:49 2018 +0900
----------------------------------------------------------------------
BUILDING.txt | 10 +---------
start-build-env.sh | 13 +++++++------
2 files changed, 8 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10b5da85/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index d35e3af..d781c2c 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -26,19 +26,11 @@ The easiest way to get an environment with all the appropriate tools is by means
of the provided Docker config.
This requires a recent version of docker (1.4.1 and higher are known to work).
-On Linux:
+On Linux / Mac:
Install Docker and run this command:
$ ./start-build-env.sh
-On Mac:
- First make sure Virtualbox and docker toolbox are installed.
- You can use docker toolbox as described in http://docs.docker.com/mac/step_one/.
- $ docker-machine create --driver virtualbox \
- --virtualbox-memory "4096" hadoopdev
- $ eval $(docker-machine env hadoopdev)
- $ ./start-build-env.sh
-
The prompt which is then presented is located at a mounted version of the source tree
and all required tools for testing and building have been installed and configured.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10b5da85/start-build-env.sh
----------------------------------------------------------------------
diff --git a/start-build-env.sh b/start-build-env.sh
index e3d9f2f..08cdfb0 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -21,9 +21,14 @@ cd "$(dirname "$0")" # connect to root
docker build -t hadoop-build dev-support/docker
+USER_NAME=${SUDO_USER:=$USER}
+USER_ID=$(id -u "${USER_NAME}")
+
+if [ "$(uname -s)" = "Darwin" ]; then
+ GROUP_ID=100
+fi
+
if [ "$(uname -s)" = "Linux" ]; then
- USER_NAME=${SUDO_USER:=$USER}
- USER_ID=$(id -u "${USER_NAME}")
GROUP_ID=$(id -g "${USER_NAME}")
# man docker-run
# When using SELinux, mounted directories may not be accessible
@@ -51,10 +56,6 @@ if [ "$(uname -s)" = "Linux" ]; then
done
fi
fi
-else # boot2docker uid and gid
- USER_NAME=$USER
- USER_ID=1000
- GROUP_ID=50
fi
docker build -t "hadoop-build-${USER_ID}" - <<UserSpecificDocker
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDDS-844. Add logic for pipeline
teardown after timeout. Contributed by Lokesh Jain.
Posted by br...@apache.org.
HDDS-844. Add logic for pipeline teardown after timeout. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cfb915f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cfb915f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cfb915f3
Branch: refs/heads/HDFS-13891
Commit: cfb915f3df43c84d3fb36eb2a430a25946b2ddb7
Parents: be0708c
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Mon Nov 19 12:11:05 2018 +0530
Committer: Mukul Kumar Singh <ms...@apache.org>
Committed: Mon Nov 19 12:11:05 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 13 --
.../hadoop/hdds/scm/XceiverClientRatis.java | 48 -----
.../scm/client/ContainerOperationClient.java | 3 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +-
.../hadoop/hdds/scm/XceiverClientSpi.java | 11 --
.../common/src/main/resources/ozone-default.xml | 13 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 14 +-
.../scm/pipeline/PipelineActionHandler.java | 26 +--
.../scm/pipeline/PipelineReportHandler.java | 12 +-
.../hdds/scm/pipeline/PipelineStateMap.java | 6 -
.../scm/pipeline/RatisPipelineProvider.java | 10 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 176 +++++++++++++++++++
.../scm/server/StorageContainerManager.java | 4 +-
.../org/apache/hadoop/hdds/scm/TestUtils.java | 30 +++-
.../hdds/scm/pipeline/TestPipelineClose.java | 138 +++++++++++----
.../scm/pipeline/TestPipelineStateManager.java | 14 --
.../scm/pipeline/TestSCMPipelineManager.java | 4 +-
.../freon/TestFreonWithPipelineDestroy.java | 109 ++++++++++++
18 files changed, 464 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index bbd3340..c6b19ab 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -288,19 +288,6 @@ public class XceiverClientGrpc extends XceiverClientSpi {
}
}
- /**
- * Create a pipeline.
- */
- @Override
- public void createPipeline() {
- // For stand alone pipeline, there is no notion called setup pipeline.
- }
-
- @Override
- public void destroyPipeline() {
- // For stand alone pipeline, there is no notion called destroy pipeline.
- }
-
@Override
public void watchForCommit(long index, long timeout)
throws InterruptedException, ExecutionException, TimeoutException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index f6083ec..6b3b001 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.io.MultipleIOException;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.thirdparty.com.google.protobuf
@@ -27,7 +26,6 @@ import org.apache.ratis.thirdparty.com.google.protobuf
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
@@ -36,19 +34,14 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.ratis.RatisHelper;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.protocol.RaftClientReply;
-import org.apache.ratis.protocol.RaftGroup;
-import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.apache.ratis.util.CheckedBiConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import java.util.Objects;
import java.util.Collection;
import java.util.concurrent.CompletableFuture;
@@ -97,22 +90,6 @@ public final class XceiverClientRatis extends XceiverClientSpi {
this.retryPolicy = retryPolicy;
}
- @Override
- public void createPipeline() throws IOException {
- final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
- LOG.debug("creating pipeline:{} with {}", pipeline.getId(), group);
- callRatisRpc(pipeline.getNodes(),
- (raftClient, peer) -> raftClient.groupAdd(group, peer.getId()));
- }
-
- @Override
- public void destroyPipeline() throws IOException {
- final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
- LOG.debug("destroying pipeline:{} with {}", pipeline.getId(), group);
- callRatisRpc(pipeline.getNodes(), (raftClient, peer) -> raftClient
- .groupRemove(group.getGroupId(), true, peer.getId()));
- }
-
/**
* Returns Ratis as pipeline Type.
*
@@ -123,31 +100,6 @@ public final class XceiverClientRatis extends XceiverClientSpi {
return HddsProtos.ReplicationType.RATIS;
}
- private void callRatisRpc(List<DatanodeDetails> datanodes,
- CheckedBiConsumer<RaftClient, RaftPeer, IOException> rpc)
- throws IOException {
- if (datanodes.isEmpty()) {
- return;
- }
-
- final List<IOException> exceptions =
- Collections.synchronizedList(new ArrayList<>());
- datanodes.parallelStream().forEach(d -> {
- final RaftPeer p = RatisHelper.toRaftPeer(d);
- try (RaftClient client = RatisHelper
- .newRaftClient(rpcType, p, retryPolicy)) {
- rpc.accept(client, p);
- } catch (IOException ioe) {
- exceptions.add(
- new IOException("Failed invoke Ratis rpc " + rpc + " for " + d,
- ioe));
- }
- });
- if (!exceptions.isEmpty()) {
- throw MultipleIOException.createIOException(exceptions);
- }
- }
-
@Override
public Pipeline getPipeline() {
return pipeline;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
index 8e9f4ad..ef72e38 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
@@ -157,7 +157,8 @@ public class ContainerOperationClient implements ScmClient {
// ObjectStageChangeRequestProto.Op.create,
// ObjectStageChangeRequestProto.Stage.begin);
- client.createPipeline();
+ // client.createPipeline();
+ // TODO: Use PipelineManager to createPipeline
//storageContainerLocationClient.notifyObjectStageChange(
// ObjectStageChangeRequestProto.Type.pipeline,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 595135d..896caed 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -289,11 +289,11 @@ public final class ScmConfigKeys {
public static final String
OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
- public static final String OZONE_SCM_PIPELINE_CREATION_LEASE_TIMEOUT =
- "ozone.scm.pipeline.creation.lease.timeout";
+ public static final String OZONE_SCM_PIPELINE_DESTROY_TIMEOUT =
+ "ozone.scm.pipeline.destroy.timeout";
- public static final String
- OZONE_SCM_PIPELINE_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
+ public static final String OZONE_SCM_PIPELINE_DESTROY_TIMEOUT_DEFAULT =
+ "300s";
public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
"ozone.scm.block.deletion.max.retry";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
index 9eb49ae..7000660 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
@@ -119,17 +119,6 @@ public abstract class XceiverClientSpi implements Closeable {
throws IOException, ExecutionException, InterruptedException;
/**
- * Create a pipeline.
- */
- public abstract void createPipeline() throws IOException;
-
- /**
- * Destroy a pipeline.
- * @throws IOException
- */
- public abstract void destroyPipeline() throws IOException;
-
- /**
* Returns pipeline Type.
*
* @return - {Stand_Alone, Ratis or Chained}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index b31e490..865335c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -821,6 +821,8 @@
OM/SCM eventually. So a 30 second HB seems to work. This assumes that
replication strategy used is Ratis if not, this value should be set to
something smaller like 3 seconds.
+ ozone.scm.pipeline.close.timeout should also be adjusted accordingly,
+ if the default value for this config is not used.
</description>
</property>
<property>
@@ -1183,15 +1185,12 @@
postfix (ns,ms,s,m,h,d)</description>
</property>
<property>
- <name>ozone.scm.pipeline.creation.lease.timeout</name>
- <value>60s</value>
+ <name>ozone.scm.pipeline.destroy.timeout</name>
+ <value>300s</value>
<tag>OZONE, SCM, PIPELINE</tag>
<description>
- Pipeline creation timeout in milliseconds to be used by SCM. When
- BEGIN_CREATE event happens the pipeline is moved from ALLOCATED to
- CREATING state, SCM will now wait for the configured amount of time
- to get COMPLETE_CREATE event if it doesn't receive it will move the
- pipeline to DELETING.
+ Once a pipeline is closed, SCM should wait for the above configured time
+ before destroying a pipeline.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
index 268fe5b..4055449 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
@@ -18,9 +18,13 @@
package org.apache.hadoop.hdds.scm.node;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.RatisPipelineUtils;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.slf4j.Logger;
@@ -38,21 +42,25 @@ public class StaleNodeHandler implements EventHandler<DatanodeDetails> {
private final NodeManager nodeManager;
private final PipelineManager pipelineManager;
+ private final Configuration conf;
public StaleNodeHandler(NodeManager nodeManager,
- PipelineManager pipelineManager) {
+ PipelineManager pipelineManager, OzoneConfiguration conf) {
this.nodeManager = nodeManager;
this.pipelineManager = pipelineManager;
+ this.conf = conf;
}
@Override
public void onMessage(DatanodeDetails datanodeDetails,
- EventPublisher publisher) {
+ EventPublisher publisher) {
Set<PipelineID> pipelineIds =
nodeManager.getPipelines(datanodeDetails);
for (PipelineID pipelineID : pipelineIds) {
try {
- pipelineManager.finalizePipeline(pipelineID);
+ Pipeline pipeline = pipelineManager.getPipeline(pipelineID);
+ RatisPipelineUtils
+ .finalizeAndDestroyPipeline(pipelineManager, pipeline, conf, true);
} catch (IOException e) {
LOG.info("Could not finalize pipeline={} for dn={}", pipelineID,
datanodeDetails);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java
index a44ce9d..c467b9e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java
@@ -17,10 +17,10 @@
package org.apache.hadoop.hdds.scm.pipeline;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.PipelineAction;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .PipelineActionsFromDatanode;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -32,16 +32,19 @@ import java.io.IOException;
/**
* Handles pipeline actions from datanode.
*/
-public class PipelineActionHandler implements
- EventHandler<PipelineActionsFromDatanode> {
+public class PipelineActionHandler
+ implements EventHandler<PipelineActionsFromDatanode> {
- public static final Logger LOG = LoggerFactory.getLogger(
- PipelineActionHandler.class);
+ public static final Logger LOG =
+ LoggerFactory.getLogger(PipelineActionHandler.class);
private final PipelineManager pipelineManager;
+ private final Configuration ozoneConf;
- public PipelineActionHandler(PipelineManager pipelineManager) {
+ public PipelineActionHandler(PipelineManager pipelineManager,
+ OzoneConfiguration conf) {
this.pipelineManager = pipelineManager;
+ this.ozoneConf = conf;
}
@Override
@@ -53,7 +56,10 @@ public class PipelineActionHandler implements
try {
pipelineID = PipelineID.
getFromProtobuf(action.getClosePipeline().getPipelineID());
- pipelineManager.finalizePipeline(pipelineID);
+ Pipeline pipeline = pipelineManager.getPipeline(pipelineID);
+ RatisPipelineUtils
+ .finalizeAndDestroyPipeline(pipelineManager, pipeline, ozoneConf,
+ true);
} catch (IOException ioe) {
LOG.error("Could not execute pipeline action={} pipeline={} {}",
action, pipelineID, ioe);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
index 6c31a12..2d4bae1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReport;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
-import org.apache.hadoop.hdds.scm.XceiverClientRatis;
import org.apache.hadoop.hdds.scm.server
.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -76,11 +75,11 @@ public class PipelineReportHandler implements
private void processPipelineReport(PipelineReport report, DatanodeDetails dn)
throws IOException {
PipelineID pipelineID = PipelineID.getFromProtobuf(report.getPipelineID());
- Pipeline pipeline = null;
+ Pipeline pipeline;
try {
pipeline = pipelineManager.getPipeline(pipelineID);
} catch (PipelineNotFoundException e) {
- //TODO: introduce per datanode command for pipeline destroy
+ RatisPipelineUtils.destroyPipeline(dn, pipelineID, conf);
return;
}
@@ -93,14 +92,9 @@ public class PipelineReportHandler implements
} else if (pipeline.isClosed()) {
int numContainers = pipelineManager.getNumberOfContainers(pipelineID);
if (numContainers == 0) {
- // remove the pipeline from the pipeline manager
- pipelineManager.removePipeline(pipelineID);
// since all the containers have been closed the pipeline can be
// destroyed
- try (XceiverClientRatis client =
- XceiverClientRatis.newXceiverClientRatis(pipeline, conf)) {
- client.destroyPipeline();
- }
+ RatisPipelineUtils.destroyPipeline(pipelineManager, pipeline, conf);
}
} else {
// In OPEN state case just report the datanode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
index 85790b2..20dfa03 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
@@ -242,12 +242,6 @@ class PipelineStateMap {
String.format("Pipeline with %s is not yet closed", pipelineID));
}
- Set<ContainerID> containerIDs = pipeline2container.get(pipelineID);
- if (containerIDs.size() != 0) {
- throw new IOException(
- String.format("Pipeline with %s is not empty", pipelineID));
- }
-
pipelineMap.remove(pipelineID);
pipeline2container.remove(pipelineID);
return pipeline;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index 590cd27..3a5bd41 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.XceiverClientRatis;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom;
import org.apache.hadoop.hdds.scm.node.NodeManager;
@@ -133,12 +132,7 @@ public class RatisPipelineProvider implements PipelineProvider {
.build();
}
- private void initializePipeline(Pipeline pipeline)
- throws IOException {
- // TODO: remove old code in XceiverClientRatis#newXceiverClientRatis
- try (XceiverClientRatis client =
- XceiverClientRatis.newXceiverClientRatis(pipeline, conf)) {
- client.createPipeline();
- }
+ private void initializePipeline(Pipeline pipeline) throws IOException {
+ RatisPipelineUtils.createPipeline(pipeline, conf);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
new file mode 100644
index 0000000..dd79962
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.io.MultipleIOException;
+import org.apache.ratis.RatisHelper;
+import org.apache.ratis.client.RaftClient;
+import org.apache.ratis.protocol.RaftGroup;
+import org.apache.ratis.protocol.RaftGroupId;
+import org.apache.ratis.protocol.RaftPeer;
+import org.apache.ratis.retry.RetryPolicy;
+import org.apache.ratis.rpc.SupportedRpcType;
+import org.apache.ratis.util.CheckedBiConsumer;
+import org.apache.ratis.util.TimeDuration;
+import org.apache.ratis.util.TimeoutScheduler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Utility class for Ratis pipelines. Contains methods to create and destroy
+ * ratis pipelines.
+ */
+public final class RatisPipelineUtils {
+
+ private static TimeoutScheduler timeoutScheduler =
+ TimeoutScheduler.newInstance(1);
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RatisPipelineUtils.class);
+
+ private RatisPipelineUtils() {
+ }
+
+ /**
+ * Sends ratis command to create pipeline on all the datanodes.
+ * @param pipeline - Pipeline to be created
+ * @param ozoneConf - Ozone Confinuration
+ * @throws IOException if creation fails
+ */
+ public static void createPipeline(Pipeline pipeline, Configuration ozoneConf)
+ throws IOException {
+ final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
+ LOG.debug("creating pipeline:{} with {}", pipeline.getId(), group);
+ callRatisRpc(pipeline.getNodes(), ozoneConf,
+ (raftClient, peer) -> raftClient.groupAdd(group, peer.getId()));
+ }
+
+ /**
+ * Removes pipeline from SCM. Sends ratis command to destroy pipeline on all
+ * the datanodes.
+ * @param pipelineManager - SCM pipeline manager
+ * @param pipeline - Pipeline to be destroyed
+ * @param ozoneConf - Ozone configuration
+ * @throws IOException
+ */
+ static void destroyPipeline(PipelineManager pipelineManager,
+ Pipeline pipeline, Configuration ozoneConf) throws IOException {
+ final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
+ LOG.debug("destroying pipeline:{} with {}", pipeline.getId(), group);
+ // remove the pipeline from the pipeline manager
+ pipelineManager.removePipeline(pipeline.getId());
+ for (DatanodeDetails dn : pipeline.getNodes()) {
+ destroyPipeline(dn, pipeline.getId(), ozoneConf);
+ }
+ }
+
+ /**
+ * Finalizes pipeline in the SCM. Removes pipeline and sends ratis command to
+ * destroy pipeline on the datanodes immediately or after timeout based on the
+ * value of onTimeout parameter.
+ * @param pipelineManager - SCM pipeline manager
+ * @param pipeline - Pipeline to be destroyed
+ * @param ozoneConf - Ozone Configuration
+ * @param onTimeout - if true pipeline is removed and destroyed on datanodes
+ * after timeout
+ * @throws IOException
+ */
+ public static void finalizeAndDestroyPipeline(PipelineManager pipelineManager,
+ Pipeline pipeline, Configuration ozoneConf, boolean onTimeout)
+ throws IOException {
+ final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
+ LOG.debug("destroying pipeline:{} with {}", pipeline.getId(), group);
+ pipelineManager.finalizePipeline(pipeline.getId());
+ if (onTimeout) {
+ long pipelineDestroyTimeoutInMillis = ozoneConf
+ .getTimeDuration(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT,
+ ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ TimeDuration timeoutDuration = TimeDuration
+ .valueOf(pipelineDestroyTimeoutInMillis, TimeUnit.MILLISECONDS);
+ timeoutScheduler.onTimeout(timeoutDuration,
+ () -> destroyPipeline(pipelineManager, pipeline, ozoneConf), LOG,
+ () -> String.format("Destroy pipeline failed for pipeline:%s with %s",
+ pipeline.getId(), group));
+ } else {
+ destroyPipeline(pipelineManager, pipeline, ozoneConf);
+ }
+ }
+
+ /**
+ * Sends ratis command to destroy pipeline on the given datanode.
+ * @param dn - Datanode on which pipeline needs to be destroyed
+ * @param pipelineID - ID of pipeline to be destroyed
+ * @param ozoneConf - Ozone configuration
+ * @throws IOException
+ */
+ static void destroyPipeline(DatanodeDetails dn, PipelineID pipelineID,
+ Configuration ozoneConf) throws IOException {
+ final String rpcType = ozoneConf
+ .get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
+ ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
+ final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
+ final RaftPeer p = RatisHelper.toRaftPeer(dn);
+ RaftClient client = RatisHelper
+ .newRaftClient(SupportedRpcType.valueOfIgnoreCase(rpcType), p,
+ retryPolicy);
+ client
+ .groupRemove(RaftGroupId.valueOf(pipelineID.getId()), true, p.getId());
+ }
+
+ private static void callRatisRpc(List<DatanodeDetails> datanodes,
+ Configuration ozoneConf,
+ CheckedBiConsumer<RaftClient, RaftPeer, IOException> rpc)
+ throws IOException {
+ if (datanodes.isEmpty()) {
+ return;
+ }
+
+ final String rpcType = ozoneConf
+ .get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
+ ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
+ final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
+ final List<IOException> exceptions =
+ Collections.synchronizedList(new ArrayList<>());
+
+ datanodes.parallelStream().forEach(d -> {
+ final RaftPeer p = RatisHelper.toRaftPeer(d);
+ try (RaftClient client = RatisHelper
+ .newRaftClient(SupportedRpcType.valueOfIgnoreCase(rpcType), p,
+ retryPolicy)) {
+ rpc.accept(client, p);
+ } catch (IOException ioe) {
+ exceptions.add(
+ new IOException("Failed invoke Ratis rpc " + rpc + " for " + d,
+ ioe));
+ }
+ });
+ if (!exceptions.isEmpty()) {
+ throw MultipleIOException.createIOException(exceptions);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 08ee382..fe00b54 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -223,7 +223,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
NewNodeHandler newNodeHandler = new NewNodeHandler();
StaleNodeHandler staleNodeHandler =
- new StaleNodeHandler(scmNodeManager, pipelineManager);
+ new StaleNodeHandler(scmNodeManager, pipelineManager, conf);
DeadNodeHandler deadNodeHandler = new DeadNodeHandler(scmNodeManager,
containerManager);
ContainerActionsHandler actionsHandler = new ContainerActionsHandler();
@@ -239,7 +239,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
pipelineManager, containerManager);
PipelineActionHandler pipelineActionHandler =
- new PipelineActionHandler(pipelineManager);
+ new PipelineActionHandler(pipelineManager, conf);
long watcherTimeout =
conf.getTimeDuration(ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
index 8fbe7fb..debde65 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
@@ -17,12 +17,17 @@
package org.apache.hadoop.hdds.scm;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ClosePipelineInfo;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineActionsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReport;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode;
import org.apache.hadoop.hdds.scm.server
.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
import org.apache.hadoop.hdds.scm.container.ContainerID;
@@ -305,19 +310,34 @@ public final class TestUtils {
return PipelineReportsProto.newBuilder().build();
}
- public static PipelineReportFromDatanode getRandomPipelineReportFromDatanode(
- DatanodeDetails dn,
- org.apache.hadoop.hdds.scm.pipeline.PipelineID... pipelineIDs) {
+ public static PipelineReportFromDatanode getPipelineReportFromDatanode(
+ DatanodeDetails dn, PipelineID... pipelineIDs) {
PipelineReportsProto.Builder reportBuilder =
PipelineReportsProto.newBuilder();
- for (org.apache.hadoop.hdds.scm.pipeline.PipelineID pipelineID :
- pipelineIDs) {
+ for (PipelineID pipelineID : pipelineIDs) {
reportBuilder.addPipelineReport(
PipelineReport.newBuilder().setPipelineID(pipelineID.getProtobuf()));
}
return new PipelineReportFromDatanode(dn, reportBuilder.build());
}
+ public static PipelineActionsFromDatanode getPipelineActionFromDatanode(
+ DatanodeDetails dn, PipelineID... pipelineIDs) {
+ PipelineActionsProto.Builder actionsProtoBuilder =
+ PipelineActionsProto.newBuilder();
+ for (PipelineID pipelineID : pipelineIDs) {
+ ClosePipelineInfo closePipelineInfo =
+ ClosePipelineInfo.newBuilder().setPipelineID(pipelineID.getProtobuf())
+ .setReason(ClosePipelineInfo.Reason.PIPELINE_FAILED)
+ .setDetailedReason("").build();
+ actionsProtoBuilder.addPipelineActions(PipelineAction.newBuilder()
+ .setClosePipeline(closePipelineInfo)
+ .setAction(PipelineAction.Action.CLOSE)
+ .build());
+ }
+ return new PipelineActionsFromDatanode(dn, actionsProtoBuilder.build());
+ }
+
/**
* Creates container report with the given ContainerInfo(s).
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
index 4f8943e..356f005 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
@@ -17,22 +17,32 @@
*/
package org.apache.hadoop.hdds.scm.pipeline;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.junit.AfterClass;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.util.List;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
@@ -43,35 +53,36 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.R
*/
public class TestPipelineClose {
- private static MiniOzoneCluster cluster;
- private static OzoneConfiguration conf;
- private static StorageContainerManager scm;
- private static ContainerWithPipeline ratisContainer1;
- private static ContainerWithPipeline ratisContainer2;
- private static ContainerManager containerManager;
- private static PipelineManager pipelineManager;
+ private MiniOzoneCluster cluster;
+ private OzoneConfiguration conf;
+ private StorageContainerManager scm;
+ private ContainerWithPipeline ratisContainer;
+ private ContainerManager containerManager;
+ private PipelineManager pipelineManager;
+ private long pipelineDestroyTimeoutInMillis;
/**
* Create a MiniDFSCluster for testing.
*
* @throws IOException
*/
- @BeforeClass
- public static void init() throws Exception {
+ @Before
+ public void init() throws Exception {
conf = new OzoneConfiguration();
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(6).build();
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).build();
+ conf.setTimeDuration(HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL, 1000,
+ TimeUnit.MILLISECONDS);
+ pipelineDestroyTimeoutInMillis = 5000;
+ conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT,
+ pipelineDestroyTimeoutInMillis, TimeUnit.MILLISECONDS);
cluster.waitForClusterToBeReady();
scm = cluster.getStorageContainerManager();
containerManager = scm.getContainerManager();
pipelineManager = scm.getPipelineManager();
- ContainerInfo containerInfo1 = containerManager
+ ContainerInfo containerInfo = containerManager
.allocateContainer(RATIS, THREE, "testOwner");
- ratisContainer1 = new ContainerWithPipeline(containerInfo1,
- pipelineManager.getPipeline(containerInfo1.getPipelineID()));
- ContainerInfo containerInfo2 = containerManager
- .allocateContainer(RATIS, THREE, "testOwner");
- ratisContainer2 = new ContainerWithPipeline(containerInfo2,
- pipelineManager.getPipeline(containerInfo2.getPipelineID()));
+ ratisContainer = new ContainerWithPipeline(containerInfo,
+ pipelineManager.getPipeline(containerInfo.getPipelineID()));
pipelineManager = scm.getPipelineManager();
// At this stage, there should be 2 pipeline one with 1 open container each.
// Try closing the both the pipelines, one with a closed container and
@@ -81,8 +92,8 @@ public class TestPipelineClose {
/**
* Shutdown MiniDFSCluster.
*/
- @AfterClass
- public static void shutdown() {
+ @After
+ public void shutdown() {
if (cluster != null) {
cluster.shutdown();
}
@@ -91,9 +102,9 @@ public class TestPipelineClose {
@Test
public void testPipelineCloseWithClosedContainer() throws IOException {
Set<ContainerID> set = pipelineManager
- .getContainersInPipeline(ratisContainer1.getPipeline().getId());
+ .getContainersInPipeline(ratisContainer.getPipeline().getId());
- ContainerID cId = ratisContainer1.getContainerInfo().containerID();
+ ContainerID cId = ratisContainer.getContainerInfo().containerID();
Assert.assertEquals(1, set.size());
set.forEach(containerID -> Assert.assertEquals(containerID, cId));
@@ -105,16 +116,16 @@ public class TestPipelineClose {
.updateContainerState(cId, HddsProtos.LifeCycleEvent.CLOSE);
Set<ContainerID> setClosed = pipelineManager
- .getContainersInPipeline(ratisContainer1.getPipeline().getId());
+ .getContainersInPipeline(ratisContainer.getPipeline().getId());
Assert.assertEquals(0, setClosed.size());
- pipelineManager.finalizePipeline(ratisContainer1.getPipeline().getId());
+ pipelineManager.finalizePipeline(ratisContainer.getPipeline().getId());
Pipeline pipeline1 = pipelineManager
- .getPipeline(ratisContainer1.getPipeline().getId());
+ .getPipeline(ratisContainer.getPipeline().getId());
Assert.assertEquals(Pipeline.PipelineState.CLOSED,
pipeline1.getPipelineState());
pipelineManager.removePipeline(pipeline1.getId());
- for (DatanodeDetails dn : ratisContainer1.getPipeline().getNodes()) {
+ for (DatanodeDetails dn : ratisContainer.getPipeline().getNodes()) {
// Assert that the pipeline has been removed from Node2PipelineMap as well
Assert.assertEquals(scm.getScmNodeManager().getPipelines(
dn).size(), 0);
@@ -125,17 +136,80 @@ public class TestPipelineClose {
public void testPipelineCloseWithOpenContainer() throws IOException,
TimeoutException, InterruptedException {
Set<ContainerID> setOpen = pipelineManager.getContainersInPipeline(
- ratisContainer2.getPipeline().getId());
+ ratisContainer.getPipeline().getId());
Assert.assertEquals(1, setOpen.size());
- ContainerID cId2 = ratisContainer2.getContainerInfo().containerID();
- pipelineManager.finalizePipeline(ratisContainer2.getPipeline().getId());
+ ContainerID cId2 = ratisContainer.getContainerInfo().containerID();
+ pipelineManager.finalizePipeline(ratisContainer.getPipeline().getId());
Assert.assertEquals(Pipeline.PipelineState.CLOSED,
pipelineManager.getPipeline(
- ratisContainer2.getPipeline().getId()).getPipelineState());
+ ratisContainer.getPipeline().getId()).getPipelineState());
Pipeline pipeline2 = pipelineManager
- .getPipeline(ratisContainer2.getPipeline().getId());
+ .getPipeline(ratisContainer.getPipeline().getId());
Assert.assertEquals(Pipeline.PipelineState.CLOSED,
pipeline2.getPipelineState());
}
+
+ @Test
+ public void testPipelineCloseWithPipelineAction() throws Exception {
+ List<DatanodeDetails> dns = ratisContainer.getPipeline().getNodes();
+ PipelineActionsFromDatanode
+ pipelineActionsFromDatanode = TestUtils
+ .getPipelineActionFromDatanode(dns.get(0),
+ ratisContainer.getPipeline().getId());
+ // send closing action for pipeline
+ PipelineActionHandler pipelineActionHandler =
+ new PipelineActionHandler(pipelineManager, conf);
+ pipelineActionHandler
+ .onMessage(pipelineActionsFromDatanode, new EventQueue());
+ Thread.sleep((int) (pipelineDestroyTimeoutInMillis * 1.2));
+ OzoneContainer ozoneContainer =
+ cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+ .getContainer();
+ List<PipelineReport> pipelineReports =
+ ozoneContainer.getPipelineReport().getPipelineReportList();
+ for (PipelineReport pipelineReport : pipelineReports) {
+ // ensure the pipeline is not reported by any dn
+ Assert.assertNotEquals(
+ PipelineID.getFromProtobuf(pipelineReport.getPipelineID()),
+ ratisContainer.getPipeline().getId());
+ }
+
+ try {
+ pipelineManager.getPipeline(ratisContainer.getPipeline().getId());
+ Assert.fail("Pipeline should not exist in SCM");
+ } catch (PipelineNotFoundException e) {
+ }
+ }
+
+ @Test
+ public void testPipelineCloseWithPipelineReport() throws IOException {
+ Pipeline pipeline = ratisContainer.getPipeline();
+ pipelineManager.finalizePipeline(pipeline.getId());
+ // remove pipeline from SCM
+ pipelineManager.removePipeline(pipeline.getId());
+
+ for (DatanodeDetails dn : pipeline.getNodes()) {
+ PipelineReportFromDatanode pipelineReport =
+ TestUtils.getPipelineReportFromDatanode(dn, pipeline.getId());
+ PipelineReportHandler pipelineReportHandler =
+ new PipelineReportHandler(pipelineManager, conf);
+ // on receiving pipeline report for the pipeline, pipeline report handler
+ // should destroy the pipeline for the dn
+ pipelineReportHandler.onMessage(pipelineReport, new EventQueue());
+ }
+
+ OzoneContainer ozoneContainer =
+ cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+ .getContainer();
+ List<PipelineReport> pipelineReports =
+ ozoneContainer.getPipelineReport().getPipelineReportList();
+ for (PipelineReport pipelineReport : pipelineReports) {
+ // pipeline should not be reported by any dn
+ Assert.assertNotEquals(
+ PipelineID.getFromProtobuf(pipelineReport.getPipelineID()),
+ ratisContainer.getPipeline().getId());
+ }
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
index 0f5692e..d404b84 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
@@ -323,15 +323,6 @@ public class TestPipelineStateManager {
// close the pipeline
stateManager.finalizePipeline(pipeline.getId());
-
- try {
- stateManager.removePipeline(pipeline.getId());
- Assert.fail("Pipeline should not have been removed");
- } catch (IOException e) {
- // can not remove a pipeline which already has containers
- Assert.assertTrue(e.getMessage().contains("not empty"));
- }
-
// remove containers and then remove the pipeline
removePipeline(pipeline);
}
@@ -423,11 +414,6 @@ public class TestPipelineStateManager {
private void removePipeline(Pipeline pipeline) throws IOException {
stateManager.finalizePipeline(pipeline.getId());
- Set<ContainerID> containerIDs =
- stateManager.getContainers(pipeline.getId());
- for (ContainerID containerID : containerIDs) {
- stateManager.removeContainerFromPipeline(pipeline.getId(), containerID);
- }
stateManager.removePipeline(pipeline.getId());
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
index 4487766..ad53015 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -148,7 +148,7 @@ public class TestSCMPipelineManager {
new PipelineReportHandler(pipelineManager, conf);
for (DatanodeDetails dn: pipeline.getNodes()) {
PipelineReportFromDatanode pipelineReportFromDatanode =
- TestUtils.getRandomPipelineReportFromDatanode(dn, pipeline.getId());
+ TestUtils.getPipelineReportFromDatanode(dn, pipeline.getId());
// pipeline is not healthy until all dns report
Assert.assertFalse(
pipelineManager.getPipeline(pipeline.getId()).isHealthy());
@@ -168,7 +168,7 @@ public class TestSCMPipelineManager {
for (DatanodeDetails dn: pipeline.getNodes()) {
PipelineReportFromDatanode pipelineReportFromDatanode =
- TestUtils.getRandomPipelineReportFromDatanode(dn, pipeline.getId());
+ TestUtils.getPipelineReportFromDatanode(dn, pipeline.getId());
// pipeline report for a closed pipeline should destroy the pipeline
// and remove it from the pipeline manager
pipelineReportHandler
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb915f3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java
new file mode 100644
index 0000000..280cf90
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.RatisPipelineUtils;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests Freon with Pipeline destroy.
+ */
+public class TestFreonWithPipelineDestroy {
+
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration conf;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setHbProcessorInterval(1000)
+ .setHbInterval(1000)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testRestart() throws Exception {
+ startFreon();
+ destroyPipeline();
+ startFreon();
+ }
+
+ private void startFreon() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(1);
+ randomKeyGenerator.setNumOfBuckets(1);
+ randomKeyGenerator.setNumOfKeys(1);
+ randomKeyGenerator.setType(ReplicationType.RATIS);
+ randomKeyGenerator.setFactor(ReplicationFactor.THREE);
+ randomKeyGenerator.setKeySize(20971520);
+ randomKeyGenerator.setValidateWrites(true);
+ randomKeyGenerator.call();
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
+ Assert.assertEquals(0,
+ randomKeyGenerator.getUnsuccessfulValidationCount());
+ }
+
+ private void destroyPipeline() throws Exception {
+ XceiverServerSpi server =
+ cluster.getHddsDatanodes().get(0).getDatanodeStateMachine().
+ getContainer().getWriteChannel();
+ StorageContainerDatanodeProtocolProtos.PipelineReport report =
+ server.getPipelineReport().get(0);
+ PipelineID id = PipelineID.getFromProtobuf(report.getPipelineID());
+ PipelineManager pipelineManager =
+ cluster.getStorageContainerManager().getPipelineManager();
+ Pipeline pipeline = pipelineManager.getPipeline(id);
+ RatisPipelineUtils
+ .finalizeAndDestroyPipeline(pipelineManager, pipeline, conf, false);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: HDFS-12284. addendum to
HDFS-12284. Contributed by Inigo Goiri.
Posted by br...@apache.org.
HDFS-12284. addendum to HDFS-12284. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6647f17c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6647f17c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6647f17c
Branch: refs/heads/HDFS-13891
Commit: 6647f17c3e67d225b5af51a2cee6da0cc3ce1686
Parents: bb40d22
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Wed Nov 7 07:37:02 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:34:52 2018 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6647f17c/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
index 9f515bc..947c91a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
@@ -36,7 +36,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependencies>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDDS-164. Add unit test for
HddsDatanodeService. Contributed by Yiqun Lin.
Posted by br...@apache.org.
HDDS-164. Add unit test for HddsDatanodeService. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0123e519
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0123e519
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0123e519
Branch: refs/heads/HDFS-13891
Commit: 0123e5197b4e355dd4a5d97962ec64d3768a8479
Parents: 4ecdcc9
Author: Márton Elek <el...@apache.org>
Authored: Tue Nov 20 15:49:16 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Tue Nov 20 15:49:48 2018 +0100
----------------------------------------------------------------------
.../hadoop/ozone/TestHddsDatanodeService.java | 95 ++++++++++++++++++++
1 file changed, 95 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0123e519/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
new file mode 100644
index 0000000..18cd00c
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.ServicePlugin;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for {@link HddsDatanodeService}.
+ */
+public class TestHddsDatanodeService {
+ private File testDir;
+ private Configuration conf;
+ private HddsDatanodeService service;
+
+ @Before
+ public void setUp() {
+ testDir = GenericTestUtils.getRandomizedTestDir();
+ conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_ENABLED, true);
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath());
+ conf.setClass(OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY, MockService.class,
+ ServicePlugin.class);
+
+ String volumeDir = testDir + "/disk1";
+ conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, volumeDir);
+ }
+
+ @After
+ public void tearDown() {
+ FileUtil.fullyDelete(testDir);
+ }
+
+ @Test
+ public void testStartup() throws IOException {
+ service = HddsDatanodeService.createHddsDatanodeService(conf);
+ service.start(null);
+ service.join();
+
+ assertNotNull(service.getDatanodeDetails());
+ assertNotNull(service.getDatanodeDetails().getHostName());
+ assertFalse(service.getDatanodeStateMachine().isDaemonStopped());
+
+ service.stop();
+ service.close();
+ }
+
+ static class MockService implements ServicePlugin {
+
+ @Override
+ public void close() throws IOException {
+ // Do nothing
+ }
+
+ @Override
+ public void start(Object arg0) {
+ // Do nothing
+ }
+
+ @Override
+ public void stop() {
+ // Do nothing
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-8833. Avoid potential integer
overflow when computing fair shares. Contributed by liyakun.
Posted by br...@apache.org.
YARN-8833. Avoid potential integer overflow when computing fair shares. Contributed by liyakun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d027a24f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d027a24f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d027a24f
Branch: refs/heads/HDFS-13891
Commit: d027a24f0349b60efa5125c330058f123771748f
Parents: e56d9f2
Author: Weiwei Yang <ww...@apache.org>
Authored: Sun Nov 18 23:18:26 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Sun Nov 18 23:18:26 2018 +0800
----------------------------------------------------------------------
.../scheduler/fair/policies/ComputeFairShares.java | 15 +++++++++------
.../scheduler/fair/TestComputeFairShares.java | 17 +++++++++++++++++
2 files changed, 26 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d027a24f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
index 0a21b02..3fe0c68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
@@ -145,7 +145,7 @@ public class ComputeFairShares {
double right = rMax;
for (int i = 0; i < COMPUTE_FAIR_SHARES_ITERATIONS; i++) {
double mid = (left + right) / 2.0;
- int plannedResourceUsed = resourceUsedWithWeightToResourceRatio(
+ long plannedResourceUsed = resourceUsedWithWeightToResourceRatio(
mid, schedulables, type);
if (plannedResourceUsed == totalResource) {
right = mid;
@@ -174,11 +174,14 @@ public class ComputeFairShares {
* Compute the resources that would be used given a weight-to-resource ratio
* w2rRatio, for use in the computeFairShares algorithm as described in #
*/
- private static int resourceUsedWithWeightToResourceRatio(double w2rRatio,
+ private static long resourceUsedWithWeightToResourceRatio(double w2rRatio,
Collection<? extends Schedulable> schedulables, String type) {
- int resourcesTaken = 0;
+ long resourcesTaken = 0;
for (Schedulable sched : schedulables) {
- int share = computeShare(sched, w2rRatio, type);
+ long share = computeShare(sched, w2rRatio, type);
+ if (Long.MAX_VALUE - resourcesTaken < share) {
+ return Long.MAX_VALUE;
+ }
resourcesTaken += share;
}
return resourcesTaken;
@@ -188,12 +191,12 @@ public class ComputeFairShares {
* Compute the resources assigned to a Schedulable given a particular
* weight-to-resource ratio w2rRatio.
*/
- private static int computeShare(Schedulable sched, double w2rRatio,
+ private static long computeShare(Schedulable sched, double w2rRatio,
String type) {
double share = sched.getWeight() * w2rRatio;
share = Math.max(share, sched.getMinShare().getResourceValue(type));
share = Math.min(share, sched.getMaxShare().getResourceValue(type));
- return (int) share;
+ return (long) share;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d027a24f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java
index c3bcb3b..b1666d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java
@@ -19,7 +19,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.junit.Assert;
@@ -205,4 +208,18 @@ public class TestComputeFairShares {
Assert.assertEquals(shares[i], scheds.get(i).getFairShare().getVirtualCores());
}
}
+
+ /**
+ * Test computeShares will not enter into infinite loop.
+ */
+ @Test(timeout = 10000)
+ public void testResourceUsedWithWeightToResourceRatio() {
+ Collection<Schedulable> schedulables = new ArrayList<>();
+ schedulables.add(new FakeSchedulable(Integer.MAX_VALUE));
+ schedulables.add(new FakeSchedulable(Integer.MAX_VALUE));
+
+ Resource totalResource = Resource.newInstance(Integer.MAX_VALUE, 0);
+ ComputeFairShares.computeShares(
+ schedulables, totalResource, ResourceInformation.MEMORY_URI);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDDS-801. Quasi close the
container when close is not executed via Ratis. Contributed by Nanda kumar.
Posted by br...@apache.org.
HDDS-801. Quasi close the container when close is not executed via Ratis.
Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4d96400
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4d96400
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4d96400
Branch: refs/heads/HDFS-13891
Commit: c4d96400280d9a1595b82435297dbbed2668c485
Parents: ad5256e
Author: Nanda kumar <na...@apache.org>
Authored: Fri Nov 16 23:07:45 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Fri Nov 16 23:11:33 2018 +0530
----------------------------------------------------------------------
.../main/proto/DatanodeContainerProtocol.proto | 4 +-
.../container/common/impl/ContainerData.java | 17 +-
.../container/common/impl/HddsDispatcher.java | 14 +-
.../container/common/interfaces/Container.java | 15 +-
.../container/common/interfaces/Handler.java | 68 +++++-
.../common/report/ContainerReportPublisher.java | 3 +-
.../statemachine/DatanodeStateMachine.java | 2 +-
.../common/statemachine/StateContext.java | 20 --
.../CloseContainerCommandHandler.java | 138 ++++++------
.../states/endpoint/RegisterEndpointTask.java | 2 +-
.../transport/server/XceiverServerGrpc.java | 5 +
.../transport/server/XceiverServerSpi.java | 7 +
.../server/ratis/XceiverServerRatis.java | 18 +-
.../container/keyvalue/KeyValueContainer.java | 41 +++-
.../container/keyvalue/KeyValueHandler.java | 119 +++++++----
.../ozoneimpl/ContainerController.java | 124 +++++++++++
.../container/ozoneimpl/OzoneContainer.java | 124 +++++------
.../DownloadAndImportReplicator.java | 16 +-
.../OnDemandContainerReplicationSource.java | 10 +-
.../commands/CloseContainerCommand.java | 12 +-
.../StorageContainerDatanodeProtocol.proto | 8 +-
.../common/impl/TestHddsDispatcher.java | 26 ++-
.../common/interfaces/TestHandler.java | 16 +-
.../TestCloseContainerCommandHandler.java | 210 +++++++++++++++++++
.../container/keyvalue/TestKeyValueHandler.java | 8 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 2 +-
.../container/CloseContainerEventHandler.java | 4 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 3 +-
.../ozone/container/common/TestEndPoint.java | 8 +-
.../dist/src/main/smoketest/basic/basic.robot | 2 +-
.../hadoop/ozone/TestMiniOzoneCluster.java | 18 +-
.../rpc/TestCloseContainerHandlingByClient.java | 26 +--
.../TestCloseContainerByPipeline.java | 80 ++++---
.../TestCloseContainerHandler.java | 3 +-
.../container/metrics/TestContainerMetrics.java | 22 +-
.../container/server/TestContainerServer.java | 26 ++-
.../genesis/BenchMarkDatanodeDispatcher.java | 19 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 3 +-
38 files changed, 875 insertions(+), 368 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 5eecdcb..ac2d277 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -229,8 +229,8 @@ message ContainerDataProto {
enum State {
OPEN = 1;
CLOSING = 2;
- CLOSED = 3;
- QUASI_CLOSED = 4;
+ QUASI_CLOSED =3;
+ CLOSED = 4;
UNHEALTHY = 5;
INVALID = 6;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
index ad199f0..6f51517 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
@@ -237,11 +237,26 @@ public abstract class ContainerData {
* checks if the container is closed.
* @return - boolean
*/
- public synchronized boolean isClosed() {
+ public synchronized boolean isClosed() {
return ContainerDataProto.State.CLOSED == state;
}
/**
+ * checks if the container is quasi closed.
+ * @return - boolean
+ */
+ public synchronized boolean isQuasiClosed() {
+ return ContainerDataProto.State.QUASI_CLOSED == state;
+ }
+
+ /**
+ * Marks this container as quasi closed.
+ */
+ public synchronized void quasiCloseContainer() {
+ setState(ContainerDataProto.State.QUASI_CLOSED);
+ }
+
+ /**
* Marks this container as closed.
*/
public synchronized void closeContainer() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index c52d973..29f9b20 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.container.common.impl;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -76,18 +75,14 @@ public class HddsDispatcher implements ContainerDispatcher {
* XceiverServerHandler.
*/
public HddsDispatcher(Configuration config, ContainerSet contSet,
- VolumeSet volumes, StateContext context) {
+ VolumeSet volumes, Map<ContainerType, Handler> handlers,
+ StateContext context, ContainerMetrics metrics) {
this.conf = config;
this.containerSet = contSet;
this.volumeSet = volumes;
this.context = context;
- this.handlers = Maps.newHashMap();
- this.metrics = ContainerMetrics.create(conf);
- for (ContainerType containerType : ContainerType.values()) {
- handlers.put(containerType,
- Handler.getHandlerForContainerType(
- containerType, conf, containerSet, volumeSet, metrics));
- }
+ this.handlers = handlers;
+ this.metrics = metrics;
this.containerCloseThreshold = conf.getFloat(
HddsConfigKeys.HDDS_CONTAINER_CLOSE_THRESHOLD,
HddsConfigKeys.HDDS_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
@@ -347,6 +342,7 @@ public class HddsDispatcher implements ContainerDispatcher {
}
}
+ @VisibleForTesting
public Container getContainer(long containerID) {
return containerSet.getContainer(containerID);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
index 65147cc..2bda747 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
@@ -82,11 +82,24 @@ public interface Container<CONTAINERDATA extends ContainerData> extends RwLock {
ContainerProtos.ContainerDataProto.State getContainerState();
/**
- * Closes a open container, if it is already closed or does not exist a
+ * Marks the container for closing. Moves the container to CLOSING state.
+ */
+ void markContainerForClose() throws StorageContainerException;
+
+ /**
+ * Quasi Closes a open container, if it is already closed or does not exist a
* StorageContainerException is thrown.
*
* @throws StorageContainerException
*/
+ void quasiClose() throws StorageContainerException;
+
+ /**
+ * Closes a open/quasi closed container, if it is already closed or does not
+ * exist a StorageContainerException is thrown.
+ *
+ * @throws StorageContainerException
+ */
void close() throws StorageContainerException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 53e1c68..19e61c4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -29,8 +29,12 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerType;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
@@ -47,26 +51,47 @@ public abstract class Handler {
protected String scmID;
protected final ContainerMetrics metrics;
- protected Handler(Configuration config, ContainerSet contSet,
- VolumeSet volumeSet, ContainerMetrics containerMetrics) {
- conf = config;
- containerSet = contSet;
+ private final StateContext context;
+
+ protected Handler(Configuration config, StateContext context,
+ ContainerSet contSet, VolumeSet volumeSet,
+ ContainerMetrics containerMetrics) {
+ this.conf = config;
+ this.context = context;
+ this.containerSet = contSet;
this.volumeSet = volumeSet;
this.metrics = containerMetrics;
}
- public static Handler getHandlerForContainerType(ContainerType containerType,
- Configuration config, ContainerSet contSet, VolumeSet volumeSet,
- ContainerMetrics metrics) {
+ public static Handler getHandlerForContainerType(
+ final ContainerType containerType, final Configuration config,
+ final StateContext context, final ContainerSet contSet,
+ final VolumeSet volumeSet, final ContainerMetrics metrics) {
switch (containerType) {
case KeyValueContainer:
- return new KeyValueHandler(config, contSet, volumeSet, metrics);
+ return new KeyValueHandler(config, context, contSet, volumeSet, metrics);
default:
throw new IllegalArgumentException("Handler for ContainerType: " +
containerType + "doesn't exist.");
}
}
+ /**
+ * This should be called whenever there is state change. It will trigger
+ * an ICR to SCM.
+ *
+ * @param container Container for which ICR has to be sent
+ */
+ protected void sendICR(final Container container)
+ throws StorageContainerException {
+ IncrementalContainerReportProto icr = IncrementalContainerReportProto
+ .newBuilder()
+ .addReport(container.getContainerReport())
+ .build();
+ context.addReport(icr);
+ context.getParent().triggerHeartbeat();
+ }
+
public abstract ContainerCommandResponseProto handle(
ContainerCommandRequestProto msg, Container container);
@@ -80,6 +105,33 @@ public abstract class Handler {
TarContainerPacker packer)
throws IOException;
+ /**
+ * Marks the container for closing. Moves the container to CLOSING state.
+ *
+ * @param container container to update
+ * @throws IOException in case of exception
+ */
+ public abstract void markContainerForClose(Container container)
+ throws IOException;
+
+ /**
+ * Moves the Container to QUASI_CLOSED state.
+ *
+ * @param container container to be quasi closed
+ * @throws IOException
+ */
+ public abstract void quasiCloseContainer(Container container)
+ throws IOException;
+
+ /**
+ * Moves the Container to CLOSED state.
+ *
+ * @param container container to be closed
+ * @throws IOException
+ */
+ public abstract void closeContainer(Container container)
+ throws IOException;
+
public void setScmID(String scmId) {
this.scmID = scmId;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java
index ccb9a9a..b92e3b0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java
@@ -80,6 +80,7 @@ public class ContainerReportPublisher extends
@Override
protected ContainerReportsProto getReport() throws IOException {
- return getContext().getParent().getContainer().getContainerReport();
+ return getContext().getParent().getContainer()
+ .getController().getContainerReport();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 12c33ff..1d87071 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -99,7 +99,7 @@ public class DatanodeStateMachine implements Closeable {
ContainerReplicator replicator =
new DownloadAndImportReplicator(container.getContainerSet(),
- container.getDispatcher(),
+ container.getController(),
new SimpleContainerDownloader(conf), new TarContainerPacker());
supervisor =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index e928824..953f730 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -59,8 +59,6 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
-import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
-
/**
* Current Context of State Machine.
*/
@@ -116,24 +114,6 @@ public class StateContext {
}
/**
- * Get the container server port.
- * @return The container server port if available, return -1 if otherwise
- */
- public int getContainerPort() {
- return parent == null ?
- INVALID_PORT : parent.getContainer().getContainerServerPort();
- }
-
- /**
- * Gets the Ratis Port.
- * @return int , return -1 if not valid.
- */
- public int getRatisPort() {
- return parent == null ?
- INVALID_PORT : parent.getContainer().getRatisContainerServerPort();
- }
-
- /**
* Returns true if we are entering a new state.
*
* @return boolean
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
index 0838be2..9d25812 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
@@ -16,21 +16,20 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto
+ .ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.CloseContainerCommandProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
- ContainerDataProto.State;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.statemachine
.SCMConnectionManager;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
@@ -38,17 +37,19 @@ import org.apache.ratis.protocol.NotLeaderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.UUID;
/**
* Handler for close container command received from SCM.
*/
public class CloseContainerCommandHandler implements CommandHandler {
- static final Logger LOG =
+
+ private static final Logger LOG =
LoggerFactory.getLogger(CloseContainerCommandHandler.class);
+
private int invocationCount;
private long totalTime;
- private boolean cmdExecuted;
/**
* Constructs a ContainerReport handler.
@@ -67,79 +68,70 @@ public class CloseContainerCommandHandler implements CommandHandler {
@Override
public void handle(SCMCommand command, OzoneContainer ozoneContainer,
StateContext context, SCMConnectionManager connectionManager) {
- LOG.debug("Processing Close Container command.");
- invocationCount++;
- long startTime = Time.monotonicNow();
- // TODO: define this as INVALID_CONTAINER_ID in HddsConsts.java (TBA)
- long containerID = -1;
try {
- CloseContainerCommandProto closeContainerProto =
+ LOG.debug("Processing Close Container command.");
+ invocationCount++;
+ final long startTime = Time.monotonicNow();
+ final DatanodeDetails datanodeDetails = context.getParent()
+ .getDatanodeDetails();
+ final CloseContainerCommandProto closeCommand =
CloseContainerCommandProto.parseFrom(command.getProtoBufMessage());
- containerID = closeContainerProto.getContainerID();
- // CloseContainer operation is idempotent, if the container is already
- // closed, then do nothing.
- // TODO: Non-existent container should be handled properly
- Container container =
- ozoneContainer.getContainerSet().getContainer(containerID);
- if (container == null) {
- LOG.error("Container {} does not exist in datanode. "
- + "Container close failed.", containerID);
- cmdExecuted = false;
- return;
- }
- ContainerData containerData = container.getContainerData();
- State containerState = container.getContainerData().getState();
- if (containerState != State.CLOSED) {
- LOG.debug("Closing container {}.", containerID);
- // when a closeContainerCommand arrives at a Datanode and if the
- // container is open, each replica will be moved to closing state first.
- if (containerState == State.OPEN) {
- containerData.setState(State.CLOSING);
+ final ContainerController controller = ozoneContainer.getController();
+ final long containerId = closeCommand.getContainerID();
+ try {
+ // TODO: Closing of QUASI_CLOSED container.
+
+ final Container container = controller.getContainer(containerId);
+
+ if (container == null) {
+ LOG.error("Container #{} does not exist in datanode. "
+ + "Container close failed.", containerId);
+ return;
}
- // if the container is already closed, it will be just ignored.
- // ICR will get triggered to change the replica state in SCM.
- HddsProtos.PipelineID pipelineID = closeContainerProto.getPipelineID();
- HddsProtos.ReplicationType replicationType =
- closeContainerProto.getReplicationType();
-
- ContainerProtos.ContainerCommandRequestProto.Builder request =
- ContainerProtos.ContainerCommandRequestProto.newBuilder();
- request.setCmdType(ContainerProtos.Type.CloseContainer);
- request.setContainerID(containerID);
- request.setCloseContainer(
- ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
- request.setTraceID(UUID.randomUUID().toString());
- request.setDatanodeUuid(
- context.getParent().getDatanodeDetails().getUuidString());
- // submit the close container request for the XceiverServer to handle
- ozoneContainer.submitContainerRequest(request.build(), replicationType,
- pipelineID);
- // Since the container is closed, we trigger an ICR
- IncrementalContainerReportProto icr =
- IncrementalContainerReportProto.newBuilder().addReport(
- ozoneContainer.getContainerSet().getContainer(containerID)
- .getContainerReport()).build();
- context.addReport(icr);
- context.getParent().triggerHeartbeat();
- }
- } catch (Exception e) {
- if (e instanceof NotLeaderException) {
- // If the particular datanode is not the Ratis leader, the close
- // container command will not be executed by the follower but will be
- // executed by Ratis stateMachine transactions via leader to follower.
- // There can also be case where the datanode is in candidate state.
- // In these situations, NotLeaderException is thrown.
- LOG.info("Follower cannot close the container {}.", containerID);
- } else {
- LOG.error("Can't close container " + containerID, e);
+ // Move the container to CLOSING state
+ controller.markContainerForClose(containerId);
+
+ // If the container is part of open pipeline, close it via write channel
+ if (ozoneContainer.getWriteChannel()
+ .isExist(closeCommand.getPipelineID())) {
+ ContainerCommandRequestProto request =
+ getContainerCommandRequestProto(datanodeDetails,
+ closeCommand.getContainerID());
+ ozoneContainer.getWriteChannel().submitRequest(
+ request, closeCommand.getPipelineID());
+ return;
+ }
+
+ // The container is not part of any open pipeline.
+ // QUASI_CLOSE the container using ContainerController.
+ controller.quasiCloseContainer(containerId);
+ } catch (NotLeaderException e) {
+ LOG.debug("Follower cannot close container #{}.", containerId);
+ } catch (IOException e) {
+ LOG.error("Can't close container #{}", containerId, e);
+ } finally {
+ long endTime = Time.monotonicNow();
+ totalTime += endTime - startTime;
}
- } finally {
- long endTime = Time.monotonicNow();
- totalTime += endTime - startTime;
+ } catch (InvalidProtocolBufferException ex) {
+ LOG.error("Exception while closing container", ex);
}
}
+ private ContainerCommandRequestProto getContainerCommandRequestProto(
+ final DatanodeDetails datanodeDetails, final long containerId) {
+ final ContainerCommandRequestProto.Builder command =
+ ContainerCommandRequestProto.newBuilder();
+ command.setCmdType(ContainerProtos.Type.CloseContainer);
+ command.setContainerID(containerId);
+ command.setCloseContainer(
+ ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
+ command.setTraceID(UUID.randomUUID().toString());
+ command.setDatanodeUuid(datanodeDetails.getUuidString());
+ return command.build();
+ }
+
/**
* Returns the command type that this command handler handles.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
index acd4af9..9918f9d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
@@ -111,7 +111,7 @@ public final class RegisterEndpointTask implements
try {
ContainerReportsProto containerReport = datanodeContainerManager
- .getContainerReport();
+ .getController().getContainerReport();
NodeReportProto nodeReport = datanodeContainerManager.getNodeReport();
PipelineReportsProto pipelineReportsProto =
datanodeContainerManager.getPipelineReport();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index ab9f42f..992f3cb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -141,6 +141,11 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
}
@Override
+ public boolean isExist(HddsProtos.PipelineID pipelineId) {
+ return PipelineID.valueOf(id).getProtobuf().equals(pipelineId);
+ }
+
+ @Override
public List<PipelineReport> getPipelineReport() {
return Collections.singletonList(
PipelineReport.newBuilder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
index 8c3fa5c..4e0d343 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
@@ -54,6 +54,13 @@ public interface XceiverServerSpi {
throws IOException;
/**
+ * Returns true if the given pipeline exist.
+ *
+ * @return true if pipeline present, else false
+ */
+ boolean isExist(HddsProtos.PipelineID pipelineId);
+
+ /**
* Get pipeline report for the XceiverServer instance.
* @return list of report for each pipeline.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 7bf4da9..434d330 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -86,7 +86,8 @@ import java.util.concurrent.atomic.AtomicLong;
* Ozone containers.
*/
public final class XceiverServerRatis implements XceiverServerSpi {
- static final Logger LOG = LoggerFactory.getLogger(XceiverServerRatis.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(XceiverServerRatis.class);
private static final AtomicLong CALL_ID_COUNTER = new AtomicLong();
private static long nextCallId() {
@@ -458,6 +459,21 @@ public final class XceiverServerRatis implements XceiverServerSpi {
}
@Override
+ public boolean isExist(HddsProtos.PipelineID pipelineId) {
+ try {
+ for (RaftGroupId groupId : server.getGroupIds()) {
+ if (PipelineID.valueOf(
+ groupId.getUuid()).getProtobuf().equals(pipelineId)) {
+ return true;
+ }
+ }
+ return false;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
public List<PipelineReport> getPipelineReport() {
try {
Iterable<RaftGroupId> gids = server.getGroupIds();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index b82c12f..f725d22 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ContainerDataProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerType;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
@@ -267,29 +269,47 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
}
@Override
+ public void markContainerForClose() throws StorageContainerException {
+ updateContainerData(() ->
+ containerData.setState(ContainerDataProto.State.CLOSING));
+ }
+
+ @Override
+ public void quasiClose() throws StorageContainerException {
+ updateContainerData(containerData::quasiCloseContainer);
+ }
+
+ @Override
public void close() throws StorageContainerException {
+ updateContainerData(containerData::closeContainer);
+ // It is ok if this operation takes a bit of time.
+ // Close container is not expected to be instantaneous.
+ compactDB();
+ }
- //TODO: writing .container file and compaction can be done
- // asynchronously, otherwise rpc call for this will take a lot of time to
- // complete this action
+ private void updateContainerData(Runnable update)
+ throws StorageContainerException {
+ ContainerDataProto.State oldState = null;
try {
writeLock();
-
- containerData.closeContainer();
+ oldState = containerData.getState();
+ update.run();
File containerFile = getContainerFile();
// update the new container data to .container File
updateContainerFile(containerFile);
} catch (StorageContainerException ex) {
- // Failed to update .container file. Reset the state to CLOSING
- containerData.setState(ContainerProtos.ContainerDataProto.State.CLOSING);
+ if (oldState != null) {
+ // Failed to update .container file. Reset the state to CLOSING
+ containerData.setState(oldState);
+ }
throw ex;
} finally {
writeUnlock();
}
+ }
- // It is ok if this operation takes a bit of time.
- // Close container is not expected to be instantaneous.
+ private void compactDB() throws StorageContainerException {
try {
MetadataStore db = BlockUtils.getDB(containerData, config);
db.compactDB();
@@ -549,6 +569,9 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
case CLOSING:
state = ContainerReplicaProto.State.CLOSING;
break;
+ case QUASI_CLOSED:
+ state = ContainerReplicaProto.State.QUASI_CLOSED;
+ break;
case CLOSED:
state = ContainerReplicaProto.State.CLOSED;
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index f970c72..4a5ef18 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerDataProto;
+ .ContainerDataProto.State;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
@@ -57,6 +57,7 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume
.RoundRobinVolumeChoosingPolicy;
@@ -109,9 +110,9 @@ public class KeyValueHandler extends Handler {
private final long maxContainerSize;
private final AutoCloseableLock handlerLock;
- public KeyValueHandler(Configuration config, ContainerSet contSet,
- VolumeSet volSet, ContainerMetrics metrics) {
- super(config, contSet, volSet, metrics);
+ public KeyValueHandler(Configuration config, StateContext context,
+ ContainerSet contSet, VolumeSet volSet, ContainerMetrics metrics) {
+ super(config, context, contSet, volSet, metrics);
containerType = ContainerType.KeyValueContainer;
blockManager = new BlockManagerImpl(config);
chunkManager = new ChunkManagerImpl();
@@ -372,20 +373,10 @@ public class KeyValueHandler extends Handler {
request.getTraceID());
return ContainerUtils.malformedRequest(request);
}
-
- long containerID = kvContainer.getContainerData().getContainerID();
try {
- checkContainerOpen(kvContainer);
- // TODO : The close command should move the container to either quasi
- // closed/closed depending upon how the closeContainer gets executed.
- // If it arrives by Standalone, it will be moved to Quasi Closed or
- // otherwise moved to Closed state if it gets executed via Ratis.
- kvContainer.close();
+ markContainerForClose(kvContainer);
+ closeContainer(kvContainer);
} catch (StorageContainerException ex) {
- if (ex.getResult() == CLOSED_CONTAINER_IO) {
- LOG.debug("Container {} is already closed.", containerID);
- return ContainerUtils.getSuccessResponse(request);
- }
return ContainerUtils.logAndReturnError(LOG, ex, request);
} catch (IOException ex) {
return ContainerUtils.logAndReturnError(LOG,
@@ -745,38 +736,39 @@ public class KeyValueHandler extends Handler {
private void checkContainerOpen(KeyValueContainer kvContainer)
throws StorageContainerException {
- ContainerDataProto.State containerState = kvContainer.getContainerState();
+ final State containerState = kvContainer.getContainerState();
- /**
+ /*
* In a closing state, follower will receive transactions from leader.
* Once the leader is put to closing state, it will reject further requests
* from clients. Only the transactions which happened before the container
* in the leader goes to closing state, will arrive here even the container
* might already be in closing state here.
*/
- if (containerState == ContainerDataProto.State.OPEN
- || containerState == ContainerDataProto.State.CLOSING) {
+ if (containerState == State.OPEN || containerState == State.CLOSING) {
return;
- } else {
- String msg = "Requested operation not allowed as ContainerState is " +
- containerState;
- ContainerProtos.Result result = null;
- switch (containerState) {
- case CLOSED:
- result = CLOSED_CONTAINER_IO;
- break;
- case UNHEALTHY:
- result = CONTAINER_UNHEALTHY;
- break;
- case INVALID:
- result = INVALID_CONTAINER_STATE;
- break;
- default:
- result = CONTAINER_INTERNAL_ERROR;
- }
+ }
- throw new StorageContainerException(msg, result);
+ final ContainerProtos.Result result;
+ switch (containerState) {
+ case QUASI_CLOSED:
+ result = CLOSED_CONTAINER_IO;
+ break;
+ case CLOSED:
+ result = CLOSED_CONTAINER_IO;
+ break;
+ case UNHEALTHY:
+ result = CONTAINER_UNHEALTHY;
+ break;
+ case INVALID:
+ result = INVALID_CONTAINER_STATE;
+ break;
+ default:
+ result = CONTAINER_INTERNAL_ERROR;
}
+ String msg = "Requested operation not allowed as ContainerState is " +
+ containerState;
+ throw new StorageContainerException(msg, result);
}
public Container importContainer(long containerID, long maxSize,
@@ -796,4 +788,55 @@ public class KeyValueHandler extends Handler {
return container;
}
+
+ @Override
+ public void markContainerForClose(Container container)
+ throws IOException {
+ State currentState = container.getContainerState();
+ // Move the container to CLOSING state only if it's OPEN
+ if (currentState == State.OPEN) {
+ container.markContainerForClose();
+ sendICR(container);
+ }
+ }
+
+ @Override
+ public void quasiCloseContainer(Container container)
+ throws IOException {
+ final State state = container.getContainerState();
+ // Quasi close call is idempotent.
+ if (state == State.QUASI_CLOSED) {
+ return;
+ }
+ // The container has to be in CLOSING state.
+ if (state != State.CLOSING) {
+ ContainerProtos.Result error = state == State.INVALID ?
+ INVALID_CONTAINER_STATE : CONTAINER_INTERNAL_ERROR;
+ throw new StorageContainerException("Cannot quasi close container #" +
+ container.getContainerData().getContainerID() + " while in " +
+ state + " state.", error);
+ }
+ container.quasiClose();
+ sendICR(container);
+ }
+
+ @Override
+ public void closeContainer(Container container)
+ throws IOException {
+ final State state = container.getContainerState();
+ // Close call is idempotent.
+ if (state == State.CLOSED) {
+ return;
+ }
+ // The container has to be either in CLOSING or in QUASI_CLOSED state.
+ if (state != State.CLOSING && state != State.QUASI_CLOSED) {
+ ContainerProtos.Result error = state == State.INVALID ?
+ INVALID_CONTAINER_STATE : CONTAINER_INTERNAL_ERROR;
+ throw new StorageContainerException("Cannot close container #" +
+ container.getContainerData().getContainerID() + " while in " +
+ state + " state.", error);
+ }
+ container.close();
+ sendICR(container);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
new file mode 100644
index 0000000..41b74e7
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.ozoneimpl;
+
+import org.apache.hadoop.hdds.protocol.datanode.proto
+ .ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ContainerDataProto.State;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
+import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Control plane for container management in datanode.
+ */
+public class ContainerController {
+
+ private final ContainerSet containerSet;
+ private final Map<ContainerType, Handler> handlers;
+
+ public ContainerController(final ContainerSet containerSet,
+ final Map<ContainerType, Handler> handlers) {
+ this.containerSet = containerSet;
+ this.handlers = handlers;
+ }
+
+ /**
+ * Returns the Container given a container id.
+ *
+ * @param containerId ID of the container
+ * @return Container
+ */
+ public Container getContainer(final long containerId) {
+ return containerSet.getContainer(containerId);
+ }
+
+ /**
+ * Marks the container for closing. Moves the container to CLOSING state.
+ *
+ * @param containerId Id of the container to update
+ * @throws IOException in case of exception
+ */
+ public void markContainerForClose(final long containerId)
+ throws IOException {
+ Container container = containerSet.getContainer(containerId);
+
+ if (container.getContainerState() == State.OPEN) {
+ getHandler(container).markContainerForClose(container);
+ }
+ }
+
+ /**
+ * Returns the container report.
+ *
+ * @return ContainerReportsProto
+ * @throws IOException in case of exception
+ */
+ public ContainerReportsProto getContainerReport()
+ throws IOException {
+ return containerSet.getContainerReport();
+ }
+
+ /**
+ * Quasi closes a container given its id.
+ *
+ * @param containerId Id of the container to quasi close
+ * @throws IOException in case of exception
+ */
+ public void quasiCloseContainer(final long containerId) throws IOException {
+ final Container container = containerSet.getContainer(containerId);
+ getHandler(container).quasiCloseContainer(container);
+ }
+
+ /**
+ * Closes a container given its id.
+ *
+ * @param containerId Id of the container to close
+ * @throws IOException in case of exception
+ */
+ public void closeContainer(final long containerId) throws IOException {
+ final Container container = containerSet.getContainer(containerId);
+ getHandler(container).closeContainer(container);
+ }
+
+ public Container importContainer(final ContainerType type,
+ final long containerId, final long maxSize,
+ final FileInputStream rawContainerStream, final TarContainerPacker packer)
+ throws IOException {
+ return handlers.get(type).importContainer(
+ containerId, maxSize, rawContainerStream, packer);
+ }
+
+ /**
+ * Given a container, returns its handler instance.
+ *
+ * @param container Container
+ * @return handler of the container
+ */
+ private Handler getHandler(final Container container) {
+ return handlers.get(container.getContainerType());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 9fac3cb..a89b50a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -19,18 +19,20 @@
package org.apache.hadoop.ozone.container.ozoneimpl;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.PipelineID;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto
+ .ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc;
import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi;
@@ -47,27 +49,26 @@ import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
-import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
-
/**
* Ozone main class sets up the network servers and initializes the container
* layer.
*/
public class OzoneContainer {
- public static final Logger LOG = LoggerFactory.getLogger(
+ private static final Logger LOG = LoggerFactory.getLogger(
OzoneContainer.class);
private final HddsDispatcher hddsDispatcher;
- private final DatanodeDetails dnDetails;
+ private final Map<ContainerType, Handler> handlers;
private final OzoneConfiguration config;
private final VolumeSet volumeSet;
private final ContainerSet containerSet;
- private final Map<ReplicationType, XceiverServerSpi> servers;
+ private final XceiverServerSpi writeChannel;
+ private final XceiverServerSpi readChannel;
+ private final ContainerController controller;
/**
* Construct OzoneContainer object.
@@ -78,31 +79,42 @@ public class OzoneContainer {
*/
public OzoneContainer(DatanodeDetails datanodeDetails, OzoneConfiguration
conf, StateContext context) throws IOException {
- this.dnDetails = datanodeDetails;
this.config = conf;
this.volumeSet = new VolumeSet(datanodeDetails.getUuidString(), conf);
this.containerSet = new ContainerSet();
buildContainerSet();
- hddsDispatcher = new HddsDispatcher(config, containerSet, volumeSet,
- context);
- servers = new HashMap<>();
- servers.put(ReplicationType.STAND_ALONE,
- new XceiverServerGrpc(datanodeDetails, config, hddsDispatcher,
- createReplicationService()));
- servers.put(ReplicationType.RATIS, XceiverServerRatis
- .newXceiverServerRatis(datanodeDetails, config, hddsDispatcher,
- context));
+ final ContainerMetrics metrics = ContainerMetrics.create(conf);
+ this.handlers = Maps.newHashMap();
+ for (ContainerType containerType : ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(
+ containerType, conf, context, containerSet, volumeSet, metrics));
+ }
+ this.hddsDispatcher = new HddsDispatcher(config, containerSet, volumeSet,
+ handlers, context, metrics);
+
+ /*
+ * ContainerController is the control plane
+ * XceiverServerRatis is the write channel
+ * XceiverServerGrpc is the read channel
+ */
+ this.controller = new ContainerController(containerSet, handlers);
+ this.writeChannel = XceiverServerRatis.newXceiverServerRatis(
+ datanodeDetails, config, hddsDispatcher, context);
+ this.readChannel = new XceiverServerGrpc(
+ datanodeDetails, config, hddsDispatcher, createReplicationService());
+
}
private GrpcReplicationService createReplicationService() {
return new GrpcReplicationService(
- new OnDemandContainerReplicationSource(containerSet));
+ new OnDemandContainerReplicationSource(controller));
}
/**
* Build's container map.
*/
- public void buildContainerSet() {
+ private void buildContainerSet() {
Iterator<HddsVolume> volumeSetIterator = volumeSet.getVolumesList()
.iterator();
ArrayList<Thread> volumeThreads = new ArrayList<Thread>();
@@ -111,7 +123,6 @@ public class OzoneContainer {
// And also handle disk failure tolerance need to be added
while (volumeSetIterator.hasNext()) {
HddsVolume volume = volumeSetIterator.next();
- File hddsVolumeRootDir = volume.getHddsRootDir();
Thread thread = new Thread(new ContainerReader(volumeSet, volume,
containerSet, config));
thread.start();
@@ -135,9 +146,8 @@ public class OzoneContainer {
*/
public void start() throws IOException {
LOG.info("Attempting to start container services.");
- for (XceiverServerSpi serverinstance : servers.values()) {
- serverinstance.start();
- }
+ writeChannel.start();
+ readChannel.start();
hddsDispatcher.init();
}
@@ -147,9 +157,8 @@ public class OzoneContainer {
public void stop() {
//TODO: at end of container IO integration work.
LOG.info("Attempting to stop container services.");
- for(XceiverServerSpi serverinstance: servers.values()) {
- serverinstance.stop();
- }
+ writeChannel.stop();
+ readChannel.stop();
hddsDispatcher.shutdown();
}
@@ -163,58 +172,24 @@ public class OzoneContainer {
* @return - container report.
* @throws IOException
*/
- public StorageContainerDatanodeProtocolProtos.ContainerReportsProto
- getContainerReport() throws IOException {
- return this.containerSet.getContainerReport();
- }
public PipelineReportsProto getPipelineReport() {
PipelineReportsProto.Builder pipelineReportsProto =
- PipelineReportsProto.newBuilder();
- for (XceiverServerSpi serverInstance : servers.values()) {
- pipelineReportsProto
- .addAllPipelineReport(serverInstance.getPipelineReport());
- }
+ PipelineReportsProto.newBuilder();
+ pipelineReportsProto.addAllPipelineReport(writeChannel.getPipelineReport());
return pipelineReportsProto.build();
}
- /**
- * Submit ContainerRequest.
- * @param request
- * @param replicationType
- * @param pipelineID
- */
- public void submitContainerRequest(
- ContainerProtos.ContainerCommandRequestProto request,
- ReplicationType replicationType,
- PipelineID pipelineID) throws IOException {
- LOG.info("submitting {} request over {} server for container {}",
- request.getCmdType(), replicationType, request.getContainerID());
- Preconditions.checkState(servers.containsKey(replicationType));
- servers.get(replicationType).submitRequest(request, pipelineID);
- }
-
- private int getPortByType(ReplicationType replicationType) {
- return servers.containsKey(replicationType) ?
- servers.get(replicationType).getIPCPort() : INVALID_PORT;
+ public XceiverServerSpi getWriteChannel() {
+ return writeChannel;
}
- /**
- * Returns the container servers IPC port.
- *
- * @return Container servers IPC port.
- */
- public int getContainerServerPort() {
- return getPortByType(ReplicationType.STAND_ALONE);
+ public XceiverServerSpi getReadChannel() {
+ return readChannel;
}
- /**
- * Returns the Ratis container Server IPC port.
- *
- * @return Ratis port.
- */
- public int getRatisContainerServerPort() {
- return getPortByType(ReplicationType.RATIS);
+ public ContainerController getController() {
+ return controller;
}
/**
@@ -230,11 +205,6 @@ public class OzoneContainer {
return this.hddsDispatcher;
}
- @VisibleForTesting
- public XceiverServerSpi getServer(ReplicationType replicationType) {
- return servers.get(replicationType);
- }
-
public VolumeSet getVolumeSet() {
return volumeSet;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
index 5ef5841..e8f0d17 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
@@ -28,9 +28,8 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.replication.ReplicationTask.Status;
import org.slf4j.Logger;
@@ -49,7 +48,7 @@ public class DownloadAndImportReplicator implements ContainerReplicator {
private final ContainerSet containerSet;
- private final ContainerDispatcher containerDispatcher;
+ private final ContainerController controller;
private final ContainerDownloader downloader;
@@ -57,11 +56,11 @@ public class DownloadAndImportReplicator implements ContainerReplicator {
public DownloadAndImportReplicator(
ContainerSet containerSet,
- ContainerDispatcher containerDispatcher,
+ ContainerController controller,
ContainerDownloader downloader,
TarContainerPacker packer) {
this.containerSet = containerSet;
- this.containerDispatcher = containerDispatcher;
+ this.controller = controller;
this.downloader = downloader;
this.packer = packer;
}
@@ -80,10 +79,9 @@ public class DownloadAndImportReplicator implements ContainerReplicator {
try (FileInputStream tempContainerTarStream = new FileInputStream(
tarFilePath.toFile())) {
- Handler handler = containerDispatcher.getHandler(
- originalContainerData.getContainerType());
-
- Container container = handler.importContainer(containerID,
+ Container container = controller.importContainer(
+ originalContainerData.getContainerType(),
+ containerID,
originalContainerData.getMaxSize(),
tempContainerTarStream,
packer);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/OnDemandContainerReplicationSource.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/OnDemandContainerReplicationSource.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/OnDemandContainerReplicationSource.java
index d557b54..28b8713 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/OnDemandContainerReplicationSource.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/OnDemandContainerReplicationSource.java
@@ -20,12 +20,12 @@ package org.apache.hadoop.ozone.container.replication;
import java.io.IOException;
import java.io.OutputStream;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker;
import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,13 +39,13 @@ public class OnDemandContainerReplicationSource
private static final Logger LOG =
LoggerFactory.getLogger(ContainerReplicationSource.class);
- private ContainerSet containerSet;
+ private ContainerController controller;
private ContainerPacker packer = new TarContainerPacker();
public OnDemandContainerReplicationSource(
- ContainerSet containerSet) {
- this.containerSet = containerSet;
+ ContainerController controller) {
+ this.controller = controller;
}
@Override
@@ -57,7 +57,7 @@ public class OnDemandContainerReplicationSource
public void copyData(long containerId, OutputStream destination)
throws IOException {
- Container container = containerSet.getContainer(containerId);
+ Container container = controller.getContainer(containerId);
Preconditions
.checkNotNull(container, "Container is not found " + containerId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
index 7849bcd..4fe6ae4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.protocol.commands;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.protocol.proto
@@ -30,14 +29,11 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
public class CloseContainerCommand
extends SCMCommand<CloseContainerCommandProto> {
- private HddsProtos.ReplicationType replicationType;
- private PipelineID pipelineID;
+ private final PipelineID pipelineID;
- public CloseContainerCommand(long containerID,
- HddsProtos.ReplicationType replicationType,
- PipelineID pipelineID) {
+ public CloseContainerCommand(final long containerID,
+ final PipelineID pipelineID) {
super(containerID);
- this.replicationType = replicationType;
this.pipelineID = pipelineID;
}
@@ -65,7 +61,6 @@ public class CloseContainerCommand
return CloseContainerCommandProto.newBuilder()
.setContainerID(getId())
.setCmdId(getId())
- .setReplicationType(replicationType)
.setPipelineID(pipelineID.getProtobuf())
.build();
}
@@ -74,7 +69,6 @@ public class CloseContainerCommand
CloseContainerCommandProto closeContainerProto) {
Preconditions.checkNotNull(closeContainerProto);
return new CloseContainerCommand(closeContainerProto.getCmdId(),
- closeContainerProto.getReplicationType(),
PipelineID.getFromProtobuf(closeContainerProto.getPipelineID()));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index 33ea307..9fdef7d 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -140,8 +140,8 @@ message ContainerReplicaProto {
enum State {
OPEN = 1;
CLOSING = 2;
- CLOSED = 3;
- QUASI_CLOSED = 4;
+ QUASI_CLOSED = 3;
+ CLOSED = 4;
UNHEALTHY = 5;
INVALID = 6;
}
@@ -289,9 +289,9 @@ This command asks the datanode to close a specific container.
*/
message CloseContainerCommandProto {
required int64 containerID = 1;
- required hadoop.hdds.ReplicationType replicationType = 2;
+ required PipelineID pipelineID = 2;
+ // cmdId will be removed
required int64 cmdId = 3;
- required PipelineID pipelineID = 4;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index 35cda00..085e6f9 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.common.impl;
+import com.google.common.collect.Maps;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.StorageUnit;
@@ -26,6 +27,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto
+ .ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.datanode.proto
.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
@@ -33,7 +36,9 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.WriteChunkRequestProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerAction;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
@@ -47,6 +52,7 @@ import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
+import java.util.Map;
import java.util.UUID;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -77,8 +83,15 @@ public class TestHddsDispatcher {
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(),
scmId.toString());
containerSet.addContainer(container);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerType containerType : ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(
+ containerType, conf, null, containerSet, volumeSet, metrics));
+ }
HddsDispatcher hddsDispatcher = new HddsDispatcher(
- conf, containerSet, volumeSet, context);
+ conf, containerSet, volumeSet, handlers, context, metrics);
hddsDispatcher.setScmId(scmId.toString());
ContainerCommandResponseProto responseOne = hddsDispatcher.dispatch(
getWriteChunkRequest(dd.getUuidString(), 1L, 1L));
@@ -113,8 +126,15 @@ public class TestHddsDispatcher {
ContainerSet containerSet = new ContainerSet();
VolumeSet volumeSet = new VolumeSet(dd.getUuidString(), conf);
StateContext context = Mockito.mock(StateContext.class);
- HddsDispatcher hddsDispatcher =
- new HddsDispatcher(conf, containerSet, volumeSet, context);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerType containerType : ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(
+ containerType, conf, null, containerSet, volumeSet, metrics));
+ }
+ HddsDispatcher hddsDispatcher = new HddsDispatcher(
+ conf, containerSet, volumeSet, handlers, context, metrics);
hddsDispatcher.setScmId(scmId.toString());
ContainerCommandRequestProto writeChunkRequest =
getWriteChunkRequest(dd.getUuidString(), 1L, 1L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
index b658295..e763289 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.ozone.container.common.interfaces;
+import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
@@ -32,6 +34,8 @@ import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
import org.mockito.Mockito;
+import java.util.Map;
+
/**
* Tests Handler interface.
*/
@@ -50,8 +54,16 @@ public class TestHandler {
this.conf = new Configuration();
this.containerSet = Mockito.mock(ContainerSet.class);
this.volumeSet = Mockito.mock(VolumeSet.class);
-
- this.dispatcher = new HddsDispatcher(conf, containerSet, volumeSet, null);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerProtos.ContainerType containerType :
+ ContainerProtos.ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(
+ containerType, conf, null, containerSet, volumeSet, metrics));
+ }
+ this.dispatcher = new HddsDispatcher(
+ conf, containerSet, volumeSet, handlers, null, metrics);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
new file mode 100644
index 0000000..0cf95b7
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.ratis.RatisHelper;
+import org.apache.ratis.client.RaftClient;
+import org.apache.ratis.protocol.RaftGroup;
+import org.apache.ratis.protocol.RaftGroupId;
+import org.apache.ratis.protocol.RaftPeer;
+import org.apache.ratis.retry.RetryPolicy;
+import org.apache.ratis.rpc.SupportedRpcType;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.UUID;
+
+/**
+ * Test cases to verify CloseContainerCommandHandler in datanode.
+ */
+public class TestCloseContainerCommandHandler {
+
+ private static final StateContext CONTEXT = Mockito.mock(StateContext.class);
+ private static File testDir;
+
+
+ private OzoneContainer getOzoneContainer(final OzoneConfiguration conf,
+ final DatanodeDetails datanodeDetails) throws IOException {
+ testDir = GenericTestUtils.getTestDir(
+ TestCloseContainerCommandHandler.class.getName() + UUID.randomUUID());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath());
+ conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testDir.getPath());
+
+ return new OzoneContainer(datanodeDetails, conf, CONTEXT);
+ }
+
+
+ @Test
+ public void testCloseContainerViaRatis()
+ throws IOException, InterruptedException {
+ final OzoneConfiguration conf = new OzoneConfiguration();
+ final DatanodeDetails datanodeDetails = randomDatanodeDetails();
+ final OzoneContainer container = getOzoneContainer(conf, datanodeDetails);
+ container.getDispatcher().setScmId(UUID.randomUUID().toString());
+ container.start();
+ // Give some time for ratis for leader election.
+ final PipelineID pipelineID = PipelineID.randomId();
+ final RaftGroupId raftGroupId = RaftGroupId.valueOf(pipelineID.getId());
+ final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(conf);
+ final RaftPeer peer = RatisHelper.toRaftPeer(datanodeDetails);
+ final RaftGroup group = RatisHelper.newRaftGroup(raftGroupId,
+ Collections.singleton(datanodeDetails));
+ final RaftClient client = RatisHelper.newRaftClient(
+ SupportedRpcType.GRPC, peer, retryPolicy);
+ System.out.println(client.groupAdd(group, peer.getId()).isSuccess());
+ Thread.sleep(2000);
+ final ContainerID containerId = ContainerID.valueof(1);
+ ContainerProtos.ContainerCommandRequestProto.Builder request =
+ ContainerProtos.ContainerCommandRequestProto.newBuilder();
+ request.setCmdType(ContainerProtos.Type.CreateContainer);
+ request.setContainerID(containerId.getId());
+ request.setCreateContainer(
+ ContainerProtos.CreateContainerRequestProto.getDefaultInstance());
+ request.setTraceID(UUID.randomUUID().toString());
+ request.setDatanodeUuid(datanodeDetails.getUuidString());
+ container.getWriteChannel().submitRequest(
+ request.build(), pipelineID.getProtobuf());
+
+ Assert.assertEquals(ContainerProtos.ContainerDataProto.State.OPEN,
+ container.getContainerSet().getContainer(
+ containerId.getId()).getContainerState());
+
+ // We have created a container via ratis. Now close the container on ratis.
+ final CloseContainerCommandHandler closeHandler =
+ new CloseContainerCommandHandler();
+ final CloseContainerCommand command = new CloseContainerCommand(
+ containerId.getId(), pipelineID);
+ final DatanodeStateMachine datanodeStateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
+
+ Mockito.when(datanodeStateMachine.getDatanodeDetails())
+ .thenReturn(datanodeDetails);
+ Mockito.when(CONTEXT.getParent()).thenReturn(datanodeStateMachine);
+
+ closeHandler.handle(command, container, CONTEXT, null);
+
+ Assert.assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED,
+ container.getContainerSet().getContainer(
+ containerId.getId()).getContainerState());
+
+ Mockito.verify(datanodeStateMachine, Mockito.times(2)).triggerHeartbeat();
+ container.stop();
+ }
+
+ @Test
+ public void testCloseContainerViaStandalone()
+ throws IOException, InterruptedException {
+ final OzoneConfiguration conf = new OzoneConfiguration();
+ final DatanodeDetails datanodeDetails = randomDatanodeDetails();
+ final OzoneContainer container = getOzoneContainer(conf, datanodeDetails);
+ container.getDispatcher().setScmId(UUID.randomUUID().toString());
+ container.start();
+ // Give some time for ratis for leader election.
+ final PipelineID pipelineID = PipelineID.randomId();
+ final RaftGroupId raftGroupId = RaftGroupId.valueOf(pipelineID.getId());
+ final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(conf);
+ final RaftPeer peer = RatisHelper.toRaftPeer(datanodeDetails);
+ final RaftGroup group = RatisHelper.newRaftGroup(raftGroupId,
+ Collections.singleton(datanodeDetails));
+ final RaftClient client = RatisHelper.newRaftClient(
+ SupportedRpcType.GRPC, peer, retryPolicy);
+ System.out.println(client.groupAdd(group, peer.getId()).isSuccess());
+ Thread.sleep(2000);
+ final ContainerID containerId = ContainerID.valueof(2);
+ ContainerProtos.ContainerCommandRequestProto.Builder request =
+ ContainerProtos.ContainerCommandRequestProto.newBuilder();
+ request.setCmdType(ContainerProtos.Type.CreateContainer);
+ request.setContainerID(containerId.getId());
+ request.setCreateContainer(
+ ContainerProtos.CreateContainerRequestProto.getDefaultInstance());
+ request.setTraceID(UUID.randomUUID().toString());
+ request.setDatanodeUuid(datanodeDetails.getUuidString());
+ container.getWriteChannel().submitRequest(
+ request.build(), pipelineID.getProtobuf());
+
+ Assert.assertEquals(ContainerProtos.ContainerDataProto.State.OPEN,
+ container.getContainerSet().getContainer(
+ containerId.getId()).getContainerState());
+
+ // We have created a container via ratis. Now quasi close it
+ final CloseContainerCommandHandler closeHandler =
+ new CloseContainerCommandHandler();
+ // Specify a pipeline which doesn't exist in the datanode.
+ final CloseContainerCommand command = new CloseContainerCommand(
+ containerId.getId(), PipelineID.randomId());
+ final DatanodeStateMachine datanodeStateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
+
+ Mockito.when(datanodeStateMachine.getDatanodeDetails())
+ .thenReturn(datanodeDetails);
+ Mockito.when(CONTEXT.getParent()).thenReturn(datanodeStateMachine);
+
+ closeHandler.handle(command, container, CONTEXT, null);
+
+ Assert.assertEquals(ContainerProtos.ContainerDataProto.State.QUASI_CLOSED,
+ container.getContainerSet().getContainer(
+ containerId.getId()).getContainerState());
+
+ Mockito.verify(datanodeStateMachine, Mockito.times(2)).triggerHeartbeat();
+ container.stop();
+ }
+
+ /**
+ * Creates a random DatanodeDetails.
+ * @return DatanodeDetails
+ */
+ private static DatanodeDetails randomDatanodeDetails() {
+ String ipAddress = "127.0.0.1";
+ DatanodeDetails.Port containerPort = DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.STANDALONE, 0);
+ DatanodeDetails.Port ratisPort = DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.RATIS, 0);
+ DatanodeDetails.Port restPort = DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.REST, 0);
+ DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
+ builder.setUuid(UUID.randomUUID().toString())
+ .setHostName("localhost")
+ .setIpAddress(ipAddress)
+ .addPort(containerPort)
+ .addPort(ratisPort)
+ .addPort(restPort);
+ return builder.build();
+ }
+
+ @AfterClass
+ public static void teardown() throws IOException {
+ FileUtils.deleteDirectory(testDir);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 7fc065f..265fb18 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -54,6 +54,7 @@ import static org.mockito.Mockito.times;
import java.io.File;
+import java.io.IOException;
import java.util.UUID;
/**
@@ -224,7 +225,7 @@ public class TestKeyValueHandler {
interval[0] = 2;
ContainerMetrics metrics = new ContainerMetrics(interval);
VolumeSet volumeSet = new VolumeSet(UUID.randomUUID().toString(), conf);
- KeyValueHandler keyValueHandler = new KeyValueHandler(conf, cset,
+ KeyValueHandler keyValueHandler = new KeyValueHandler(conf, null, cset,
volumeSet, metrics);
assertEquals("org.apache.hadoop.ozone.container.common" +
".volume.RoundRobinVolumeChoosingPolicy",
@@ -235,7 +236,7 @@ public class TestKeyValueHandler {
conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY,
"org.apache.hadoop.ozone.container.common.impl.HddsDispatcher");
try {
- new KeyValueHandler(conf, cset, volumeSet, metrics);
+ new KeyValueHandler(conf, null, cset, volumeSet, metrics);
} catch (RuntimeException ex) {
GenericTestUtils.assertExceptionContains("class org.apache.hadoop" +
".ozone.container.common.impl.HddsDispatcher not org.apache" +
@@ -261,7 +262,7 @@ public class TestKeyValueHandler {
@Test
- public void testCloseInvalidContainer() {
+ public void testCloseInvalidContainer() throws IOException {
long containerID = 1234L;
Configuration conf = new Configuration();
KeyValueContainerData kvData = new KeyValueContainerData(containerID,
@@ -282,6 +283,7 @@ public class TestKeyValueHandler {
Mockito.when(handler.handleCloseContainer(any(), any()))
.thenCallRealMethod();
+ doCallRealMethod().when(handler).closeContainer(any());
// Closing invalid container should return error response.
ContainerProtos.ContainerCommandResponseProto response =
handler.handleCloseContainer(closeContainerRequest, container);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index 6f96f4b..34030e8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -172,7 +172,7 @@ public class BlockManagerImpl implements EventHandler<Boolean>,
LOG.warn("Unable to allocate container.");
}
} catch (IOException ex) {
- LOG.warn("Unable to allocate container: {}", ex);
+ LOG.warn("Unable to allocate container.", ex);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d96400/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
index 719d763..fd73711 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
@@ -74,8 +74,8 @@ public class CloseContainerEventHandler implements EventHandler<ContainerID> {
if (container.getState() == LifeCycleState.CLOSING) {
final CloseContainerCommand closeContainerCommand =
- new CloseContainerCommand(containerID.getId(),
- container.getReplicationType(), container.getPipelineID());
+ new CloseContainerCommand(
+ containerID.getId(), container.getPipelineID());
getNodes(container).forEach(node -> publisher.fireEvent(
DATANODE_COMMAND,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HDFS-14011. RBF: Add more
information to HdfsFileStatus for a mount point. Contributed by Akira
Ajisaka.
Posted by br...@apache.org.
HDFS-14011. RBF: Add more information to HdfsFileStatus for a mount point. Contributed by Akira Ajisaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee21095c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee21095c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee21095c
Branch: refs/heads/HDFS-13891
Commit: ee21095c2d6911483c7a303319740ca7670499e7
Parents: 507ab98
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Oct 23 14:34:29 2018 +0800
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:34:52 2018 +0530
----------------------------------------------------------------------
.../resolver/FileSubclusterResolver.java | 6 ++-
.../federation/router/RouterClientProtocol.java | 30 ++++++++---
.../router/RouterQuotaUpdateService.java | 9 ++--
.../hdfs/server/federation/MockResolver.java | 17 +++---
.../federation/router/TestRouterMountTable.java | 55 +++++++++++++++++++-
.../router/TestRouterRpcMultiDestination.java | 5 +-
6 files changed, 97 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21095c/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
index 5aa5ec9..6432bb0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
@@ -61,8 +61,10 @@ public interface FileSubclusterResolver {
* cache.
*
* @param path Path to get the mount points under.
- * @return List of mount points present at this path or zero-length list if
- * none are found.
+ * @return List of mount points present at this path. Return zero-length
+ * list if the path is a mount point but there are no mount points
+ * under the path. Return null if the path is not a mount point
+ * and there are no mount points under the path.
* @throws IOException Throws exception if the data is not available.
*/
List<String> getMountPoints(String path) throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21095c/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
index ddbc014..de94eaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
@@ -718,6 +718,9 @@ public class RouterClientProtocol implements ClientProtocol {
date = dates.get(src);
}
ret = getMountPointStatus(src, children.size(), date);
+ } else if (children != null) {
+ // The src is a mount point, but there are no files or directories
+ ret = getMountPointStatus(src, 0, 0);
}
}
@@ -1714,13 +1717,26 @@ public class RouterClientProtocol implements ClientProtocol {
FsPermission permission = FsPermission.getDirDefault();
String owner = this.superUser;
String group = this.superGroup;
- try {
- // TODO support users, it should be the user for the pointed folder
- UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
- owner = ugi.getUserName();
- group = ugi.getPrimaryGroupName();
- } catch (IOException e) {
- LOG.error("Cannot get the remote user: {}", e.getMessage());
+ if (subclusterResolver instanceof MountTableResolver) {
+ try {
+ MountTableResolver mountTable = (MountTableResolver) subclusterResolver;
+ MountTable entry = mountTable.getMountPoint(name);
+ if (entry != null) {
+ permission = entry.getMode();
+ owner = entry.getOwnerName();
+ group = entry.getGroupName();
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot get mount point: {}", e.getMessage());
+ }
+ } else {
+ try {
+ UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
+ owner = ugi.getUserName();
+ group = ugi.getPrimaryGroupName();
+ } catch (IOException e) {
+ LOG.error("Cannot get remote user: {}", e.getMessage());
+ }
}
long inodeId = 0;
return new HdfsFileStatus.Builder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21095c/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
index 4813b53..9bfd705 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
@@ -87,11 +87,12 @@ public class RouterQuotaUpdateService extends PeriodicService {
QuotaUsage currentQuotaUsage = null;
- // Check whether destination path exists in filesystem. If destination
- // is not present, reset the usage. For other mount entry get current
- // quota usage
+ // Check whether destination path exists in filesystem. When the
+ // mtime is zero, the destination is not present and reset the usage.
+ // This is because mount table does not have mtime.
+ // For other mount entry get current quota usage
HdfsFileStatus ret = this.rpcServer.getFileInfo(src);
- if (ret == null) {
+ if (ret == null || ret.getModificationTime() == 0) {
currentQuotaUsage = new RouterQuotaUsage.Builder()
.fileAndDirectoryCount(0)
.quota(nsQuota)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21095c/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
index f5636ce..9bff007 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -303,15 +303,16 @@ public class MockResolver
@Override
public List<String> getMountPoints(String path) throws IOException {
+ // Mounts only supported under root level
+ if (!path.equals("/")) {
+ return null;
+ }
List<String> mounts = new ArrayList<>();
- if (path.equals("/")) {
- // Mounts only supported under root level
- for (String mount : this.locations.keySet()) {
- if (mount.length() > 1) {
- // Remove leading slash, this is the behavior of the mount tree,
- // return only names.
- mounts.add(mount.replace("/", ""));
- }
+ for (String mount : this.locations.keySet()) {
+ if (mount.length() > 1) {
+ // Remove leading slash, this is the behavior of the mount tree,
+ // return only names.
+ mounts.add(mount.replace("/", ""));
}
}
return mounts;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21095c/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
index 4d8ffe1..d2b78d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -43,8 +44,12 @@ import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.util.Time;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -59,9 +64,11 @@ public class TestRouterMountTable {
private static RouterContext routerContext;
private static MountTableResolver mountTable;
private static ClientProtocol routerProtocol;
+ private static long startTime;
@BeforeClass
public static void globalSetUp() throws Exception {
+ startTime = Time.now();
// Build and start a federated cluster
cluster = new StateStoreDFSCluster(false, 1);
@@ -92,6 +99,21 @@ public class TestRouterMountTable {
}
}
+ @After
+ public void clearMountTable() throws IOException {
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTableManager = client.getMountTableManager();
+ GetMountTableEntriesRequest req1 =
+ GetMountTableEntriesRequest.newInstance("/");
+ GetMountTableEntriesResponse response =
+ mountTableManager.getMountTableEntries(req1);
+ for (MountTable entry : response.getEntries()) {
+ RemoveMountTableEntryRequest req2 =
+ RemoveMountTableEntryRequest.newInstance(entry.getSourcePath());
+ mountTableManager.removeMountTableEntry(req2);
+ }
+ }
+
@Test
public void testReadOnly() throws Exception {
@@ -157,7 +179,6 @@ public class TestRouterMountTable {
*/
@Test
public void testListFilesTime() throws Exception {
- Long beforeCreatingTime = Time.now();
// Add mount table entry
MountTable addEntry = MountTable.newInstance(
"/testdir", Collections.singletonMap("ns0", "/testdir"));
@@ -211,10 +232,40 @@ public class TestRouterMountTable {
Long expectedTime = pathModTime.get(currentFile);
assertEquals(currentFile, fileName);
- assertTrue(currentTime > beforeCreatingTime);
+ assertTrue(currentTime > startTime);
assertEquals(currentTime, expectedTime);
}
// Verify the total number of results found/matched
assertEquals(pathModTime.size(), listing.getPartialListing().length);
}
+
+ /**
+ * Verify that the file listing contains correct permission.
+ */
+ @Test
+ public void testMountTablePermissions() throws Exception {
+ // Add mount table entries
+ MountTable addEntry = MountTable.newInstance(
+ "/testdir1", Collections.singletonMap("ns0", "/testdir1"));
+ addEntry.setGroupName("group1");
+ addEntry.setOwnerName("owner1");
+ addEntry.setMode(FsPermission.createImmutable((short)0775));
+ assertTrue(addMountTable(addEntry));
+ addEntry = MountTable.newInstance(
+ "/testdir2", Collections.singletonMap("ns0", "/testdir2"));
+ addEntry.setGroupName("group2");
+ addEntry.setOwnerName("owner2");
+ addEntry.setMode(FsPermission.createImmutable((short)0755));
+ assertTrue(addMountTable(addEntry));
+
+ HdfsFileStatus fs = routerProtocol.getFileInfo("/testdir1");
+ assertEquals("group1", fs.getGroup());
+ assertEquals("owner1", fs.getOwner());
+ assertEquals((short) 0775, fs.getPermission().toShort());
+
+ fs = routerProtocol.getFileInfo("/testdir2");
+ assertEquals("group2", fs.getGroup());
+ assertEquals("owner2", fs.getOwner());
+ assertEquals((short) 0755, fs.getPermission().toShort());
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21095c/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
index 7e09760..94b712f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
@@ -123,8 +123,9 @@ public class TestRouterRpcMultiDestination extends TestRouterRpc {
RouterContext rc = getRouterContext();
Router router = rc.getRouter();
FileSubclusterResolver subclusterResolver = router.getSubclusterResolver();
- for (String mount : subclusterResolver.getMountPoints(path)) {
- requiredPaths.add(mount);
+ List<String> mountList = subclusterResolver.getMountPoints(path);
+ if (mountList != null) {
+ requiredPaths.addAll(mountList);
}
// Get files/dirs from the Namenodes
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-8953. [CSI] CSI driver
adaptor module support in NodeManager. Contributed by Weiwei Yang.
Posted by br...@apache.org.
YARN-8953. [CSI] CSI driver adaptor module support in NodeManager. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fb14e06
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fb14e06
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fb14e06
Branch: refs/heads/HDFS-13891
Commit: 5fb14e0635f850c63e009f954b8539a401d0f90a
Parents: 27ffec7
Author: Sunil G <su...@apache.org>
Authored: Tue Nov 20 08:02:11 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Nov 20 08:02:11 2018 +0530
----------------------------------------------------------------------
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 2 +
.../apache/hadoop/yarn/api/CsiAdaptorPB.java | 31 +++
.../hadoop/yarn/api/CsiAdaptorProtocol.java | 39 +++
.../protocolrecords/GetPluginInfoRequest.java | 30 +++
.../protocolrecords/GetPluginInfoResponse.java | 43 ++++
.../ValidateVolumeCapabilitiesRequest.java | 117 +++++++++
.../ValidateVolumeCapabilitiesResponse.java | 46 ++++
.../hadoop/yarn/conf/YarnConfiguration.java | 12 +
.../src/main/proto/YarnCsiAdaptor.proto | 34 +++
.../src/main/proto/yarn_csi_adaptor.proto | 69 +++++
.../client/CsiAdaptorProtocolPBClientImpl.java | 91 +++++++
.../CsiAdaptorProtocolPBServiceImpl.java | 75 ++++++
.../impl/pb/GetPluginInfoRequestPBImpl.java | 60 +++++
.../impl/pb/GetPluginInfoResponsePBImpl.java | 84 ++++++
...ValidateVolumeCapabilitiesRequestPBImpl.java | 121 +++++++++
...alidateVolumeCapabilitiesResponsePBImpl.java | 87 +++++++
.../src/main/resources/yarn-default.xml | 13 +
.../hadoop-yarn/hadoop-yarn-csi/pom.xml | 12 +
.../csi/adaptor/CsiAdaptorProtocolService.java | 122 +++++++++
.../hadoop/yarn/csi/adaptor/package-info.java | 21 ++
.../hadoop/yarn/csi/client/CsiClient.java | 4 +
.../hadoop/yarn/csi/client/CsiClientImpl.java | 11 +
.../GetPluginInfoResponseProtoTranslator.java | 44 ++++
.../yarn/csi/translator/ProtoTranslator.java | 49 ++++
.../csi/translator/ProtoTranslatorFactory.java | 66 +++++
...olumeCapabilitiesRequestProtoTranslator.java | 93 +++++++
...lumeCapabilitiesResponseProtoTranslator.java | 48 ++++
.../yarn/csi/translator/package-info.java | 21 ++
.../hadoop/yarn/csi/utils/ConfigUtils.java | 61 +++++
.../yarn/csi/adaptor/TestCsiAdaptorService.java | 256 +++++++++++++++++++
.../TestGetPluginInfoRequestResponse.java | 66 +++++
.../TestValidateVolumeCapabilityRequest.java | 113 ++++++++
.../TestValidateVolumeCapabilityResponse.java | 61 +++++
.../hadoop/yarn/csi/adaptor/package-info.java | 21 ++
.../volume/csi/CsiAdaptorClientProtocol.java | 34 ---
.../volume/csi/CsiAdaptorClient.java | 36 ---
.../volume/csi/VolumeManager.java | 23 +-
.../volume/csi/VolumeManagerImpl.java | 91 ++++++-
.../volume/csi/lifecycle/Volume.java | 8 +
.../volume/csi/lifecycle/VolumeImpl.java | 58 +++--
.../csi/processor/VolumeAMSProcessor.java | 17 +-
.../volume/csi/TestVolumeLifecycle.java | 100 +++++---
.../volume/csi/TestVolumeProcessor.java | 32 ++-
43 files changed, 2273 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index 4f11ad6..832e98c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -124,6 +124,8 @@
<include>server/application_history_server.proto</include>
<include>client_SCM_protocol.proto</include>
<include>server/SCM_Admin_protocol.proto</include>
+ <include>yarn_csi_adaptor.proto</include>
+ <include>YarnCsiAdaptor.proto</include>
</includes>
</source>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPB.java
new file mode 100644
index 0000000..2a7a114
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPB.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtocol;
+
+/**
+ * Interface for the CSI adaptor protocol.
+ */
+@ProtocolInfo(
+ protocolName = "CsiAdaptorPB",
+ protocolVersion = 1)
+public interface CsiAdaptorPB extends
+ CsiAdaptorProtocol.CsiAdaptorProtocolService.BlockingInterface {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java
new file mode 100644
index 0000000..0822163
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+import java.io.IOException;
+
+/**
+ * CSI adaptor delegates all the calls from YARN to a CSI driver.
+ */
+public interface CsiAdaptorProtocol {
+
+ GetPluginInfoResponse getPluginInfo(GetPluginInfoRequest request)
+ throws YarnException, IOException;
+
+ ValidateVolumeCapabilitiesResponse validateVolumeCapacity(
+ ValidateVolumeCapabilitiesRequest request) throws YarnException,
+ IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoRequest.java
new file mode 100644
index 0000000..a32c93c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoRequest.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Get plugin info request.
+ */
+public abstract class GetPluginInfoRequest {
+
+ public static GetPluginInfoRequest newInstance() {
+ return Records.newRecord(GetPluginInfoRequest.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoResponse.java
new file mode 100644
index 0000000..9577200
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoResponse.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Get plugin info response.
+ */
+public abstract class GetPluginInfoResponse {
+
+ public static GetPluginInfoResponse newInstance(
+ String driverName, String version) {
+ GetPluginInfoResponse response =
+ Records.newRecord(GetPluginInfoResponse.class);
+ response.setDriverName(driverName);
+ response.setVersion(version);
+ return response;
+ }
+
+ public abstract void setDriverName(String driverName);
+
+ public abstract String getDriverName();
+
+ public abstract void setVersion(String version);
+
+ public abstract String getVersion();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesRequest.java
new file mode 100644
index 0000000..0ab1b1f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesRequest.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.yarn.util.Records;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * YARN internal message used to validate volume capabilities
+ * with a CSI driver controller plugin.
+ */
+public abstract class ValidateVolumeCapabilitiesRequest {
+
+ /**
+ * Volume access mode.
+ */
+ public enum AccessMode {
+ UNKNOWN,
+ SINGLE_NODE_WRITER,
+ SINGLE_NODE_READER_ONLY,
+ MULTI_NODE_READER_ONLY,
+ MULTI_NODE_SINGLE_WRITER,
+ MULTI_NODE_MULTI_WRITER,
+ }
+
+ /**
+ * Volume type.
+ */
+ public enum VolumeType {
+ BLOCK,
+ FILE_SYSTEM
+ }
+
+ /**
+ * Volume capability.
+ */
+ public static class VolumeCapability {
+
+ private AccessMode mode;
+ private VolumeType type;
+ private List<String> flags;
+
+ public VolumeCapability(AccessMode accessMode, VolumeType volumeType,
+ List<String> mountFlags) {
+ this.mode = accessMode;
+ this.type = volumeType;
+ this.flags = mountFlags;
+ }
+
+ public AccessMode getAccessMode() {
+ return mode;
+ }
+
+ public VolumeType getVolumeType() {
+ return type;
+ }
+
+ public List<String> getMountFlags() {
+ return flags;
+ }
+ }
+
+ public static ValidateVolumeCapabilitiesRequest newInstance(
+ String volumeId, List<VolumeCapability> volumeCapabilities,
+ Map<String, String> volumeAttributes) {
+ ValidateVolumeCapabilitiesRequest
+ request =
+ Records.newRecord(
+ ValidateVolumeCapabilitiesRequest.class);
+ request.setVolumeId(volumeId);
+ request.setVolumeAttributes(volumeAttributes);
+ for (VolumeCapability capability : volumeCapabilities) {
+ request.addVolumeCapability(capability);
+ }
+ return request;
+ }
+
+ public static ValidateVolumeCapabilitiesRequest newInstance(
+ String volumeId, Map<String, String> volumeAttributes) {
+ ValidateVolumeCapabilitiesRequest
+ request =
+ Records.newRecord(
+ ValidateVolumeCapabilitiesRequest.class);
+ request.setVolumeId(volumeId);
+ request.setVolumeAttributes(volumeAttributes);
+ return request;
+ }
+
+ public abstract void setVolumeId(String volumeId);
+
+ public abstract String getVolumeId();
+
+ public abstract void setVolumeAttributes(Map<String, String> attributes);
+
+ public abstract Map<String, String> getVolumeAttributes();
+
+ public abstract void addVolumeCapability(VolumeCapability volumeCapability);
+
+ public abstract List<VolumeCapability> getVolumeCapabilities();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesResponse.java
new file mode 100644
index 0000000..a72ba0a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesResponse.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * YARN internal message used to represent the response of
+ * volume capabilities validation with a CSI driver controller plugin.
+ */
+public abstract class ValidateVolumeCapabilitiesResponse {
+
+ public static ValidateVolumeCapabilitiesResponse newInstance(
+ boolean supported, String responseMessage) {
+ ValidateVolumeCapabilitiesResponse
+ record =
+ Records.newRecord(
+ ValidateVolumeCapabilitiesResponse.class);
+ record.setResponseMessage(responseMessage);
+ record.setSupported(supported);
+ return record;
+ }
+
+ public abstract void setSupported(boolean supported);
+
+ public abstract boolean isSupported();
+
+ public abstract void setResponseMessage(String responseMessage);
+
+ public abstract String getResponseMessage();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index e88d594..2f25284 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3428,6 +3428,18 @@ public class YarnConfiguration extends Configuration {
false;
////////////////////////////////
+ // CSI Volume configs
+ ////////////////////////////////
+ /**
+ * One or more socket addresses for csi-adaptor.
+ * Multiple addresses are delimited by ",".
+ */
+ public static final String NM_CSI_ADAPTOR_PREFIX =
+ NM_PREFIX + "csi-driver-adaptor.";
+ public static final String NM_CSI_ADAPTOR_ADDRESSES =
+ NM_CSI_ADAPTOR_PREFIX + "addresses";
+
+ ////////////////////////////////
// Other Configs
////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/YarnCsiAdaptor.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/YarnCsiAdaptor.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/YarnCsiAdaptor.proto
new file mode 100644
index 0000000..9dcb8a7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/YarnCsiAdaptor.proto
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.yarn.proto";
+option java_outer_classname = "CsiAdaptorProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.yarn;
+
+import "yarn_csi_adaptor.proto";
+
+service CsiAdaptorProtocolService {
+
+ rpc getPluginInfo (GetPluginInfoRequest)
+ returns (GetPluginInfoResponse);
+
+ rpc validateVolumeCapacity (ValidateVolumeCapabilitiesRequest)
+ returns (ValidateVolumeCapabilitiesResponse);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_csi_adaptor.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_csi_adaptor.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_csi_adaptor.proto
new file mode 100644
index 0000000..c9adbea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_csi_adaptor.proto
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+option java_package = "org.apache.hadoop.yarn.proto";
+option java_outer_classname = "CsiAdaptorProtos";
+option java_generate_equals_and_hash = true;
+package hadoop.yarn;
+
+import "yarn_protos.proto";
+
+message ValidateVolumeCapabilitiesRequest {
+ required string volume_id = 1;
+ repeated VolumeCapability volume_capabilities = 2;
+ repeated StringStringMapProto volume_attributes = 3;
+}
+
+message ValidateVolumeCapabilitiesResponse {
+ // True if the Plugin supports the specified capabilities for the
+ // given volume. This field is REQUIRED.
+ required bool supported = 1;
+
+ // Message to the CO if `supported` above is false. This field is
+ // OPTIONAL.
+ // An empty string is equal to an unspecified field value.
+ optional string message = 2;
+}
+
+message VolumeCapability {
+ enum VolumeType {
+ BLOCK = 0;
+ FILE_SYSTEM = 1;
+ }
+
+ enum AccessMode {
+ UNKNOWN = 0;
+ SINGLE_NODE_WRITER = 1;
+ SINGLE_NODE_READER_ONLY = 2;
+ MULTI_NODE_READER_ONLY = 3;
+ MULTI_NODE_SINGLE_WRITER = 4;
+ MULTI_NODE_MULTI_WRITER = 5;
+ }
+
+ required VolumeType volume_type = 1;
+ required AccessMode access_mode = 2;
+ repeated string mount_flags = 3;
+}
+
+message GetPluginInfoRequest {
+ // Intentionally empty.
+}
+
+message GetPluginInfoResponse {
+ required string name = 1;
+ required string vendor_version = 2;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/CsiAdaptorProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/CsiAdaptorProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/CsiAdaptorProtocolPBClientImpl.java
new file mode 100644
index 0000000..2e10f72
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/CsiAdaptorProtocolPBClientImpl.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.impl.pb.client;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.yarn.api.CsiAdaptorPB;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesResponsePBImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+/**
+ * CSI adaptor client implementation.
+ */
+public class CsiAdaptorProtocolPBClientImpl
+ implements CsiAdaptorProtocol, Closeable {
+
+ private final CsiAdaptorPB proxy;
+
+ public CsiAdaptorProtocolPBClientImpl(long clientVersion,
+ InetSocketAddress addr, Configuration conf) throws IOException {
+ RPC.setProtocolEngine(conf, CsiAdaptorPB.class, ProtobufRpcEngine.class);
+ this.proxy = RPC.getProxy(CsiAdaptorPB.class, clientVersion, addr, conf);
+ }
+
+ @Override
+ public GetPluginInfoResponse getPluginInfo(
+ GetPluginInfoRequest request) throws YarnException, IOException {
+ CsiAdaptorProtos.GetPluginInfoRequest requestProto =
+ ((GetPluginInfoRequestPBImpl) request).getProto();
+ try {
+ return new GetPluginInfoResponsePBImpl(
+ proxy.getPluginInfo(null, requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
+
+ @Override
+ public ValidateVolumeCapabilitiesResponse validateVolumeCapacity(
+ ValidateVolumeCapabilitiesRequest request)
+ throws YarnException, IOException {
+ CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest requestProto =
+ ((ValidateVolumeCapabilitiesRequestPBImpl) request).getProto();
+ try {
+ return new ValidateVolumeCapabilitiesResponsePBImpl(
+ proxy.validateVolumeCapacity(null, requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if(this.proxy != null) {
+ RPC.stopProxy(this.proxy);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/CsiAdaptorProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/CsiAdaptorProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/CsiAdaptorProtocolPBServiceImpl.java
new file mode 100644
index 0000000..9a19435
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/CsiAdaptorProtocolPBServiceImpl.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.impl.pb.service;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.yarn.api.CsiAdaptorPB;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesResponsePBImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
+
+import java.io.IOException;
+
+/**
+ * CSI adaptor server side implementation, this is hosted on a node manager.
+ */
+public class CsiAdaptorProtocolPBServiceImpl implements CsiAdaptorPB {
+
+ private final CsiAdaptorProtocol real;
+ public CsiAdaptorProtocolPBServiceImpl(CsiAdaptorProtocol impl) {
+ this.real = impl;
+ }
+
+ @Override
+ public CsiAdaptorProtos.GetPluginInfoResponse getPluginInfo(
+ RpcController controller, CsiAdaptorProtos.GetPluginInfoRequest request)
+ throws ServiceException {
+ try {
+ GetPluginInfoRequest req =
+ new GetPluginInfoRequestPBImpl(request);
+ GetPluginInfoResponse response = real.getPluginInfo(req);
+ return ((GetPluginInfoResponsePBImpl) response).getProto();
+ } catch (YarnException | IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse
+ validateVolumeCapacity(RpcController controller,
+ CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest request)
+ throws ServiceException {
+ try {
+ ValidateVolumeCapabilitiesRequestPBImpl req =
+ new ValidateVolumeCapabilitiesRequestPBImpl(request);
+ ValidateVolumeCapabilitiesResponse response =
+ real.validateVolumeCapacity(req);
+ return ((ValidateVolumeCapabilitiesResponsePBImpl) response).getProto();
+ } catch (YarnException | IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java
new file mode 100644
index 0000000..0d1c2e5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
+
+/**
+ * Get plugin info request protobuf impl.
+ */
+public class GetPluginInfoRequestPBImpl extends GetPluginInfoRequest {
+
+ private CsiAdaptorProtos.GetPluginInfoRequest.Builder builder;
+
+ public GetPluginInfoRequestPBImpl(
+ CsiAdaptorProtos.GetPluginInfoRequest requestProto) {
+ this.builder = requestProto.toBuilder();
+ }
+
+ public GetPluginInfoRequestPBImpl() {
+ this.builder = CsiAdaptorProtos.GetPluginInfoRequest.newBuilder();
+ }
+
+ public CsiAdaptorProtos.GetPluginInfoRequest getProto() {
+ Preconditions.checkNotNull(builder);
+ return builder.build();
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java
new file mode 100644
index 0000000..141fd6d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
+
+/**
+ * Get plugin info response protobuf impl.
+ */
+public class GetPluginInfoResponsePBImpl extends GetPluginInfoResponse {
+
+ private CsiAdaptorProtos.GetPluginInfoResponse.Builder builder;
+
+ public GetPluginInfoResponsePBImpl(
+ CsiAdaptorProtos.GetPluginInfoResponse responseProto) {
+ this.builder = responseProto.toBuilder();
+ }
+
+ public GetPluginInfoResponsePBImpl() {
+ this.builder = CsiAdaptorProtos.GetPluginInfoResponse.newBuilder();
+ }
+
+ @Override
+ public void setDriverName(String driverName) {
+ Preconditions.checkNotNull(builder);
+ builder.setName(driverName);
+ }
+
+ @Override
+ public String getDriverName() {
+ Preconditions.checkNotNull(builder);
+ return builder.getName();
+ }
+
+ @Override
+ public void setVersion(String version) {
+ Preconditions.checkNotNull(builder);
+ builder.setVendorVersion(version);
+ }
+
+ @Override
+ public String getVersion() {
+ Preconditions.checkNotNull(builder);
+ return builder.getVendorVersion();
+ }
+
+ public CsiAdaptorProtos.GetPluginInfoResponse getProto() {
+ Preconditions.checkNotNull(builder);
+ return builder.build();
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java
new file mode 100644
index 0000000..14bd89d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * PB wrapper for CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest.
+ */
+public class ValidateVolumeCapabilitiesRequestPBImpl extends
+ ValidateVolumeCapabilitiesRequest {
+
+ private CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest.Builder builder;
+
+ public ValidateVolumeCapabilitiesRequestPBImpl(
+ CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest proto) {
+ this.builder = proto.toBuilder();
+ }
+
+ public ValidateVolumeCapabilitiesRequestPBImpl() {
+ this.builder = CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest
+ .newBuilder();
+ }
+
+ @Override
+ public String getVolumeId() {
+ Preconditions.checkNotNull(builder);
+ return builder.getVolumeId();
+ }
+
+ @Override
+ public void setVolumeAttributes(Map<String, String> attributes) {
+ Preconditions.checkNotNull(builder);
+ builder.addAllVolumeAttributes(ProtoUtils.convertToProtoFormat(attributes));
+ }
+
+ @Override
+ public void setVolumeId(String volumeId) {
+ Preconditions.checkNotNull(builder);
+ builder.setVolumeId(volumeId);
+ }
+
+ @Override
+ public void addVolumeCapability(VolumeCapability volumeCapability) {
+ Preconditions.checkNotNull(builder);
+ CsiAdaptorProtos.VolumeCapability vc =
+ CsiAdaptorProtos.VolumeCapability.newBuilder()
+ .setAccessMode(CsiAdaptorProtos.VolumeCapability.AccessMode
+ .valueOf(volumeCapability.getAccessMode().ordinal()))
+ .setVolumeType(CsiAdaptorProtos.VolumeCapability.VolumeType
+ .valueOf(volumeCapability.getVolumeType().ordinal()))
+ .addAllMountFlags(volumeCapability.getMountFlags())
+ .build();
+ builder.addVolumeCapabilities(vc);
+ }
+
+ @Override
+ public List<VolumeCapability> getVolumeCapabilities() {
+ Preconditions.checkNotNull(builder);
+ List<VolumeCapability> caps = new ArrayList<>(
+ builder.getVolumeCapabilitiesCount());
+ builder.getVolumeCapabilitiesList().forEach(capability -> {
+ VolumeCapability vc = new VolumeCapability(
+ AccessMode.valueOf(capability.getAccessMode().name()),
+ VolumeType.valueOf(capability.getVolumeType().name()),
+ capability.getMountFlagsList());
+ caps.add(vc);
+ });
+ return caps;
+ }
+
+ @Override
+ public Map<String, String> getVolumeAttributes() {
+ Preconditions.checkNotNull(builder);
+ return ProtoUtils.convertStringStringMapProtoListToMap(
+ builder.getVolumeAttributesList());
+ }
+
+ public CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest getProto() {
+ Preconditions.checkNotNull(builder);
+ return builder.build();
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java
new file mode 100644
index 0000000..aa33ab7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
+
+/**
+ * PB wrapper for CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse.
+ */
+public class ValidateVolumeCapabilitiesResponsePBImpl
+ extends ValidateVolumeCapabilitiesResponse {
+
+ private CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse.Builder builder;
+
+ public ValidateVolumeCapabilitiesResponsePBImpl() {
+ this.builder = CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse
+ .newBuilder();
+ }
+
+ public ValidateVolumeCapabilitiesResponsePBImpl(
+ CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse response) {
+ this.builder = response.toBuilder();
+ }
+
+ @Override
+ public void setSupported(boolean supported) {
+ Preconditions.checkNotNull(builder);
+ this.builder.setSupported(supported);
+ }
+
+ @Override
+ public boolean isSupported() {
+ Preconditions.checkNotNull(builder);
+ return builder.getSupported();
+ }
+
+ @Override
+ public void setResponseMessage(String message) {
+ Preconditions.checkNotNull(builder);
+ this.builder.setMessage(message);
+ }
+
+ @Override
+ public String getResponseMessage() {
+ Preconditions.checkNotNull(builder);
+ return this.builder.getMessage();
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ public CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse getProto() {
+ Preconditions.checkNotNull(builder);
+ return builder.build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index f5493bc..e7a0e14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -4037,4 +4037,17 @@
<name>yarn.node-attribute.fs-store.impl.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore</value>
</property>
+
+ <!-- CSI configuration -->
+ <property>
+ <description>
+ CSI driver adaptor addresses on a node manager.
+ This configuration will be loaded by the resource manager to initiate
+ a client for each adaptor in order to communicate with CSI drivers.
+ Note, these addresses should be mapped to the adaptor addresses which
+ runs the controller plugin.
+ </description>
+ <name>yarn.nodemanager.csi-driver-adaptor.addresses</name>
+ <value></value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
index 41f5098..27d8452 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
@@ -84,6 +84,18 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
<scope>compile</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/CsiAdaptorProtocolService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/CsiAdaptorProtocolService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/CsiAdaptorProtocolService.java
new file mode 100644
index 0000000..f94275f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/CsiAdaptorProtocolService.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.csi.adaptor;
+
+import com.google.common.annotations.VisibleForTesting;
+import csi.v0.Csi;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+import org.apache.hadoop.yarn.csi.client.CsiClient;
+import org.apache.hadoop.yarn.csi.client.CsiClientImpl;
+import org.apache.hadoop.yarn.csi.translator.ProtoTranslatorFactory;
+import org.apache.hadoop.yarn.csi.utils.ConfigUtils;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+/**
+ * This is a Hadoop RPC server, we uses the Hadoop RPC framework here
+ * because we need to stick to the security model current Hadoop supports.
+ */
+public class CsiAdaptorProtocolService extends AbstractService
+ implements CsiAdaptorProtocol {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CsiAdaptorProtocolService.class);
+
+ private Server server;
+ private InetSocketAddress adaptorServiceAddress;
+ private CsiClient csiClient;
+ private String csiDriverName;
+
+ public CsiAdaptorProtocolService(String driverName,
+ String domainSocketPath) {
+ super(CsiAdaptorProtocolService.class.getName());
+ this.csiClient = new CsiClientImpl(domainSocketPath);
+ this.csiDriverName = driverName;
+ }
+
+ @VisibleForTesting
+ public void setCsiClient(CsiClient client) {
+ this.csiClient = client;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ adaptorServiceAddress = ConfigUtils
+ .getCsiAdaptorAddressForDriver(csiDriverName, conf);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ Configuration conf = getConfig();
+ YarnRPC rpc = YarnRPC.create(conf);
+ this.server = rpc.getServer(
+ CsiAdaptorProtocol.class,
+ this, adaptorServiceAddress, conf, null, 1);
+ this.server.start();
+ LOG.info("{} started, listening on address: {}",
+ CsiAdaptorProtocolService.class.getName(),
+ adaptorServiceAddress.toString());
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (this.server != null) {
+ this.server.stop();
+ }
+ super.serviceStop();
+ }
+
+ @Override
+ public GetPluginInfoResponse getPluginInfo(
+ GetPluginInfoRequest request) throws YarnException, IOException {
+ Csi.GetPluginInfoResponse response = csiClient.getPluginInfo();
+ return ProtoTranslatorFactory.getTranslator(
+ GetPluginInfoResponse.class, Csi.GetPluginInfoResponse.class)
+ .convertFrom(response);
+ }
+
+ @Override
+ public ValidateVolumeCapabilitiesResponse validateVolumeCapacity(
+ ValidateVolumeCapabilitiesRequest request) throws YarnException,
+ IOException {
+ Csi.ValidateVolumeCapabilitiesRequest req = ProtoTranslatorFactory
+ .getTranslator(ValidateVolumeCapabilitiesRequest.class,
+ Csi.ValidateVolumeCapabilitiesRequest.class)
+ .convertTo(request);
+ Csi.ValidateVolumeCapabilitiesResponse response =
+ csiClient.validateVolumeCapabilities(req);
+ return ProtoTranslatorFactory.getTranslator(
+ ValidateVolumeCapabilitiesResponse.class,
+ Csi.ValidateVolumeCapabilitiesResponse.class)
+ .convertFrom(response);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java
new file mode 100644
index 0000000..919aab5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains CSI adaptor classes.
+ */
+package org.apache.hadoop.yarn.csi.adaptor;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClient.java
index 5bb9ce9..d31c0c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClient.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.csi.client;
+import csi.v0.Csi;
import csi.v0.Csi.GetPluginInfoResponse;
import java.io.IOException;
@@ -36,4 +37,7 @@ public interface CsiClient {
* @throws IOException when unable to get plugin info from the driver.
*/
GetPluginInfoResponse getPluginInfo() throws IOException;
+
+ Csi.ValidateVolumeCapabilitiesResponse validateVolumeCapabilities(
+ Csi.ValidateVolumeCapabilitiesRequest request) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClientImpl.java
index 58dd292..5b3d2e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClientImpl.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.csi.client;
+import csi.v0.Csi;
import csi.v0.Csi.GetPluginInfoRequest;
import csi.v0.Csi.GetPluginInfoResponse;
import org.apache.hadoop.yarn.csi.utils.GrpcHelper;
@@ -48,4 +49,14 @@ public class CsiClientImpl implements CsiClient {
return client.createIdentityBlockingStub().getPluginInfo(request);
}
}
+
+ @Override
+ public Csi.ValidateVolumeCapabilitiesResponse validateVolumeCapabilities(
+ Csi.ValidateVolumeCapabilitiesRequest request) throws IOException {
+ try (CsiGrpcClient client = CsiGrpcClient.newBuilder()
+ .setDomainSocketAddress(address).build()) {
+ return client.createControllerBlockingStub()
+ .validateVolumeCapabilities(request);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/GetPluginInfoResponseProtoTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/GetPluginInfoResponseProtoTranslator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/GetPluginInfoResponseProtoTranslator.java
new file mode 100644
index 0000000..c4f042e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/GetPluginInfoResponseProtoTranslator.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.csi.translator;
+
+import csi.v0.Csi;
+import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * Protobuf message translator for GetPluginInfoResponse and
+ * Csi.GetPluginInfoResponse.
+ */
+public class GetPluginInfoResponseProtoTranslator implements
+ ProtoTranslator<GetPluginInfoResponse, Csi.GetPluginInfoResponse> {
+
+ @Override public Csi.GetPluginInfoResponse convertTo(
+ GetPluginInfoResponse messageA) throws YarnException {
+ return Csi.GetPluginInfoResponse.newBuilder()
+ .setName(messageA.getDriverName())
+ .setVendorVersion(messageA.getVersion())
+ .build();
+ }
+
+ @Override public GetPluginInfoResponse convertFrom(
+ Csi.GetPluginInfoResponse messageB) throws YarnException {
+ return GetPluginInfoResponse.newInstance(messageB.getName(),
+ messageB.getVendorVersion());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslator.java
new file mode 100644
index 0000000..93cb441
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslator.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.csi.translator;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * ProtoTranslator converts a YARN side message to CSI proto message
+ * and vice versa. Each CSI proto message should have a corresponding
+ * YARN side message implementation, and a transformer to convert them
+ * one to the other. This layer helps we to hide CSI spec messages
+ * from YARN components.
+ *
+ * @param <A> YARN side internal messages
+ * @param <B> CSI proto messages
+ */
+public interface ProtoTranslator<A, B> {
+
+ /**
+ * Convert message from type A to type B.
+ * @param messageA
+ * @return messageB
+ * @throws YarnException
+ */
+ B convertTo(A messageA) throws YarnException;
+
+ /**
+ * Convert message from type B to type A.
+ * @param messageB
+ * @return messageA
+ * @throws YarnException
+ */
+ A convertFrom(B messageB) throws YarnException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslatorFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslatorFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslatorFactory.java
new file mode 100644
index 0000000..5eb76ff
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslatorFactory.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.csi.translator;
+
+import csi.v0.Csi;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+
+/**
+ * Factory class to get desired proto transformer instance.
+ */
+public final class ProtoTranslatorFactory {
+
+ private ProtoTranslatorFactory() {
+ // hide constructor for the factory class
+ }
+
+ /**
+ * Get a {@link ProtoTranslator} based on the given input message
+ * types. If the type is not supported, a IllegalArgumentException
+ * will be thrown. When adding more transformers to this factory class,
+ * note each transformer works exactly for one message to another
+ * (and vice versa). For each type of the message, make sure there is
+ * a corresponding unit test added, such as
+ * TestValidateVolumeCapabilitiesRequest.
+ *
+ * @param yarnProto yarn proto message
+ * @param csiProto CSI proto message
+ * @param <A> yarn proto message
+ * @param <B> CSI proto message
+ * @throws IllegalArgumentException
+ * when given types are not supported
+ * @return
+ * a proto message transformer that transforms
+ * YARN internal proto message to CSI
+ */
+ public static <A, B> ProtoTranslator<A, B> getTranslator(
+ Class<A> yarnProto, Class<B> csiProto) {
+ if (yarnProto == ValidateVolumeCapabilitiesRequest.class
+ && csiProto == Csi.ValidateVolumeCapabilitiesRequest.class) {
+ return new ValidateVolumeCapabilitiesRequestProtoTranslator();
+ } else if (yarnProto == ValidateVolumeCapabilitiesResponse.class
+ && csiProto == Csi.ValidateVolumeCapabilitiesResponse.class) {
+ return new ValidationVolumeCapabilitiesResponseProtoTranslator();
+ }
+ throw new IllegalArgumentException("A problem is found while processing"
+ + " proto message translating. Unexpected message types,"
+ + " no transformer is found can handle the transformation from type "
+ + yarnProto.getName() + " <-> " + csiProto.getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidateVolumeCapabilitiesRequestProtoTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidateVolumeCapabilitiesRequestProtoTranslator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidateVolumeCapabilitiesRequestProtoTranslator.java
new file mode 100644
index 0000000..a74c47a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidateVolumeCapabilitiesRequestProtoTranslator.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.csi.translator;
+
+import csi.v0.Csi;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeCapability;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeType;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Proto message translator for ValidateVolumeCapabilitiesRequest.
+ * @param <A> ValidateVolumeCapabilitiesRequest
+ * @param <B> Csi.ValidateVolumeCapabilitiesRequest
+ */
+public class ValidateVolumeCapabilitiesRequestProtoTranslator<A, B>
+ implements ProtoTranslator<ValidateVolumeCapabilitiesRequest,
+ Csi.ValidateVolumeCapabilitiesRequest> {
+
+ @Override
+ public Csi.ValidateVolumeCapabilitiesRequest convertTo(
+ ValidateVolumeCapabilitiesRequest request) throws YarnException {
+ Csi.ValidateVolumeCapabilitiesRequest.Builder buidler =
+ Csi.ValidateVolumeCapabilitiesRequest.newBuilder();
+ buidler.setVolumeId(request.getVolumeId());
+ if (request.getVolumeCapabilities() != null
+ && request.getVolumeCapabilities().size() > 0) {
+ buidler.putAllVolumeAttributes(request.getVolumeAttributes());
+ }
+ for (VolumeCapability cap :
+ request.getVolumeCapabilities()) {
+ Csi.VolumeCapability.AccessMode accessMode =
+ Csi.VolumeCapability.AccessMode.newBuilder()
+ .setModeValue(cap.getAccessMode().ordinal())
+ .build();
+ Csi.VolumeCapability.MountVolume mountVolume =
+ Csi.VolumeCapability.MountVolume.newBuilder()
+ .addAllMountFlags(cap.getMountFlags())
+ .build();
+ Csi.VolumeCapability capability =
+ Csi.VolumeCapability.newBuilder()
+ .setAccessMode(accessMode)
+ .setMount(mountVolume)
+ .build();
+ buidler.addVolumeCapabilities(capability);
+ }
+ return buidler.build();
+ }
+
+ @Override
+ public ValidateVolumeCapabilitiesRequest convertFrom(
+ Csi.ValidateVolumeCapabilitiesRequest request) throws YarnException {
+ ValidateVolumeCapabilitiesRequest result = ValidateVolumeCapabilitiesRequest
+ .newInstance(request.getVolumeId(), request.getVolumeAttributesMap());
+ for (Csi.VolumeCapability csiCap :
+ request.getVolumeCapabilitiesList()) {
+ ValidateVolumeCapabilitiesRequest.AccessMode mode =
+ ValidateVolumeCapabilitiesRequest.AccessMode
+ .valueOf(csiCap.getAccessMode().getMode().name());
+ if (!csiCap.hasMount()) {
+ throw new YarnException("Invalid request,"
+ + " mount is not found in the request.");
+ }
+ List<String> mountFlags = new ArrayList<>();
+ for (int i=0; i<csiCap.getMount().getMountFlagsCount(); i++) {
+ mountFlags.add(csiCap.getMount().getMountFlags(i));
+ }
+ VolumeCapability capability = new VolumeCapability(mode,
+ VolumeType.FILE_SYSTEM, mountFlags);
+ result.addVolumeCapability(capability);
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidationVolumeCapabilitiesResponseProtoTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidationVolumeCapabilitiesResponseProtoTranslator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidationVolumeCapabilitiesResponseProtoTranslator.java
new file mode 100644
index 0000000..fd42712
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidationVolumeCapabilitiesResponseProtoTranslator.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.csi.translator;
+
+import csi.v0.Csi;
+import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * Proto message translator for ValidateVolumeCapabilitiesResponse.
+ * @param <A> ValidateVolumeCapabilitiesResponse
+ * @param <B> Csi.ValidateVolumeCapabilitiesResponse
+ */
+public class ValidationVolumeCapabilitiesResponseProtoTranslator<A, B>
+ implements ProtoTranslator<ValidateVolumeCapabilitiesResponse,
+ Csi.ValidateVolumeCapabilitiesResponse> {
+
+ @Override
+ public Csi.ValidateVolumeCapabilitiesResponse convertTo(
+ ValidateVolumeCapabilitiesResponse response) throws YarnException {
+ return Csi.ValidateVolumeCapabilitiesResponse.newBuilder()
+ .setSupported(response.isSupported())
+ .setMessage(response.getResponseMessage())
+ .build();
+ }
+
+ @Override
+ public ValidateVolumeCapabilitiesResponse convertFrom(
+ Csi.ValidateVolumeCapabilitiesResponse response) throws YarnException {
+ return ValidateVolumeCapabilitiesResponse.newInstance(
+ response.getSupported(), response.getMessage());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/package-info.java
new file mode 100644
index 0000000..c0964cd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains classes for protocol translation between YARN and CSI.
+ */
+package org.apache.hadoop.yarn.csi.translator;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/utils/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/utils/ConfigUtils.java
new file mode 100644
index 0000000..77e6955
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/utils/ConfigUtils.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.csi.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+import java.net.InetSocketAddress;
+
+/**
+ * Utility class to load configurations.
+ */
+public final class ConfigUtils {
+
+ private ConfigUtils() {
+ // Hide constructor for utility class.
+ }
+ /**
+ * Resolve the CSI adaptor address for a CSI driver from configuration.
+ * Expected configuration property name is
+ * yarn.nodemanager.csi-driver-adaptor.${driverName}.address.
+ * @param driverName
+ * @param conf
+ * @return adaptor service address
+ * @throws YarnException
+ */
+ public static InetSocketAddress getCsiAdaptorAddressForDriver(
+ String driverName, Configuration conf) throws YarnException {
+ String configName = YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
+ + driverName + ".address";
+ String errorMessage = "Failed to load CSI adaptor address for driver "
+ + driverName + ", configuration property " + configName
+ + " is not defined or invalid.";
+ try {
+ InetSocketAddress address = conf
+ .getSocketAddr(configName, null, -1);
+ if (address == null) {
+ throw new YarnException(errorMessage);
+ }
+ return address;
+ } catch (IllegalArgumentException e) {
+ throw new YarnException(errorMessage);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDDS-860. Fix TestDataValidate
unit tests. Contributed by Shashikant Banerjee.
Posted by br...@apache.org.
HDDS-860. Fix TestDataValidate unit tests. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8b3dfa6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8b3dfa6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8b3dfa6
Branch: refs/heads/HDFS-13891
Commit: c8b3dfa6250cd74fb3e449748595117b244089da
Parents: 14e1a0a
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Wed Nov 21 11:28:20 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Wed Nov 21 11:28:20 2018 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8b3dfa6/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index 5d72f557..d85b829 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -269,7 +269,7 @@ public final class RandomKeyGenerator implements Callable<Void> {
processor.shutdown();
processor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
-
+ completed = true;
progressbar.shutdown();
if (validateWrites) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDDS-791. Support Range header for
ozone s3 object download. Contributed by Bharat Viswanadham.
Posted by br...@apache.org.
HDDS-791. Support Range header for ozone s3 object download. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22867def
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22867def
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22867def
Branch: refs/heads/HDFS-13891
Commit: 22867deffa13f73332f7749db349e489045cd900
Parents: f63e4e4
Author: Márton Elek <el...@apache.org>
Authored: Wed Nov 21 17:04:53 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Wed Nov 21 17:04:53 2018 +0100
----------------------------------------------------------------------
.../src/main/smoketest/s3/objectputget.robot | 116 ++++++++++++++++++-
.../ozone/s3/endpoint/ObjectEndpoint.java | 74 ++++++++++--
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 +
.../ozone/s3/io/S3WrapperInputStream.java | 79 +++++++++++++
.../apache/hadoop/ozone/s3/io/package-info.java | 23 ++++
.../hadoop/ozone/s3/util/RangeHeader.java | 89 ++++++++++++++
.../apache/hadoop/ozone/s3/util/S3Consts.java | 15 +++
.../apache/hadoop/ozone/s3/util/S3utils.java | 68 +++++++++++
.../hadoop/ozone/s3/util/TestS3utils.java | 93 +++++++++++++++
9 files changed, 554 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
index f650de8..b2fdaeb 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
@@ -29,14 +29,128 @@ ${BUCKET} generated
*** Test Cases ***
Put object to s3
- Execute date > /tmp/testfile
+ Execute echo "Randomtext" > /tmp/testfile
${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key putobject/f1 --body /tmp/testfile
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix putobject/
Should contain ${result} f1
+ Execute touch -f /tmp/zerobyte
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key putobject/zerobyte --body /tmp/zerobyte
+ ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix putobject/
+ Should contain ${result} zerobyte
+
#This test depends on the previous test case. Can't be executes alone
Get object from s3
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 /tmp/testfile.result
${checksumbefore} = Execute md5sum /tmp/testfile | awk '{print $1}'
${checksumafter} = Execute md5sum /tmp/testfile.result | awk '{print $1}'
Should Be Equal ${checksumbefore} ${checksumafter}
+
+Get Partial object from s3 with both start and endoffset
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 0-4/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=0 bs=1 count=5 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile1.result
+ Should Be Equal ${expectedData} ${actualData}
+
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=2-4 /tmp/testfile1.result1
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 2-4/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=2 bs=1 count=3 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile1.result1
+ Should Be Equal ${expectedData} ${actualData}
+
+# end offset greater than file size and start with in file length
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=2-1000 /tmp/testfile1.result2
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 2-10/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=2 bs=1 count=9 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile1.result2
+ Should Be Equal ${expectedData} ${actualData}
+
+Get Partial object from s3 with both start and endoffset(start offset and endoffset is greater than file size)
+ ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=10000-10000 /tmp/testfile2.result 255
+ Should contain ${result} InvalidRange
+
+
+Get Partial object from s3 with both start and endoffset(end offset is greater than file size)
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-10000 /tmp/testfile2.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 0-10/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute cat /tmp/testfile
+ ${actualData} = Execute cat /tmp/testfile2.result
+ Should Be Equal ${expectedData} ${actualData}
+
+Get Partial object from s3 with only start offset
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0- /tmp/testfile3.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 0-10/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute cat /tmp/testfile
+ ${actualData} = Execute cat /tmp/testfile3.result
+ Should Be Equal ${expectedData} ${actualData}
+
+Get Partial object from s3 with both start and endoffset which are equal
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-0 /tmp/testfile4.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 0-0/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=0 bs=1 count=1 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile4.result
+ Should Be Equal ${expectedData} ${actualData}
+
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=4-4 /tmp/testfile5.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 4-4/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=4 bs=1 count=1 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile5.result
+ Should Be Equal ${expectedData} ${actualData}
+
+Get Partial object from s3 to get last n bytes
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=-4 /tmp/testfile6.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 7-10/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=7 bs=1 count=4 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile6.result
+ Should Be Equal ${expectedData} ${actualData}
+
+# if end is greater than file length, returns whole file
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=-10000 /tmp/testfile7.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 0-10/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute cat /tmp/testfile
+ ${actualData} = Execute cat /tmp/testfile7.result
+ Should Be Equal ${expectedData} ${actualData}
+
+Incorrect values for end and start offset
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=-11-10000 /tmp/testfile8.result
+ Should not contain ${result} ContentRange
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute cat /tmp/testfile
+ ${actualData} = Execute cat /tmp/testfile8.result
+ Should Be Equal ${expectedData} ${actualData}
+
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=11-8 /tmp/testfile9.result
+ Should not contain ${result} ContentRange
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute cat /tmp/testfile
+ ${actualData} = Execute cat /tmp/testfile8.result
+ Should Be Equal ${expectedData} ${actualData}
+
+Zero byte file
+ ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-0 /tmp/testfile2.result 255
+ Should contain ${result} InvalidRange
+
+ ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-1 /tmp/testfile2.result 255
+ Should contain ${result} InvalidRange
+
+ ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255
+ Should contain ${result} InvalidRange
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 149add5..580b724 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -51,15 +51,18 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.IOUtils;
+
+import org.apache.hadoop.ozone.s3.io.S3WrapperInputStream;
+import org.apache.hadoop.ozone.s3.util.RangeHeader;
import org.apache.hadoop.ozone.s3.util.S3StorageType;
+import org.apache.hadoop.ozone.s3.util.S3utils;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
-import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.*;
/**
* Key level rest endpoints.
@@ -165,16 +168,73 @@ public class ObjectEndpoint extends EndpointBase {
@PathParam("bucket") String bucketName,
@PathParam("path") String keyPath,
InputStream body) throws IOException, OS3Exception {
-
try {
OzoneBucket bucket = getBucket(bucketName);
- OzoneInputStream key = bucket
- .readKey(keyPath);
+ OzoneKeyDetails keyDetails = bucket.getKey(keyPath);
+
+ long length = keyDetails.getDataSize();
+
+ LOG.debug("Data length of the key {} is {}", keyPath, length);
+
+ String rangeHeaderVal = headers.getHeaderString(RANGE_HEADER);
+ RangeHeader rangeHeader = null;
+
+ LOG.debug("range Header provided value is {}", rangeHeaderVal);
+
+ if (rangeHeaderVal != null) {
+ rangeHeader = S3utils.parseRangeHeader(rangeHeaderVal,
+ length);
+ LOG.debug("range Header provided value is {}", rangeHeader);
+ if (rangeHeader.isInValidRange()) {
+ OS3Exception exception = S3ErrorTable.newError(S3ErrorTable
+ .INVALID_RANGE, rangeHeaderVal);
+ throw exception;
+ }
+ }
+ ResponseBuilder responseBuilder;
+
+ if (rangeHeaderVal == null || rangeHeader.isReadFull()) {
+ StreamingOutput output = dest -> {
+ try (OzoneInputStream key = bucket.readKey(keyPath)) {
+ IOUtils.copy(key, dest);
+ }
+ };
+ responseBuilder = Response.ok(output);
+
+ } else {
+ LOG.info("range Header provided value is {}", rangeHeader);
+ OzoneInputStream key = bucket.readKey(keyPath);
+
+ long startOffset = rangeHeader.getStartOffset();
+ long endOffset = rangeHeader.getEndOffset();
+ long copyLength;
+ if (startOffset == endOffset) {
+ // if range header is given as bytes=0-0, then we should return 1
+ // byte from start offset
+ copyLength = 1;
+ } else {
+ copyLength = rangeHeader.getEndOffset() - rangeHeader
+ .getStartOffset() + 1;
+ }
+ StreamingOutput output = dest -> {
+ try (S3WrapperInputStream s3WrapperInputStream =
+ new S3WrapperInputStream(
+ key.getInputStream())) {
+ IOUtils.copyLarge(s3WrapperInputStream, dest, startOffset,
+ copyLength);
+ }
+ };
+ responseBuilder = Response.ok(output);
- StreamingOutput output = dest -> IOUtils.copy(key, dest);
- ResponseBuilder responseBuilder = Response.ok(output);
+ String contentRangeVal = RANGE_HEADER_SUPPORTED_UNIT + " " +
+ rangeHeader.getStartOffset() + "-" + rangeHeader.getEndOffset() +
+ "/" + length;
+ responseBuilder.header(CONTENT_RANGE_HEADER, contentRangeVal);
+ }
+ responseBuilder.header(ACCEPT_RANGE_HEADER,
+ RANGE_HEADER_SUPPORTED_UNIT);
for (String responseHeader : customizableGetHeaders) {
String headerValue = headers.getHeaderString(responseHeader);
if (headerValue != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
index 78ec9db..4ca36e4 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.s3.exception;
import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
import static java.net.HttpURLConnection.HTTP_CONFLICT;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_NOT_SATISFIABLE;
/**
* This class represents errors from Ozone S3 service.
@@ -58,6 +59,10 @@ public final class S3ErrorTable {
public static final OS3Exception INVALID_REQUEST = new OS3Exception(
"InvalidRequest", "Invalid Request", HTTP_BAD_REQUEST);
+ public static final OS3Exception INVALID_RANGE = new OS3Exception(
+ "InvalidRange", "The requested range is not satisfiable",
+ RANGE_NOT_SATISFIABLE);
+
/**
* Create a new instance of Error.
* @param e Error Template
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
new file mode 100644
index 0000000..654962b
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.io;
+
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * S3Wrapper Input Stream which encapsulates ChunkGroupInputStream from ozone.
+ */
+public class S3WrapperInputStream extends FSInputStream {
+ private final ChunkGroupInputStream inputStream;
+
+ /**
+ * Constructs S3WrapperInputStream with ChunkInputStream.
+ *
+ * @param inputStream
+ */
+ public S3WrapperInputStream(InputStream inputStream) {
+ this.inputStream = (ChunkGroupInputStream) inputStream;
+ }
+
+ @Override
+ public int read() throws IOException {
+ return inputStream.read();
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ return inputStream.read(b, off, len);
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ inputStream.close();
+ }
+
+ @Override
+ public int available() throws IOException {
+ return inputStream.available();
+ }
+
+ public InputStream getInputStream() {
+ return inputStream;
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ inputStream.seek(pos);
+ }
+ @Override
+ public long getPos() throws IOException {
+ return inputStream.getPos();
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java
new file mode 100644
index 0000000..5167e60
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains Ozone S3 wrapper stream related classes.
+ */
+
+package org.apache.hadoop.ozone.s3.io;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/RangeHeader.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/RangeHeader.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/RangeHeader.java
new file mode 100644
index 0000000..5f5c827
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/RangeHeader.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.util;
+
+/**
+ * Ranger Header class which hold startoffset, endoffset of the Range header
+ * value provided as part of get object.
+ *
+ */
+public class RangeHeader {
+ private long startOffset;
+ private long endOffset;
+ private boolean readFull;
+ private boolean inValidRange;
+
+
+ /**
+ * Construct RangeHeader object.
+ * @param startOffset
+ * @param endOffset
+ */
+ public RangeHeader(long startOffset, long endOffset, boolean full,
+ boolean invalid) {
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ this.readFull = full;
+ this.inValidRange = invalid;
+ }
+
+ /**
+ * Return startOffset.
+ *
+ * @return startOffset
+ */
+ public long getStartOffset() {
+ return startOffset;
+ }
+
+ /**
+ * Return endoffset.
+ *
+ * @return endoffset
+ */
+ public long getEndOffset() {
+ return endOffset;
+ }
+
+ /**
+ * Return a flag whether after parsing range header, when the provided
+ * values are with in a range, and whole file read is required.
+ *
+ * @return readFull
+ */
+ public boolean isReadFull() {
+ return readFull;
+ }
+
+ /**
+ * Return a flag, whether range header values are correct or not.
+ *
+ * @return isInValidRange
+ */
+ public boolean isInValidRange() {
+ return inValidRange;
+ }
+
+
+ public String toString() {
+ return "startOffset - [" + startOffset + "]" + "endOffset - [" +
+ endOffset + "]" + " readFull - [ " + readFull + "]" + " invalidRange " +
+ "- [ " + inValidRange + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 6ece7df..38c4e6a 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.s3.util;
import org.apache.hadoop.classification.InterfaceAudience;
+import java.util.regex.Pattern;
+
/**
* Set of constants used for S3 implementation.
*/
@@ -35,4 +37,17 @@ public final class S3Consts {
public static final String STORAGE_CLASS_HEADER = "x-amz-storage-class";
public static final String ENCODING_TYPE = "url";
+ // Constants related to Range Header
+ public static final String RANGE_HEADER_SUPPORTED_UNIT = "bytes";
+ public static final String RANGE_HEADER = "Range";
+ public static final String ACCEPT_RANGE_HEADER = "Accept-Ranges";
+ public static final String CONTENT_RANGE_HEADER = "Content-Range";
+
+
+ public static final Pattern RANGE_HEADER_MATCH_PATTERN =
+ Pattern.compile("bytes=(?<start>[0-9]*)-(?<end>[0-9]*)");
+
+ //Error code 416 is Range Not Satisfiable
+ public static final int RANGE_NOT_SATISFIABLE = 416;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
index 652ba7f..3ce9bbd 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
@@ -21,14 +21,20 @@ package org.apache.hadoop.ozone.s3.util;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+
import java.nio.charset.StandardCharsets;
+import java.util.regex.Matcher;
+import static org.apache.hadoop.ozone.s3.util.S3Consts
+ .RANGE_HEADER_MATCH_PATTERN;
/**
* Utility class for S3.
*/
+@InterfaceAudience.Private
public final class S3utils {
private S3utils() {
@@ -88,4 +94,66 @@ public final class S3utils {
return null;
}
}
+
+
+ /**
+ * Parse the rangeHeader and set the start and end offset.
+ * @param rangeHeaderVal
+ * @param length
+ *
+ * @return RangeHeader
+ */
+ public static RangeHeader parseRangeHeader(String rangeHeaderVal, long
+ length) {
+ long start = 0;
+ long end = 0;
+ boolean noStart = false;
+ boolean readFull = false;
+ boolean inValidRange = false;
+ RangeHeader rangeHeader;
+ Matcher matcher = RANGE_HEADER_MATCH_PATTERN.matcher(rangeHeaderVal);
+ if (matcher.matches()) {
+ if (!matcher.group("start").equals("")) {
+ start = Integer.parseInt(matcher.group("start"));
+ } else {
+ noStart = true;
+ }
+ if (!matcher.group("end").equals("")) {
+ end = Integer.parseInt(matcher.group("end"));
+ } else {
+ end = length - 1;
+ }
+ if (noStart) {
+ if (end < length) {
+ start = length - end;
+ } else {
+ start = 0;
+ }
+ end = length - 1;
+ } else {
+ if (start >= length) {
+ readFull = true;
+ if (end >= length) {
+ inValidRange = true;
+ } else {
+ start = 0;
+ end = length - 1;
+ }
+ } else {
+ if (end >= length) {
+ end = length - 1;
+ }
+ }
+ }
+ } else {
+ // Byte specification is not matching or start and endoffset provided
+ // are not matching with regex.
+ start = 0;
+ end = length - 1;
+ readFull = true;
+ }
+ rangeHeader = new RangeHeader(start, end, readFull, inValidRange);
+ return rangeHeader;
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3utils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3utils.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3utils.java
new file mode 100644
index 0000000..14f5d24
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3utils.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test class to test S3utils.
+ */
+public class TestS3utils {
+
+ @Test
+ public void testRangeHeaderParser() {
+
+ RangeHeader rangeHeader;
+
+
+ //range is with in file length
+ rangeHeader = S3utils.parseRangeHeader("bytes=0-8", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(8, rangeHeader.getEndOffset());
+ assertEquals(false, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ //range is with in file length, both start and end offset are same
+ rangeHeader = S3utils.parseRangeHeader("bytes=0-0", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(0, rangeHeader.getEndOffset());
+ assertEquals(false, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ //range is not with in file length, both start and end offset are greater
+ // than length
+ rangeHeader = S3utils.parseRangeHeader("bytes=11-10", 10);
+ assertEquals(true, rangeHeader.isInValidRange());
+
+ // range is satisfying, one of the range is with in the length. So, read
+ // full file
+ rangeHeader = S3utils.parseRangeHeader("bytes=11-8", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(9, rangeHeader.getEndOffset());
+ assertEquals(true, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ // bytes spec is wrong
+ rangeHeader = S3utils.parseRangeHeader("mb=11-8", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(9, rangeHeader.getEndOffset());
+ assertEquals(true, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ // range specified is invalid
+ rangeHeader = S3utils.parseRangeHeader("bytes=-11-8", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(9, rangeHeader.getEndOffset());
+ assertEquals(true, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ //Last n bytes
+ rangeHeader = S3utils.parseRangeHeader("bytes=-6", 10);
+ assertEquals(4, rangeHeader.getStartOffset());
+ assertEquals(9, rangeHeader.getEndOffset());
+ assertEquals(false, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ rangeHeader = S3utils.parseRangeHeader("bytes=-106", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(9, rangeHeader.getEndOffset());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+
+
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: HDDS-817. Create SCM metrics for
disk from node report. Contributed by Bharat Viswanadham.
Posted by br...@apache.org.
HDDS-817. Create SCM metrics for disk from node report. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0cc6794
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0cc6794
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0cc6794
Branch: refs/heads/HDFS-13891
Commit: d0cc679441da436d7004b38d0eb83af3891e6e09
Parents: 1a00b4e
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Nov 20 14:22:30 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Tue Nov 20 14:22:30 2018 +0800
----------------------------------------------------------------------
.../hadoop/hdds/scm/node/NodeManagerMXBean.java | 7 ++
.../hadoop/hdds/scm/node/NodeStateManager.java | 14 +++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 60 ++++++++++
.../hdds/scm/container/MockNodeManager.java | 9 ++
.../testutils/ReplicationNodeManagerMock.java | 5 +
.../ozone/scm/TestSCMNodeManagerMXBean.java | 112 +++++++++++++++++++
6 files changed, 207 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0cc6794/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java
index d84cf53..e1b51ef 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java
@@ -35,4 +35,11 @@ public interface NodeManagerMXBean {
* @return A state to number of nodes that in this state mapping
*/
Map<String, Integer> getNodeCount();
+
+ /**
+ * Get the disk metrics like capacity, usage and remaining based on the
+ * storage type.
+ */
+ Map<String, Long> getNodeInfo();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0cc6794/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
index cddd3ae..a26b0cc 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
@@ -274,6 +274,20 @@ public class NodeStateManager implements Runnable, Closeable {
}
/**
+ * Get information about the node.
+ *
+ * @param datanodeUUID datanode UUID
+ *
+ * @return DatanodeInfo
+ *
+ * @throws NodeNotFoundException if the node is not present
+ */
+ public DatanodeInfo getNode(UUID datanodeUUID)
+ throws NodeNotFoundException {
+ return nodeStateMap.getNodeInfo(datanodeUUID);
+ }
+
+ /**
* Updates the last heartbeat time of the node.
*
* @throws NodeNotFoundException if the node is not present
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0cc6794/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 24672cf..374ff90 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.node;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
@@ -58,6 +59,7 @@ import org.slf4j.LoggerFactory;
import javax.management.ObjectName;
import java.io.IOException;
import java.net.InetAddress;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -182,6 +184,16 @@ public class SCMNodeManager
SCMNodeStat stat;
try {
stat = nodeStateManager.getNodeStat(dnId);
+
+ // Updating the storage report for the datanode.
+ // I dont think we will get NotFound exception, as we are taking
+ // nodeInfo from nodeStateMap, as I see it is not being removed from
+ // the map, just we change the states. And during first time
+ // registration we call this, after adding to nodeStateMap. And also
+ // from eventhandler it is called only if it has node Report.
+ DatanodeInfo datanodeInfo = nodeStateManager.getNode(dnId);
+ datanodeInfo.updateStorageReports(nodeReport.getStorageReportList());
+
} catch (NodeNotFoundException e) {
LOG.debug("SCM updateNodeStat based on heartbeat from previous " +
"dead datanode {}", dnId);
@@ -361,6 +373,52 @@ public class SCMNodeManager
return nodeCountMap;
}
+ @Override
+ public Map<String, Long> getNodeInfo() {
+ long diskCapacity = 0L;
+ long diskUsed = 0L;
+ long diskRemaning = 0L;
+
+ long ssdCapacity = 0L;
+ long ssdUsed = 0L;
+ long ssdRemaining = 0L;
+
+ List<DatanodeDetails> healthyNodes = getNodes(NodeState.HEALTHY);
+ List<DatanodeDetails> staleNodes = getNodes(NodeState.STALE);
+
+ List<DatanodeDetails> datanodes = new ArrayList<>(healthyNodes);
+ datanodes.addAll(staleNodes);
+
+ for (DatanodeDetails datanodeDetails : datanodes) {
+ DatanodeInfo dnInfo = (DatanodeInfo) datanodeDetails;
+ List<StorageReportProto> storageReportProtos = dnInfo.getStorageReports();
+ for (StorageReportProto reportProto : storageReportProtos) {
+ if (reportProto.getStorageType() ==
+ StorageContainerDatanodeProtocolProtos.StorageTypeProto.DISK) {
+ diskCapacity += reportProto.getCapacity();
+ diskRemaning += reportProto.getRemaining();
+ diskUsed += reportProto.getScmUsed();
+ } else if (reportProto.getStorageType() ==
+ StorageContainerDatanodeProtocolProtos.StorageTypeProto.SSD) {
+ ssdCapacity += reportProto.getCapacity();
+ ssdRemaining += reportProto.getRemaining();
+ ssdUsed += reportProto.getScmUsed();
+ }
+ }
+ }
+
+ Map<String, Long> nodeInfo = new HashMap<>();
+ nodeInfo.put("DISKCapacity", diskCapacity);
+ nodeInfo.put("DISKUsed", diskUsed);
+ nodeInfo.put("DISKRemaining", diskRemaning);
+
+ nodeInfo.put("SSDCapacity", ssdCapacity);
+ nodeInfo.put("SSDUsed", ssdUsed);
+ nodeInfo.put("SSDRemaining", ssdRemaining);
+ return nodeInfo;
+ }
+
+
/**
* Get set of pipelines a datanode is part of.
* @param datanodeDetails - datanodeID
@@ -463,4 +521,6 @@ public class SCMNodeManager
public List<SCMCommand> getCommandQueue(UUID dnID) {
return commandQueue.getCommand(dnID);
}
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0cc6794/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index d5e68a8..69cc215 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -396,6 +396,15 @@ public class MockNodeManager implements NodeManager {
return nodeCountMap;
}
+ @Override
+ public Map<String, Long> getNodeInfo() {
+ Map<String, Long> nodeInfo = new HashMap<>();
+ nodeInfo.put("Capacity", aggregateStat.getCapacity().get());
+ nodeInfo.put("Used", aggregateStat.getScmUsed().get());
+ nodeInfo.put("Remaining", aggregateStat.getRemaining().get());
+ return nodeInfo;
+ }
+
/**
* Makes it easy to add a container.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0cc6794/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 2d786c5..3f31708 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -73,6 +73,11 @@ public class ReplicationNodeManagerMock implements NodeManager {
return null;
}
+ @Override
+ public Map<String, Long> getNodeInfo() {
+ return null;
+ }
+
/**
* Gets all Live Datanodes that is currently communicating with SCM.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0cc6794/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java
new file mode 100644
index 0000000..25a18d4
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.TabularData;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Class which tests the SCMNodeManagerInfo Bean.
+ */
+public class TestSCMNodeManagerMXBean {
+ public static final Log LOG = LogFactory.getLog(TestSCMMXBean.class);
+ private static int numOfDatanodes = 3;
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration conf;
+ private static StorageContainerManager scm;
+ private static MBeanServer mbs;
+
+ @BeforeClass
+ public static void init() throws IOException, TimeoutException,
+ InterruptedException {
+ conf = new OzoneConfiguration();
+ conf.set(OZONE_SCM_STALENODE_INTERVAL, "60000ms");
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(numOfDatanodes)
+ .build();
+ cluster.waitForClusterToBeReady();
+ scm = cluster.getStorageContainerManager();
+ mbs = ManagementFactory.getPlatformMBeanServer();
+ }
+
+ @Test
+ public void testDiskUsage() throws Exception {
+ ObjectName bean = new ObjectName(
+ "Hadoop:service=SCMNodeManager,"
+ + "name=SCMNodeManagerInfo");
+
+ TabularData data = (TabularData) mbs.getAttribute(bean, "NodeInfo");
+ Map<String, Long> datanodeInfo = scm.getScmNodeManager().getNodeInfo();
+ verifyEquals(data, datanodeInfo);
+ }
+
+ @Test
+ public void testNodeCount() throws Exception {
+ ObjectName bean = new ObjectName(
+ "Hadoop:service=SCMNodeManager,"
+ + "name=SCMNodeManagerInfo");
+
+ TabularData data = (TabularData) mbs.getAttribute(bean, "NodeCount");
+ Map<String, Integer> nodeCount = scm.getScmNodeManager().getNodeCount();
+ Map<String, Long> nodeCountLong = new HashMap<>();
+ nodeCount.forEach((k, v) -> nodeCountLong.put(k, new Long(v)));
+ verifyEquals(data, nodeCountLong);
+ }
+
+ private void verifyEquals(TabularData actualData, Map<String, Long>
+ expectedData) {
+ if (actualData == null || expectedData == null) {
+ fail("Data should not be null.");
+ }
+ for (Object obj : actualData.values()) {
+ assertTrue(obj instanceof CompositeData);
+ CompositeData cds = (CompositeData) obj;
+ assertEquals(2, cds.values().size());
+ Iterator<?> it = cds.values().iterator();
+ String key = it.next().toString();
+ String value = it.next().toString();
+ long num = Long.parseLong(value);
+ assertTrue(expectedData.containsKey(key));
+ assertEquals(expectedData.remove(key).longValue(), num);
+ }
+ assertTrue(expectedData.isEmpty());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-8778. Add command line
interface to invoke interactive docker shell. Contributed by Eric Yang
Posted by br...@apache.org.
YARN-8778. Add command line interface to invoke interactive docker shell. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27ffec7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27ffec7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27ffec7b
Branch: refs/heads/HDFS-13891
Commit: 27ffec7ba77ba3d0b88b97b34aca5356e2e258f4
Parents: 397f523
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Nov 19 17:59:03 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Mon Nov 19 17:59:12 2018 -0800
----------------------------------------------------------------------
LICENSE.txt | 40 +++++
NOTICE.txt | 11 +-
.../hadoop-client-minicluster/pom.xml | 14 +-
.../hadoop/mapred/ResourceMgrDelegate.java | 7 +
hadoop-project/pom.xml | 11 ++
.../yarn/api/records/ShellContainerCommand.java | 32 ++++
.../yarn/service/client/ApiServiceClient.java | 62 +-------
.../client/TestSecureApiServiceClient.java | 4 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 8 +
.../client/api/ContainerShellWebSocket.java | 156 +++++++++++++++++++
.../hadoop/yarn/client/api/YarnClient.java | 15 ++
.../yarn/client/api/impl/YarnClientImpl.java | 58 +++++++
.../hadoop/yarn/client/cli/ApplicationCLI.java | 34 +++-
.../yarn/client/util/YarnClientUtils.java | 67 ++++++++
.../hadoop/yarn/client/cli/TestYarnCLI.java | 1 +
15 files changed, 455 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 81eb32f..dd7195e 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -2813,3 +2813,43 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations under
the License.
+
+--------------------------------------------------------------------------------
+Jline 3.9.0
+The binary distribution of this product bundles these dependencies under the
+following license:
+
+Copyright (c) 2002-2018, the original author or authors.
+All rights reserved.
+
+http://www.opensource.org/licenses/bsd-license.php
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the following
+conditions are met:
+
+Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with
+the distribution.
+
+Neither the name of JLine nor the names of its contributors
+may be used to endorse or promote products derived from this
+software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index d6e5488..5553144 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -614,10 +614,17 @@ which has the following notices:
Expert Group and released to the public domain, as explained at
http://creativecommons.org/publicdomain/zero/1.0/
-
The source and binary distribution of this product bundles modified version of
github.com/awslabs/aws-js-s3-explorer licensed under Apache 2.0 license
with the following notice:
AWS JavaScript S3 Explorer
-Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
\ No newline at end of file
+Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+The binary distribution of this product bundles binaries of
+jline 3.9.0 (https://github.com/jline/jline3)
+
+ * LICENSE:
+ * license/LICENSE.jline3.txt (BSD License)
+ * HOMEPAGE:
+ * https://github.com/jline/jline3
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index c356b19..0b820b6 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -788,7 +788,19 @@
<filter>
<artifact>org.eclipse.jetty.websocket:javax-websocket-server-impl</artifact>
<excludes>
- <exclude>*</exclude>
+ <exclude>*/**</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>org.eclipse.jetty.websocket:websocket-client</artifact>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>org.eclipse.jetty:jetty-io</artifact>
+ <excludes>
+ <exclude>*/**</exclude>
</excludes>
</filter>
</filters>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index 2cb3716..6fff094 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
+import org.apache.hadoop.yarn.api.records.ShellContainerCommand;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
@@ -560,4 +561,10 @@ public class ResourceMgrDelegate extends YarnClient {
Set<String> hostNames) throws YarnException, IOException {
return client.getNodeToAttributes(hostNames);
}
+
+ @Override
+ public void shellToContainer(ContainerId containerId,
+ ShellContainerCommand command) throws IOException {
+ throw new IOException("Operation is not supported.");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index c985d7b..6f83280 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -160,6 +160,7 @@
<junit.jupiter.version>5.3.1</junit.jupiter.version>
<junit.vintage.version>5.3.1</junit.vintage.version>
<junit.platform.version>1.3.1</junit.platform.version>
+ <jline.version>3.9.0</jline.version>
</properties>
<dependencyManagement>
@@ -697,6 +698,11 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty.websocket</groupId>
+ <artifactId>websocket-client</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
<version>2.1</version>
@@ -1178,6 +1184,11 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.jline</groupId>
+ <artifactId>jline</artifactId>
+ <version>${jline.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<version>${hsqldb.version}</version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ShellContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ShellContainerCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ShellContainerCommand.java
new file mode 100644
index 0000000..07acb9d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ShellContainerCommand.java
@@ -0,0 +1,32 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+/**
+ * Enumeration of various signal container commands.
+ */
+@Public
+@Evolving
+public enum ShellContainerCommand {
+ BASH,
+ SH
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index 38cfd11..63378dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -21,8 +21,6 @@ import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
import java.io.File;
import java.io.IOException;
import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.security.PrivilegedExceptionAction;
import java.text.MessageFormat;
import java.util.List;
import java.util.Map;
@@ -40,13 +38,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.util.YarnClientUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.api.records.Component;
@@ -60,11 +57,6 @@ import org.apache.hadoop.yarn.service.conf.RestApiConstants;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.util.RMHAUtils;
import org.eclipse.jetty.util.UrlEncoded;
-import org.ietf.jgss.GSSContext;
-import org.ietf.jgss.GSSException;
-import org.ietf.jgss.GSSManager;
-import org.ietf.jgss.GSSName;
-import org.ietf.jgss.Oid;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,54 +85,6 @@ public class ApiServiceClient extends AppAdminClient {
}
/**
- * Generate SPNEGO challenge request token.
- *
- * @param server - hostname to contact
- * @throws IOException
- * @throws InterruptedException
- */
- String generateToken(String server) throws IOException, InterruptedException {
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- LOG.debug("The user credential is {}", currentUser);
- String challenge = currentUser
- .doAs(new PrivilegedExceptionAction<String>() {
- @Override
- public String run() throws Exception {
- try {
- // This Oid for Kerberos GSS-API mechanism.
- Oid mechOid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID");
- GSSManager manager = GSSManager.getInstance();
- // GSS name for server
- GSSName serverName = manager.createName("HTTP@" + server,
- GSSName.NT_HOSTBASED_SERVICE);
- // Create a GSSContext for authentication with the service.
- // We're passing client credentials as null since we want them to
- // be read from the Subject.
- GSSContext gssContext = manager.createContext(
- serverName.canonicalize(mechOid), mechOid, null,
- GSSContext.DEFAULT_LIFETIME);
- gssContext.requestMutualAuth(true);
- gssContext.requestCredDeleg(true);
- // Establish context
- byte[] inToken = new byte[0];
- byte[] outToken = gssContext.initSecContext(inToken, 0,
- inToken.length);
- gssContext.dispose();
- // Base64 encoded and stringified token for server
- LOG.debug("Got valid challenge for host {}", serverName);
- return new String(BASE_64_CODEC.encode(outToken),
- StandardCharsets.US_ASCII);
- } catch (GSSException | IllegalAccessException
- | NoSuchFieldException | ClassNotFoundException e) {
- LOG.error("Error: {}", e);
- throw new AuthenticationException(e);
- }
- }
- });
- return challenge;
- }
-
- /**
* Calculate Resource Manager address base on working REST API.
*/
String getRMWebAddress() {
@@ -177,7 +121,7 @@ public class ApiServiceClient extends AppAdminClient {
.resource(sb.toString()).type(MediaType.APPLICATION_JSON);
if (useKerberos) {
String[] server = host.split(":");
- String challenge = generateToken(server[0]);
+ String challenge = YarnClientUtils.generateToken(server[0]);
builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
challenge);
LOG.debug("Authorization: Negotiate {}", challenge);
@@ -289,7 +233,7 @@ public class ApiServiceClient extends AppAdminClient {
if (conf.get("hadoop.http.authentication.type").equals("kerberos")) {
try {
URI url = new URI(requestPath);
- String challenge = generateToken(url.getHost());
+ String challenge = YarnClientUtils.generateToken(url.getHost());
builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + challenge);
} catch (Exception e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
index f955064..1ec8d41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.yarn.client.util.YarnClientUtils;
import org.apache.log4j.Logger;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
@@ -169,8 +170,7 @@ public class TestSecureApiServiceClient extends KerberosSecurityTestcase {
public void testHttpSpnegoChallenge() throws Exception {
UserGroupInformation.loginUserFromKeytab(clientPrincipal, keytabFile
.getCanonicalPath());
- asc = new ApiServiceClient();
- String challenge = asc.generateToken("localhost");
+ String challenge = YarnClientUtils.generateToken("localhost");
assertNotNull(challenge);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index 2e0c777..dd672c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -54,6 +54,10 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty.websocket</groupId>
+ <artifactId>websocket-client</artifactId>
+ </dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
@@ -127,6 +131,10 @@
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.jline</groupId>
+ <artifactId>jline</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java
new file mode 100644
index 0000000..4b7b2ac
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.eclipse.jetty.websocket.api.Session;
+import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose;
+import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect;
+import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
+import org.eclipse.jetty.websocket.api.annotations.WebSocket;
+import org.jline.terminal.Terminal;
+import org.jline.terminal.TerminalBuilder;
+import org.jline.reader.LineReader;
+import org.jline.reader.LineReaderBuilder;
+import org.jline.reader.impl.LineReaderImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Web socket for establishing interactive command shell connection through
+ * Node Manage to container executor.
+ */
+@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce", "YARN" })
+@InterfaceStability.Unstable
+
+@WebSocket
+public class ContainerShellWebSocket {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerShellWebSocket.class);
+
+ private Session mySession;
+ private Terminal terminal;
+ private LineReader reader;
+
+ @OnWebSocketMessage
+ public void onText(Session session, String message) throws IOException {
+ terminal.output().write(message.getBytes(Charset.forName("UTF-8")));
+ terminal.output().flush();
+ }
+
+ @OnWebSocketConnect
+ public void onConnect(Session s) {
+ initTerminal(s);
+ LOG.info(s.getRemoteAddress().getHostString() + " connected!");
+ }
+
+ @OnWebSocketClose
+ public void onClose(Session session, int status, String reason) {
+ if (status==1000) {
+ LOG.info(session.getRemoteAddress().getHostString() +
+ " closed, status: " + status);
+ } else {
+ LOG.warn(session.getRemoteAddress().getHostString() +
+ " closed, status: " + status + " Reason: " + reason);
+ }
+ }
+
+ public void run() {
+ try {
+ Reader consoleReader = new Reader();
+ Thread inputThread = new Thread(consoleReader, "consoleReader");
+ inputThread.start();
+ while (mySession.isOpen()) {
+ mySession.getRemote().flush();
+ if (consoleReader.hasData()) {
+ String message = consoleReader.read();
+ mySession.getRemote().sendString(message);
+ mySession.getRemote().sendString("\r");
+ }
+ String message = "1{}";
+ mySession.getRemote().sendString(message);
+ Thread.sleep(100);
+ mySession.getRemote().flush();
+ }
+ inputThread.join();
+ } catch (IOException | InterruptedException e) {
+ try {
+ mySession.disconnect();
+ } catch (IOException e1) {
+ LOG.error("Error closing connection: ", e1);
+ }
+ }
+ }
+
+ protected void initTerminal(final Session session) {
+ try {
+ this.mySession = session;
+ try {
+ terminal = TerminalBuilder.builder()
+ .system(true)
+ .build();
+ } catch (IOException t) {
+ terminal = TerminalBuilder.builder()
+ .system(false)
+ .streams(System.in, (OutputStream) System.out)
+ .build();
+ }
+ reader = LineReaderBuilder.builder()
+ .terminal(terminal)
+ .build();
+ } catch (IOException e) {
+ session.close(1002, e.getMessage());
+ }
+ }
+
+ class Reader implements Runnable {
+ private StringBuilder sb = new StringBuilder();
+ private boolean hasData = false;
+
+ public String read() {
+ try {
+ return sb.toString();
+ } finally {
+ hasData = false;
+ sb.setLength(0);
+ }
+ }
+
+ public boolean hasData() {
+ return hasData;
+ }
+
+ @Override
+ public void run() {
+ while (true) {
+ int c = ((LineReaderImpl) reader).readCharacter();
+ if (c == 10 || c == 13) {
+ hasData = true;
+ continue;
+ }
+ sb.append(new String(Character.toChars(c)));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 59fa6a8..b59831e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
+import org.apache.hadoop.yarn.api.records.ShellContainerCommand;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -958,4 +959,18 @@ public abstract class YarnClient extends AbstractService {
public abstract Map<String, Set<NodeAttribute>> getNodeToAttributes(
Set<String> hostNames) throws YarnException, IOException;
+ /**
+ * <p>
+ * The interface used by client to get a shell to a container.
+ * </p>
+ *
+ * @param containerId Container ID
+ * @param command Shell type
+ * @throws IOException if connection fails.
+ */
+ @Public
+ @Unstable
+ public abstract void shellToContainer(ContainerId containerId,
+ ShellContainerCommand command) throws IOException;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 227f7ed..3ec371c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.client.api.impl;
import java.io.IOException;
+import java.net.URI;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.EnumSet;
@@ -27,6 +28,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Future;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -111,15 +113,18 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
+import org.apache.hadoop.yarn.api.records.ShellContainerCommand;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.client.api.AHSClient;
+import org.apache.hadoop.yarn.client.api.ContainerShellWebSocket;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.client.util.YarnClientUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
@@ -132,6 +137,10 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.eclipse.jetty.websocket.api.Session;
+import org.eclipse.jetty.websocket.api.WebSocketException;
+import org.eclipse.jetty.websocket.client.ClientUpgradeRequest;
+import org.eclipse.jetty.websocket.client.WebSocketClient;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
@@ -1074,4 +1083,53 @@ public class YarnClientImpl extends YarnClient {
GetNodesToAttributesRequest.newInstance(hostNames);
return rmClient.getNodesToAttributes(request).getNodeToAttributes();
}
+
+ @Override
+ public void shellToContainer(ContainerId containerId,
+ ShellContainerCommand command) throws IOException {
+ try {
+ GetContainerReportRequest request = Records
+ .newRecord(GetContainerReportRequest.class);
+ request.setContainerId(containerId);
+ GetContainerReportResponse response = rmClient
+ .getContainerReport(request);
+ URI nodeHttpAddress = new URI(response.getContainerReport()
+ .getNodeHttpAddress());
+ String host = nodeHttpAddress.getHost();
+ int port = nodeHttpAddress.getPort();
+ String scheme = nodeHttpAddress.getScheme();
+ String protocol = "ws://";
+ if (scheme.equals("https")) {
+ protocol = "wss://";
+ }
+ WebSocketClient client = new WebSocketClient();
+ URI uri = URI.create(protocol + host + ":" + port + "/container/" +
+ containerId);
+ try {
+ client.start();
+ // The socket that receives events
+ ContainerShellWebSocket socket = new ContainerShellWebSocket();
+ ClientUpgradeRequest upgradeRequest = new ClientUpgradeRequest();
+ if (UserGroupInformation.isSecurityEnabled()) {
+ String challenge = YarnClientUtils.generateToken(host);
+ upgradeRequest.setHeader("Authorization", "Negotiate " + challenge);
+ }
+ // Attempt Connect
+ Future<Session> fut = client.connect(socket, uri, upgradeRequest);
+ // Wait for Connect
+ Session session = fut.get();
+ // Send a message
+ session.getRemote().sendString("stty -echo");
+ session.getRemote().sendString("\r");
+ session.getRemote().flush();
+ socket.run();
+ } finally {
+ client.stop();
+ }
+ } catch (WebSocketException e) {
+ LOG.debug("Websocket exception: " + e.getMessage());
+ } catch (Throwable t) {
+ LOG.error("Fail to shell to container: " + t.getMessage());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 480ea23..2b34154 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ShellContainerCommand;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.AppAdminClient;
@@ -111,6 +112,7 @@ public class ApplicationCLI extends YarnCLI {
public static final String COMPONENTS = "components";
public static final String VERSION = "version";
public static final String STATES = "states";
+ public static final String SHELL_CMD = "shell";
private static String firstArg = null;
@@ -311,6 +313,8 @@ public class ApplicationCLI extends YarnCLI {
opts.getOption(LIST_CMD).setArgName("Application ID");
opts.getOption(FAIL_CMD).setArgName("Application Attempt ID");
} else if (title != null && title.equalsIgnoreCase(CONTAINER)) {
+ opts.addOption(SHELL_CMD, true,
+ "Run a shell in the container.");
opts.addOption(STATUS_CMD, true,
"Prints the status of the container.");
opts.addOption(LIST_CMD, true,
@@ -323,6 +327,7 @@ public class ApplicationCLI extends YarnCLI {
"app version, -components to filter instances based on component " +
"names, -states to filter instances based on instance state.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
+ opts.getOption(SHELL_CMD).setArgName("Container ID");
opts.getOption(STATUS_CMD).setArgName("Container ID");
opts.getOption(LIST_CMD).setArgName("Application Name or Attempt ID");
opts.addOption(APP_TYPE_CMD, true, "Works with -list to " +
@@ -552,6 +557,19 @@ public class ApplicationCLI extends YarnCLI {
command = SignalContainerCommand.valueOf(signalArgs[1]);
}
signalToContainer(containerId, command);
+ } else if (cliParser.hasOption(SHELL_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, SHELL_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ final String[] shellArgs = cliParser.getOptionValues(SHELL_CMD);
+ final String containerId = shellArgs[0];
+ ShellContainerCommand command =
+ ShellContainerCommand.BASH;
+ if (shellArgs.length == 2) {
+ command = ShellContainerCommand.valueOf(shellArgs[1]);
+ }
+ shellToContainer(containerId, command);
} else if (cliParser.hasOption(LAUNCH_CMD)) {
if (hasAnyOtherCLIOptions(cliParser, opts, LAUNCH_CMD, APP_TYPE_CMD,
UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) {
@@ -806,7 +824,7 @@ public class ApplicationCLI extends YarnCLI {
}
/**
- * Signals the containerId
+ * Signals the containerId.
*
* @param containerIdStr the container id
* @param command the signal command
@@ -820,6 +838,20 @@ public class ApplicationCLI extends YarnCLI {
}
/**
+ * Shell to the containerId.
+ *
+ * @param containerIdStr the container id
+ * @param command the shell command
+ * @throws YarnException
+ */
+ private void shellToContainer(String containerIdStr,
+ ShellContainerCommand command) throws YarnException, IOException {
+ ContainerId containerId = ContainerId.fromString(containerIdStr);
+ sysout.println("Shelling to container " + containerIdStr);
+ client.shellToContainer(containerId, command);
+ }
+
+ /**
* It prints the usage of the command
*
* @param opts
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
index 1717675..abed6c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
@@ -19,15 +19,29 @@ package org.apache.hadoop.yarn.client.util;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
import com.google.common.collect.ImmutableSet;
+
+import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.ietf.jgss.GSSContext;
+import org.ietf.jgss.GSSException;
+import org.ietf.jgss.GSSManager;
+import org.ietf.jgss.GSSName;
+import org.ietf.jgss.Oid;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class is a container for utility methods that are useful when creating
@@ -35,6 +49,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
*/
public abstract class YarnClientUtils {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(YarnClientUtils.class);
+ private static final Base64 BASE_64_CODEC = new Base64(0);
private static final String ADD_LABEL_FORMAT_ERR_MSG =
"Input format for adding node-labels is not correct, it should be "
+ "labelName1[(exclusive=true/false)],LabelName2[] ..";
@@ -187,4 +204,54 @@ public abstract class YarnClientUtils {
return yarnConf;
}
+
+ /**
+ * Generate SPNEGO challenge request token.
+ *
+ * @param server - hostname to contact
+ * @throws IOException thrown if doAs failed
+ * @throws InterruptedException thrown if doAs is interrupted
+ * @return SPNEGO token challenge
+ */
+ public static String generateToken(String server) throws IOException,
+ InterruptedException {
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ LOG.debug("The user credential is {}", currentUser);
+ String challenge = currentUser
+ .doAs(new PrivilegedExceptionAction<String>() {
+ @Override
+ public String run() throws Exception {
+ try {
+ // This Oid for Kerberos GSS-API mechanism.
+ Oid mechOid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID");
+ GSSManager manager = GSSManager.getInstance();
+ // GSS name for server
+ GSSName serverName = manager.createName("HTTP@" + server,
+ GSSName.NT_HOSTBASED_SERVICE);
+ // Create a GSSContext for authentication with the service.
+ // We're passing client credentials as null since we want them to
+ // be read from the Subject.
+ GSSContext gssContext = manager.createContext(
+ serverName.canonicalize(mechOid), mechOid, null,
+ GSSContext.DEFAULT_LIFETIME);
+ gssContext.requestMutualAuth(true);
+ gssContext.requestCredDeleg(true);
+ // Establish context
+ byte[] inToken = new byte[0];
+ byte[] outToken = gssContext.initSecContext(inToken, 0,
+ inToken.length);
+ gssContext.dispose();
+ // Base64 encoded and stringified token for server
+ LOG.debug("Got valid challenge for host {}", serverName);
+ return new String(BASE_64_CODEC.encode(outToken),
+ StandardCharsets.US_ASCII);
+ } catch (GSSException | IllegalAccessException
+ | NoSuchFieldException | ClassNotFoundException e) {
+ LOG.error("Error: {}", e);
+ throw new AuthenticationException(e);
+ }
+ }
+ });
+ return challenge;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffec7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 9b1e863..672e3d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -2317,6 +2317,7 @@ public class TestYarnCLI {
pw.println(" -components <arg> Works with -list to filter instances based on input comma-separated list of component names.");
pw.println(" -help Displays help for all commands.");
pw.println(" -list <Application Name or Attempt ID> List containers for application attempt when application attempt ID is provided. When application name is provided, then it finds the instances of the application based on app's own implementation, and -appTypes option must be specified unless it is the default yarn-service type. With app name, it supports optional use of -version to filter instances based on app version, -components to filter instances based on component names, -states to filter instances based on instance state.");
+ pw.println(" -shell <Container ID> Run a shell in the container.");
pw.println(" -signal <container ID [signal command]> Signal the container.");
pw.println("The available signal commands are ");
pw.println(java.util.Arrays.asList(SignalContainerCommand.values()));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HDFS-14015. Improve error handling
in hdfsThreadDestructor in native thread local storage
Posted by br...@apache.org.
HDFS-14015. Improve error handling in hdfsThreadDestructor in native thread local storage
Change-Id: Ida1e888c9231b9e46081338e3a206d8f6faabd36
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e56d9f26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e56d9f26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e56d9f26
Branch: refs/heads/HDFS-13891
Commit: e56d9f261866bfa456590061d7e3f360f22c5a86
Parents: 2937499
Author: Daniel Templeton <te...@apache.org>
Authored: Fri Nov 16 16:24:10 2018 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Fri Nov 16 16:24:40 2018 -0800
----------------------------------------------------------------------
.../libhdfs/os/posix/thread_local_storage.c | 65 +++++++++++++++++++-
.../libhdfs/os/windows/thread_local_storage.c | 63 ++++++++++++++++++-
2 files changed, 123 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e56d9f26/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/posix/thread_local_storage.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/posix/thread_local_storage.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/posix/thread_local_storage.c
index e6b59d6..22e2fcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/posix/thread_local_storage.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/posix/thread_local_storage.c
@@ -23,12 +23,17 @@
#include <pthread.h>
#include <stdio.h>
+#define UNKNOWN "UNKNOWN"
+#define MAXTHRID 256
+
/** Key that allows us to retrieve thread-local storage */
static pthread_key_t gTlsKey;
/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */
static int gTlsKeyInitialized = 0;
+static void get_current_thread_id(JNIEnv* env, char* id, int max);
+
/**
* The function that is called whenever a thread with libhdfs thread local data
* is destroyed.
@@ -41,16 +46,26 @@ void hdfsThreadDestructor(void *v)
struct ThreadLocalState *state = (struct ThreadLocalState*)v;
JNIEnv *env = state->env;;
jint ret;
+ char thr_name[MAXTHRID];
/* Detach the current thread from the JVM */
if (env) {
ret = (*env)->GetJavaVM(env, &vm);
- if (ret) {
+
+ if (ret != 0) {
fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n",
ret);
(*env)->ExceptionDescribe(env);
} else {
- (*vm)->DetachCurrentThread(vm);
+ ret = (*vm)->DetachCurrentThread(vm);
+
+ if (ret != JNI_OK) {
+ (*env)->ExceptionDescribe(env);
+ get_current_thread_id(env, thr_name, MAXTHRID);
+
+ fprintf(stderr, "hdfsThreadDestructor: Unable to detach thread %s "
+ "from the JVM. Error code: %d\n", thr_name, ret);
+ }
}
}
@@ -62,13 +77,57 @@ void hdfsThreadDestructor(void *v)
free(state);
}
+static void get_current_thread_id(JNIEnv* env, char* id, int max) {
+ jclass cls;
+ jmethodID mth;
+ jobject thr;
+ jstring thr_name;
+ jlong thr_id = 0;
+ const char *thr_name_str;
+
+ cls = (*env)->FindClass(env, "java/lang/Thread");
+ mth = (*env)->GetStaticMethodID(env, cls, "currentThread",
+ "()Ljava/lang/Thread;");
+ thr = (*env)->CallStaticObjectMethod(env, cls, mth);
+
+ if (thr != NULL) {
+ mth = (*env)->GetMethodID(env, cls, "getId", "()J");
+ thr_id = (*env)->CallLongMethod(env, thr, mth);
+ (*env)->ExceptionDescribe(env);
+
+ mth = (*env)->GetMethodID(env, cls, "toString", "()Ljava/lang/String;");
+ thr_name = (jstring)(*env)->CallObjectMethod(env, thr, mth);
+
+ if (thr_name != NULL) {
+ thr_name_str = (*env)->GetStringUTFChars(env, thr_name, NULL);
+
+ // Treating the jlong as a long *should* be safe
+ snprintf(id, max, "%s:%ld", thr_name_str, thr_id);
+
+ // Release the char*
+ (*env)->ReleaseStringUTFChars(env, thr_name, thr_name_str);
+ } else {
+ (*env)->ExceptionDescribe(env);
+
+ // Treating the jlong as a long *should* be safe
+ snprintf(id, max, "%s:%ld", UNKNOWN, thr_id);
+ }
+ } else {
+ (*env)->ExceptionDescribe(env);
+ snprintf(id, max, "%s", UNKNOWN);
+ }
+
+ // Make sure the id is null terminated in case we overflow the max length
+ id[max - 1] = '\0';
+}
+
struct ThreadLocalState* threadLocalStorageCreate()
{
struct ThreadLocalState *state;
state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState));
if (state == NULL) {
fprintf(stderr,
- "threadLocalStorageSet: OOM - Unable to allocate thread local state\n");
+ "threadLocalStorageCreate: OOM - Unable to allocate thread local state\n");
return NULL;
}
state->lastExceptionStackTrace = NULL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e56d9f26/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/thread_local_storage.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/thread_local_storage.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/thread_local_storage.c
index 28d014d..a6f48fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/thread_local_storage.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/thread_local_storage.c
@@ -23,9 +23,14 @@
#include <stdio.h>
#include <windows.h>
+#define UNKNOWN "UNKNOWN"
+#define MAXTHRID 256
+
/** Key that allows us to retrieve thread-local storage */
static DWORD gTlsIndex = TLS_OUT_OF_INDEXES;
+static void get_current_thread_id(JNIEnv* env, char* id, int max);
+
/**
* If the current thread has a JNIEnv in thread-local storage, then detaches the
* current thread from the JVM and also frees up the ThreadLocalState object.
@@ -36,6 +41,8 @@ static void detachCurrentThreadFromJvm()
JNIEnv *env = NULL;
JavaVM *vm;
jint ret;
+ char thr_name[MAXTHRID];
+
if (threadLocalStorageGet(&state) || !state) {
return;
}
@@ -50,7 +57,15 @@ static void detachCurrentThreadFromJvm()
ret);
(*env)->ExceptionDescribe(env);
} else {
- (*vm)->DetachCurrentThread(vm);
+ ret = (*vm)->DetachCurrentThread(vm);
+
+ if (ret != JNI_OK) {
+ (*env)->ExceptionDescribe(env);
+ get_current_thread_id(env, thr_name, MAXTHRID);
+
+ fprintf(stderr, "detachCurrentThreadFromJvm: Unable to detach thread %s "
+ "from the JVM. Error code: %d\n", thr_name, ret);
+ }
}
/* Free exception strings */
@@ -61,6 +76,50 @@ static void detachCurrentThreadFromJvm()
free(state);
}
+static void get_current_thread_id(JNIEnv* env, char* id, int max) {
+ jclass cls;
+ jmethodID mth;
+ jobject thr;
+ jstring thr_name;
+ jlong thr_id = 0;
+ const char *thr_name_str;
+
+ cls = (*env)->FindClass(env, "java/lang/Thread");
+ mth = (*env)->GetStaticMethodID(env, cls, "currentThread",
+ "()Ljava/lang/Thread;");
+ thr = (*env)->CallStaticObjectMethod(env, cls, mth);
+
+ if (thr != NULL) {
+ mth = (*env)->GetMethodID(env, cls, "getId", "()J");
+ thr_id = (*env)->CallLongMethod(env, thr, mth);
+ (*env)->ExceptionDescribe(env);
+
+ mth = (*env)->GetMethodID(env, cls, "toString", "()Ljava/lang/String;");
+ thr_name = (jstring)(*env)->CallObjectMethod(env, thr, mth);
+
+ if (thr_name != NULL) {
+ thr_name_str = (*env)->GetStringUTFChars(env, thr_name, NULL);
+
+ // Treating the jlong as a long *should* be safe
+ snprintf(id, max, "%s:%ld", thr_name_str, thr_id);
+
+ // Release the char*
+ (*env)->ReleaseStringUTFChars(env, thr_name, thr_name_str);
+ } else {
+ (*env)->ExceptionDescribe(env);
+
+ // Treating the jlong as a long *should* be safe
+ snprintf(id, max, "%s:%ld", UNKNOWN, thr_id);
+ }
+ } else {
+ (*env)->ExceptionDescribe(env);
+ snprintf(id, max, "%s", UNKNOWN);
+ }
+
+ // Make sure the id is null terminated in case we overflow the max length
+ id[max - 1] = '\0';
+}
+
void hdfsThreadDestructor(void *v)
{
// Ignore 'v' since it will contain the state and we will obtain it in the below
@@ -148,7 +207,7 @@ struct ThreadLocalState* threadLocalStorageCreate()
state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState));
if (state == NULL) {
fprintf(stderr,
- "threadLocalStorageSet: OOM - Unable to allocate thread local state\n");
+ "threadLocalStorageCreate: OOM - Unable to allocate thread local state\n");
return NULL;
}
state->lastExceptionStackTrace = NULL;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-8960. [Submarine] Can't get
submarine service status using the command of "yarn app -status" under
security environment. (Zac Zhou via wangda)
Posted by br...@apache.org.
YARN-8960. [Submarine] Can't get submarine service status using the command of "yarn app -status" under security environment. (Zac Zhou via wangda)
Change-Id: I21b1addc9c32817650ea744a8f2e6b5602f2f4d4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b238144
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b238144
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b238144
Branch: refs/heads/HDFS-13891
Commit: 8b2381441558cd49b4c940b0760c8accbb2a5567
Parents: 6357803
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Nov 19 08:55:30 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Nov 19 08:55:30 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/submarine/client/cli/Cli.java | 3 -
.../yarn/submarine/client/cli/CliConstants.java | 3 +
.../yarn/submarine/client/cli/CliUtils.java | 56 ++++++++++++++++++
.../yarn/submarine/client/cli/RunJobCli.java | 12 +++-
.../client/cli/param/RunJobParameters.java | 46 ++++++++++++++-
.../yarn/submarine/common/ClientContext.java | 15 +++--
.../fs/DefaultRemoteDirectoryManager.java | 3 +-
.../yarnservice/YarnServiceJobSubmitter.java | 61 +++++++++++++++++---
.../client/cli/TestRunJobCliParsing.java | 9 ++-
9 files changed, 188 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java
index b4c5e4c..69189f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java
@@ -16,7 +16,6 @@ package org.apache.hadoop.yarn.submarine.client.cli;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.submarine.common.ClientContext;
-import org.apache.hadoop.yarn.submarine.common.fs.DefaultRemoteDirectoryManager;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory;
import org.slf4j.Logger;
@@ -43,8 +42,6 @@ public class Cli {
Configuration conf = new YarnConfiguration();
ClientContext clientContext = new ClientContext();
clientContext.setConfiguration(conf);
- clientContext.setRemoteDirectoryManager(
- new DefaultRemoteDirectoryManager(clientContext));
RuntimeFactory runtimeFactory = RuntimeFactory.getRuntimeFactory(
clientContext);
clientContext.setRuntimeFactory(runtimeFactory);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
index 454ff1c..2d7a472 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
@@ -52,4 +52,7 @@ public class CliConstants {
public static final String QUICKLINK = "quicklink";
public static final String TENSORBOARD_DOCKER_IMAGE =
"tensorboard_docker_image";
+ public static final String KEYTAB = "keytab";
+ public static final String PRINCIPAL = "principal";
+ public static final String DISTRIBUTE_KEYTAB = "distribute_keytab";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
index f85c82a..588a9e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
@@ -14,22 +14,33 @@
package org.apache.hadoop.yarn.submarine.client.cli;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.submarine.client.cli.param.RunJobParameters;
+import org.apache.hadoop.yarn.submarine.common.exception.SubmarineRuntimeException;
import org.apache.hadoop.yarn.submarine.common.fs.RemoteDirectoryManager;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.hadoop.yarn.submarine.client.cli.CliConstants.KEYTAB;
+import static org.apache.hadoop.yarn.submarine.client.cli.CliConstants.PRINCIPAL;
+
public class CliUtils {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CliUtils.class);
private final static String RES_PATTERN = "^[^=]+=\\d+\\s?\\w*$";
/**
* Replace patterns inside cli
@@ -161,4 +172,49 @@ public class CliUtils {
return false;
}
+
+ public static void doLoginIfSecure(String keytab, String principal) throws
+ IOException {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return;
+ }
+
+ if (StringUtils.isEmpty(keytab) || StringUtils.isEmpty(principal)) {
+ if (StringUtils.isNotEmpty(keytab)) {
+ SubmarineRuntimeException e = new SubmarineRuntimeException("The " +
+ "parameter of " + PRINCIPAL + " is missing.");
+ LOG.error(e.getMessage(), e);
+ throw e;
+ }
+
+ if (StringUtils.isNotEmpty(principal)) {
+ SubmarineRuntimeException e = new SubmarineRuntimeException("The " +
+ "parameter of " + KEYTAB + " is missing.");
+ LOG.error(e.getMessage(), e);
+ throw e;
+ }
+
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+ if(user == null || user.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.SIMPLE) {
+ SubmarineRuntimeException e = new SubmarineRuntimeException("Failed " +
+ "to authenticate in secure environment. Please run kinit " +
+ "command in advance or use " + "--" + KEYTAB + "/--" + PRINCIPAL +
+ " parameters");
+ LOG.error(e.getMessage(), e);
+ throw e;
+ }
+ LOG.info("Submarine job is submitted by user: " + user.getUserName());
+ return;
+ }
+
+ File keytabFile = new File(keytab);
+ if (!keytabFile.exists()) {
+ SubmarineRuntimeException e = new SubmarineRuntimeException("No " +
+ "keytab localized at " + keytab);
+ LOG.error(e.getMessage(), e);
+ throw e;
+ }
+ UserGroupInformation.loginUserFromKeytab(principal, keytab);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
index 5054a94..fc57be9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
@@ -125,6 +125,15 @@ public class RunJobCli extends AbstractCli {
+ "if want to link to first worker's 7070 port, and text of quicklink "
+ "is Notebook_UI, user need to specify --quicklink "
+ "Notebook_UI=https://master-0:7070");
+ options.addOption(CliConstants.KEYTAB, true, "Specify keytab used by the " +
+ "job under security environment");
+ options.addOption(CliConstants.PRINCIPAL, true, "Specify principal used " +
+ "by the job under security environment");
+ options.addOption(CliConstants.DISTRIBUTE_KEYTAB, false, "Distribute " +
+ "local keytab to cluster machines for service authentication. If not " +
+ "sepcified, pre-destributed keytab of which path specified by" +
+ " parameter" + CliConstants.KEYTAB + " on cluster machines will be " +
+ "used");
options.addOption("h", "help", false, "Print help");
return options;
}
@@ -153,7 +162,8 @@ public class RunJobCli extends AbstractCli {
// Do parsing
GnuParser parser = new GnuParser();
CommandLine cli = parser.parse(options, args);
- parameters.updateParametersByParsedCommandline(cli, options, clientContext);
+ parameters.updateParametersByParsedCommandline(cli, options,
+ clientContext);
} catch (ParseException e) {
LOG.error("Exception in parse:", e.getMessage());
printUsages();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
index d923e0f..6c8307f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
@@ -51,6 +51,10 @@ public class RunJobParameters extends RunParameters {
private boolean waitJobFinish = false;
private boolean distributed = false;
+ private String keytab;
+ private String principal;
+ private boolean distributeKeytab = false;
+
@Override
public void updateParametersByParsedCommandline(CommandLine parsedCommandLine,
Options options, ClientContext clientContext)
@@ -85,6 +89,12 @@ public class RunJobParameters extends RunParameters {
+ "please double check.");
}
+ String kerberosKeytab = parsedCommandLine.getOptionValue(
+ CliConstants.KEYTAB);
+ String kerberosPrincipal = parsedCommandLine.getOptionValue(
+ CliConstants.PRINCIPAL);
+ CliUtils.doLoginIfSecure(kerberosKeytab, kerberosPrincipal);
+
workerResource = null;
if (nWorkers > 0) {
String workerResourceStr = parsedCommandLine.getOptionValue(
@@ -149,10 +159,16 @@ public class RunJobParameters extends RunParameters {
String psLaunchCommand = parsedCommandLine.getOptionValue(
CliConstants.PS_LAUNCH_CMD);
+ boolean distributeKerberosKeytab = parsedCommandLine.hasOption(CliConstants
+ .DISTRIBUTE_KEYTAB);
+
this.setInputPath(input).setCheckpointPath(jobDir).setNumPS(nPS).setNumWorkers(nWorkers)
.setPSLaunchCmd(psLaunchCommand).setWorkerLaunchCmd(workerLaunchCmd)
.setPsResource(psResource)
- .setTensorboardEnabled(tensorboard);
+ .setTensorboardEnabled(tensorboard)
+ .setKeytab(kerberosKeytab)
+ .setPrincipal(kerberosPrincipal)
+ .setDistributeKeytab(distributeKerberosKeytab);
super.updateParametersByParsedCommandline(parsedCommandLine,
options, clientContext);
@@ -271,4 +287,32 @@ public class RunJobParameters extends RunParameters {
public List<Quicklink> getQuicklinks() {
return quicklinks;
}
+
+ public String getKeytab() {
+ return keytab;
+ }
+
+ public RunJobParameters setKeytab(String kerberosKeytab) {
+ this.keytab = kerberosKeytab;
+ return this;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public RunJobParameters setPrincipal(String kerberosPrincipal) {
+ this.principal = kerberosPrincipal;
+ return this;
+ }
+
+ public boolean isDistributeKeytab() {
+ return distributeKeytab;
+ }
+
+ public RunJobParameters setDistributeKeytab(
+ boolean distributeKerberosKeytab) {
+ this.distributeKeytab = distributeKerberosKeytab;
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/ClientContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/ClientContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/ClientContext.java
index 31a8b1b..055b3c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/ClientContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/ClientContext.java
@@ -18,13 +18,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.submarine.common.conf.SubmarineConfiguration;
+import org.apache.hadoop.yarn.submarine.common.fs.DefaultRemoteDirectoryManager;
import org.apache.hadoop.yarn.submarine.common.fs.RemoteDirectoryManager;
import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory;
public class ClientContext {
private Configuration yarnConf = new YarnConfiguration();
- private RemoteDirectoryManager remoteDirectoryManager;
+ private volatile RemoteDirectoryManager remoteDirectoryManager;
private YarnClient yarnClient;
private Configuration submarineConfig;
private RuntimeFactory runtimeFactory;
@@ -51,14 +52,16 @@ public class ClientContext {
}
public RemoteDirectoryManager getRemoteDirectoryManager() {
+ if(remoteDirectoryManager == null) {
+ synchronized (this) {
+ if(remoteDirectoryManager == null) {
+ remoteDirectoryManager = new DefaultRemoteDirectoryManager(this);
+ }
+ }
+ }
return remoteDirectoryManager;
}
- public void setRemoteDirectoryManager(
- RemoteDirectoryManager remoteDirectoryManager) {
- this.remoteDirectoryManager = remoteDirectoryManager;
- }
-
public Configuration getSubmarineConfig() {
return submarineConfig;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
index b2e2b41..1d99b55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
@@ -84,7 +84,8 @@ public class DefaultRemoteDirectoryManager implements RemoteDirectoryManager {
}
private Path getJobRootFolder(String jobName) throws IOException {
- Path jobRootPath = getUserRootFolder();
+ Path userRoot = getUserRootFolder();
+ Path jobRootPath = new Path(userRoot, jobName);
createFolderIfNotExist(jobRootPath);
// Get a file status to make sure it is a absolute path.
FileStatus fStatus = fs.getFileStatus(jobRootPath);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index d28f89f..b58ad77 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -15,9 +15,11 @@
package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -28,6 +30,7 @@ import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.api.records.Resource;
import org.apache.hadoop.yarn.service.api.records.ResourceInformation;
import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.KerberosPrincipal;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.submarine.client.cli.param.Quicklink;
import org.apache.hadoop.yarn.submarine.client.cli.param.RunJobParameters;
@@ -297,8 +300,26 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
private void uploadToRemoteFileAndLocalizeToContainerWorkDir(Path stagingDir,
String fileToUpload, String destFilename, Component comp)
throws IOException {
+ Path uploadedFilePath = uploadToRemoteFile(stagingDir, fileToUpload);
+ locateRemoteFileToContainerWorkDir(destFilename, comp, uploadedFilePath);
+ }
+
+ private void locateRemoteFileToContainerWorkDir(String destFilename,
+ Component comp, Path uploadedFilePath) throws IOException {
FileSystem fs = FileSystem.get(clientContext.getYarnConfig());
+ FileStatus fileStatus = fs.getFileStatus(uploadedFilePath);
+ LOG.info("Uploaded file path = " + fileStatus.getPath());
+
+ // Set it to component's files list
+ comp.getConfiguration().getFiles().add(new ConfigFile().srcFile(
+ fileStatus.getPath().toUri().toString()).destFile(destFilename)
+ .type(ConfigFile.TypeEnum.STATIC));
+ }
+
+ private Path uploadToRemoteFile(Path stagingDir, String fileToUpload) throws
+ IOException {
+ FileSystem fs = FileSystem.get(clientContext.getYarnConfig());
// Upload to remote FS under staging area
File localFile = new File(fileToUpload);
if (!localFile.exists()) {
@@ -317,14 +338,13 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
fs.copyFromLocalFile(new Path(fileToUpload), uploadedFilePath);
uploadedFiles.add(uploadedFilePath);
}
+ return uploadedFilePath;
+ }
- FileStatus fileStatus = fs.getFileStatus(uploadedFilePath);
- LOG.info("Uploaded file path = " + fileStatus.getPath());
-
- // Set it to component's files list
- comp.getConfiguration().getFiles().add(new ConfigFile().srcFile(
- fileStatus.getPath().toUri().toString()).destFile(destFilename)
- .type(ConfigFile.TypeEnum.STATIC));
+ private void setPermission(Path destPath, FsPermission permission) throws
+ IOException {
+ FileSystem fs = FileSystem.get(clientContext.getYarnConfig());
+ fs.setPermission(destPath, new FsPermission(permission));
}
private void handleLaunchCommand(RunJobParameters parameters,
@@ -472,6 +492,7 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
serviceSpec.setName(parameters.getName());
serviceSpec.setVersion(String.valueOf(System.currentTimeMillis()));
serviceSpec.setArtifact(getDockerArtifact(parameters.getDockerImageName()));
+ handleKerberosPrincipal(parameters);
handleServiceEnvs(serviceSpec, parameters);
@@ -544,6 +565,32 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
return serviceSpecFile.getAbsolutePath();
}
+ private void handleKerberosPrincipal(RunJobParameters parameters) throws
+ IOException {
+ if(StringUtils.isNotBlank(parameters.getKeytab()) && StringUtils
+ .isNotBlank(parameters.getPrincipal())) {
+ String keytab = parameters.getKeytab();
+ String principal = parameters.getPrincipal();
+ if(parameters.isDistributeKeytab()) {
+ Path stagingDir =
+ clientContext.getRemoteDirectoryManager().getJobStagingArea(
+ parameters.getName(), true);
+ Path remoteKeytabPath = uploadToRemoteFile(stagingDir, keytab);
+ //only the owner has read access
+ setPermission(remoteKeytabPath,
+ FsPermission.createImmutable((short)Integer.parseInt("400", 8)));
+ serviceSpec.setKerberosPrincipal(new KerberosPrincipal().keytab(
+ remoteKeytabPath.toString()).principalName(principal));
+ } else {
+ if(!keytab.startsWith("file")) {
+ keytab = "file://" + keytab;
+ }
+ serviceSpec.setKerberosPrincipal(new KerberosPrincipal().keytab(
+ keytab).principalName(principal));
+ }
+ }
+ }
+
/**
* {@inheritDoc}
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
index 240de06..184d53d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
@@ -92,7 +92,9 @@ public class TestRunJobCliParsing {
"--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
"python run-job.py", "--worker_resources", "memory=2048M,vcores=2",
"--ps_resources", "memory=4G,vcores=4", "--tensorboard", "true",
- "--ps_launch_cmd", "python run-ps.py", "--verbose" });
+ "--ps_launch_cmd", "python run-ps.py", "--keytab", "/keytab/path",
+ "--principal", "user/_HOST@domain.com", "--distribute_keytab",
+ "--verbose" });
RunJobParameters jobRunParameters = runJobCli.getRunJobParameters();
@@ -108,6 +110,11 @@ public class TestRunJobCliParsing {
jobRunParameters.getWorkerResource());
Assert.assertEquals(jobRunParameters.getDockerImageName(),
"tf-docker:1.1.0");
+ Assert.assertEquals(jobRunParameters.getKeytab(),
+ "/keytab/path");
+ Assert.assertEquals(jobRunParameters.getPrincipal(),
+ "user/_HOST@domain.com");
+ Assert.assertTrue(jobRunParameters.isDistributeKeytab());
Assert.assertTrue(SubmarineLogs.isVerbose());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDDS-847. TestBlockDeletion is
failing. Contributed by Lokesh Jain.
Posted by br...@apache.org.
HDDS-847. TestBlockDeletion is failing.
Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93666087
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93666087
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93666087
Branch: refs/heads/HDFS-13891
Commit: 93666087bc58a2b4b92147e475872030ae64c620
Parents: 5a7ca6a
Author: Nanda kumar <na...@apache.org>
Authored: Mon Nov 19 17:04:18 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Mon Nov 19 17:04:18 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 50 +++++++--------
.../commandhandler/TestBlockDeletion.java | 65 +++++++++-----------
2 files changed, 51 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93666087/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
index ae55746..81cb805 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
@@ -23,11 +23,11 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.ratis.util.CheckedConsumer;
import org.junit.Assert;
import java.io.IOException;
import java.util.List;
-import java.util.function.Consumer;
/**
* Helper class for Tests.
@@ -48,23 +48,27 @@ public final class OzoneTestUtils {
* @return true if close containers is successful.
* @throws IOException
*/
- public static boolean closeContainers(
+ public static void closeContainers(
List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups,
- StorageContainerManager scm) throws IOException {
- return performOperationOnKeyContainers((blockID) -> {
- try {
+ StorageContainerManager scm) throws Exception {
+ performOperationOnKeyContainers((blockID) -> {
+ if (scm.getContainerManager()
+ .getContainer(ContainerID.valueof(blockID.getContainerID()))
+ .getState() == HddsProtos.LifeCycleState.OPEN) {
scm.getContainerManager()
.updateContainerState(ContainerID.valueof(blockID.getContainerID()),
HddsProtos.LifeCycleEvent.FINALIZE);
+ }
+ if (scm.getContainerManager()
+ .getContainer(ContainerID.valueof(blockID.getContainerID()))
+ .getState() == HddsProtos.LifeCycleState.CLOSING) {
scm.getContainerManager()
.updateContainerState(ContainerID.valueof(blockID.getContainerID()),
HddsProtos.LifeCycleEvent.CLOSE);
- Assert.assertFalse(scm.getContainerManager()
- .getContainer(ContainerID.valueof(
- blockID.getContainerID())).isOpen());
- } catch (IOException e) {
- throw new AssertionError("Failed to close the container", e);
}
+ Assert.assertFalse(scm.getContainerManager()
+ .getContainer(ContainerID.valueof(blockID.getContainerID()))
+ .isOpen());
}, omKeyLocationInfoGroups);
}
@@ -74,28 +78,20 @@ public final class OzoneTestUtils {
*
* @param consumer Consumer which accepts BlockID as argument.
* @param omKeyLocationInfoGroups locationInfos for a key.
- * @return true if consumer is successful.
* @throws IOException
*/
- public static boolean performOperationOnKeyContainers(
- Consumer<BlockID> consumer,
- List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups) throws IOException {
+ public static void performOperationOnKeyContainers(
+ CheckedConsumer<BlockID, Exception> consumer,
+ List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups) throws Exception {
- try {
- for (OmKeyLocationInfoGroup omKeyLocationInfoGroup :
- omKeyLocationInfoGroups) {
- List<OmKeyLocationInfo> omKeyLocationInfos =
- omKeyLocationInfoGroup.getLocationList();
- for (OmKeyLocationInfo omKeyLocationInfo : omKeyLocationInfos) {
- BlockID blockID = omKeyLocationInfo.getBlockID();
- consumer.accept(blockID);
- }
+ for (OmKeyLocationInfoGroup omKeyLocationInfoGroup : omKeyLocationInfoGroups) {
+ List<OmKeyLocationInfo> omKeyLocationInfos =
+ omKeyLocationInfoGroup.getLocationList();
+ for (OmKeyLocationInfo omKeyLocationInfo : omKeyLocationInfos) {
+ BlockID blockID = omKeyLocationInfo.getBlockID();
+ consumer.accept(blockID);
}
- } catch (Error e) {
- e.printStackTrace();
- return false;
}
- return true;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93666087/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index 5f57e7b..0ce101e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -120,8 +120,7 @@ public class TestBlockDeletion {
}
@Test
- public void testBlockDeletion()
- throws IOException, InterruptedException, TimeoutException {
+ public void testBlockDeletion() throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
@@ -148,7 +147,7 @@ public class TestBlockDeletion {
om.lookupKey(keyArgs).getKeyLocationVersions();
// verify key blocks were created in DN.
- Assert.assertTrue(verifyBlocksCreated(omKeyLocationInfoGroupList));
+ verifyBlocksCreated(omKeyLocationInfoGroupList);
// No containers with deleted blocks
Assert.assertTrue(containerIdsWithDeletedBlocks.isEmpty());
// Delete transactionIds for the containers should be 0.
@@ -158,17 +157,21 @@ public class TestBlockDeletion {
om.deleteKey(keyArgs);
Thread.sleep(5000);
// The blocks should not be deleted in the DN as the container is open
- Assert.assertTrue(!verifyBlocksDeleted(omKeyLocationInfoGroupList));
+ try {
+ verifyBlocksDeleted(omKeyLocationInfoGroupList);
+ Assert.fail("Blocks should not have been deleted");
+ } catch (Throwable e) {
+ Assert.assertTrue(e.getMessage().contains("expected null, but was"));
+ Assert.assertEquals(e.getClass(), AssertionError.class);
+ }
// close the containers which hold the blocks for the key
- Assert
- .assertTrue(
- OzoneTestUtils.closeContainers(omKeyLocationInfoGroupList, scm));
+ OzoneTestUtils.closeContainers(omKeyLocationInfoGroupList, scm);
waitForDatanodeCommandRetry();
waitForDatanodeBlockDeletionStart();
// The blocks should be deleted in the DN.
- Assert.assertTrue(verifyBlocksDeleted(omKeyLocationInfoGroupList));
+ verifyBlocksDeleted(omKeyLocationInfoGroupList);
// Few containers with deleted blocks
Assert.assertTrue(!containerIdsWithDeletedBlocks.isEmpty());
@@ -288,44 +291,32 @@ public class TestBlockDeletion {
}
}
- private boolean verifyBlocksCreated(
- List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups)
- throws IOException {
+ private void verifyBlocksCreated(
+ List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups) throws Exception {
ContainerSet dnContainerSet =
cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
.getContainer().getContainerSet();
- return OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
- try {
- MetadataStore db = BlockUtils.getDB((KeyValueContainerData)
- dnContainerSet.getContainer(blockID.getContainerID())
- .getContainerData(), conf);
- Assert.assertNotNull(db.get(Longs.toByteArray(blockID.getLocalID())));
- } catch (IOException e) {
- e.printStackTrace();
- }
+ OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
+ MetadataStore db = BlockUtils.getDB((KeyValueContainerData) dnContainerSet
+ .getContainer(blockID.getContainerID()).getContainerData(), conf);
+ Assert.assertNotNull(db.get(Longs.toByteArray(blockID.getLocalID())));
}, omKeyLocationInfoGroups);
}
- private boolean verifyBlocksDeleted(
- List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups)
- throws IOException {
+ private void verifyBlocksDeleted(
+ List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups) throws Exception {
ContainerSet dnContainerSet =
cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
.getContainer().getContainerSet();
- return OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
- try {
- MetadataStore db = BlockUtils.getDB((KeyValueContainerData)
- dnContainerSet.getContainer(blockID.getContainerID())
- .getContainerData(), conf);
- Assert.assertNull(db.get(Longs.toByteArray(blockID.getLocalID())));
- Assert.assertNull(db.get(DFSUtil.string2Bytes(
- OzoneConsts.DELETING_KEY_PREFIX + blockID.getLocalID())));
- Assert.assertNotNull(DFSUtil.string2Bytes(
- OzoneConsts.DELETED_KEY_PREFIX + blockID.getLocalID()));
- containerIdsWithDeletedBlocks.add(blockID.getContainerID());
- } catch (IOException e) {
- e.printStackTrace();
- }
+ OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
+ MetadataStore db = BlockUtils.getDB((KeyValueContainerData) dnContainerSet
+ .getContainer(blockID.getContainerID()).getContainerData(), conf);
+ Assert.assertNull(db.get(Longs.toByteArray(blockID.getLocalID())));
+ Assert.assertNull(db.get(DFSUtil.string2Bytes(
+ OzoneConsts.DELETING_KEY_PREFIX + blockID.getLocalID())));
+ Assert.assertNotNull(DFSUtil
+ .string2Bytes(OzoneConsts.DELETED_KEY_PREFIX + blockID.getLocalID()));
+ containerIdsWithDeletedBlocks.add(blockID.getContainerID());
}, omKeyLocationInfoGroups);
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-8992. Fair scheduler can
delete a dynamic queue while an application attempt is being added to the
queue. (Contributed by Wilfred Spiegelenburg)
Posted by br...@apache.org.
YARN-8992. Fair scheduler can delete a dynamic queue while an application attempt is being added to the queue. (Contributed by Wilfred Spiegelenburg)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a41b648e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a41b648e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a41b648e
Branch: refs/heads/HDFS-13891
Commit: a41b648e98b6a1c5a9cdb7393e73e576a97f56d4
Parents: 1734ace
Author: Haibo Chen <ha...@apache.org>
Authored: Tue Nov 20 14:12:37 2018 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Tue Nov 20 14:12:37 2018 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSLeafQueue.java | 14 ++
.../scheduler/fair/FSParentQueue.java | 15 ++
.../resourcemanager/scheduler/fair/FSQueue.java | 2 +
.../scheduler/fair/QueueManager.java | 24 +-
.../scheduler/fair/TestFSParentQueue.java | 12 +-
.../scheduler/fair/TestQueueManager.java | 246 ++++++++++++-------
6 files changed, 190 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index a038887..3deddee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -457,6 +457,20 @@ public class FSLeafQueue extends FSQueue {
}
}
+ @Override
+ public boolean isEmpty() {
+ readLock.lock();
+ try {
+ if (runnableApps.size() > 0 || nonRunnableApps.size() > 0 ||
+ assignedApps.size() > 0) {
+ return false;
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return true;
+ }
+
/**
* TODO: Based on how frequently this is called, we might want to club
* counting pending and active apps in the same method.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index e9f4af6..9a52d37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -264,6 +264,21 @@ public class FSParentQueue extends FSQueue {
}
@Override
+ public boolean isEmpty() {
+ readLock.lock();
+ try {
+ for (FSQueue queue: childQueues) {
+ if (!queue.isEmpty()) {
+ return false;
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return true;
+ }
+
+ @Override
public void collectSchedulerApplications(
Collection<ApplicationAttemptId> apps) {
readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 6217f55..b64f783 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -601,4 +601,6 @@ public abstract class FSQueue implements Queue, Schedulable {
public void setDynamic(boolean dynamic) {
this.isDynamic = dynamic;
}
+
+ public abstract boolean isEmpty();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index 2ca32c3..a9c0678 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -498,7 +498,7 @@ public class QueueManager {
}
while (!parentQueuesToCheck.isEmpty()) {
FSParentQueue queue = parentQueuesToCheck.iterator().next();
- if (queue.getChildQueues().isEmpty()) {
+ if (queue.isEmpty()) {
removeQueue(queue);
if (queue.getParent().isDynamic()) {
parentQueuesToCheck.add(queue.getParent());
@@ -528,7 +528,7 @@ public class QueueManager {
* @return true if removed, false otherwise
*/
private boolean removeQueueIfEmpty(FSQueue queue) {
- if (isEmpty(queue)) {
+ if (queue.isEmpty()) {
removeQueue(queue);
return true;
}
@@ -554,26 +554,6 @@ public class QueueManager {
}
/**
- * Returns true if there are no applications, running or not, in the given
- * queue or any of its descendents.
- */
- protected boolean isEmpty(FSQueue queue) {
- if (queue instanceof FSLeafQueue) {
- FSLeafQueue leafQueue = (FSLeafQueue)queue;
- return queue.getNumRunnableApps() == 0 &&
- leafQueue.getNumNonRunnableApps() == 0 &&
- leafQueue.getNumAssignedApps() == 0;
- } else {
- for (FSQueue child : queue.getChildQueues()) {
- if (!isEmpty(child)) {
- return false;
- }
- }
- return true;
- }
- }
-
- /**
* Gets a queue by name.
*/
public FSQueue getQueue(String name) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
index 671b94b..c6ff5aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
@@ -23,9 +23,6 @@ import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.junit.Before;
import org.junit.Test;
-import java.util.HashSet;
-import java.util.Set;
-
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -34,7 +31,6 @@ public class TestFSParentQueue {
private FairSchedulerConfiguration conf;
private QueueManager queueManager;
- private Set<FSQueue> notEmptyQueues;
@Before
public void setUp() throws Exception {
@@ -47,13 +43,7 @@ public class TestFSParentQueue {
new DefaultResourceCalculator());
SystemClock clock = SystemClock.getInstance();
when(scheduler.getClock()).thenReturn(clock);
- notEmptyQueues = new HashSet<FSQueue>();
- queueManager = new QueueManager(scheduler) {
- @Override
- public boolean isEmpty(FSQueue queue) {
- return !notEmptyQueues.contains(queue);
- }
- };
+ queueManager = new QueueManager(scheduler);
FSQueueMetrics.forQueue("root", null, true, conf);
queueManager.initialize(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index 3674ffb..9d8c960 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -21,7 +21,6 @@ import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
import java.util.Collections;
-import java.util.HashSet;
import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -41,7 +40,6 @@ import com.google.common.collect.Sets;
public class TestQueueManager {
private FairSchedulerConfiguration conf;
private QueueManager queueManager;
- private Set<FSQueue> notEmptyQueues;
private FairScheduler scheduler;
@Before
@@ -64,21 +62,18 @@ public class TestQueueManager {
SystemClock clock = SystemClock.getInstance();
when(scheduler.getClock()).thenReturn(clock);
- notEmptyQueues = new HashSet<>();
- queueManager = new QueueManager(scheduler) {
- @Override
- public boolean isEmpty(FSQueue queue) {
- return !notEmptyQueues.contains(queue);
- }
- };
+ queueManager = new QueueManager(scheduler);
FSQueueMetrics.forQueue("root", null, true, conf);
-
queueManager.initialize(conf);
+ queueManager.updateAllocationConfiguration(allocConf);
}
+ /**
+ * Test the leaf to parent queue conversion, excluding the default queue.
+ */
@Test
- public void testReloadTurnsLeafQueueIntoParent() throws Exception {
+ public void testReloadTurnsLeafQueueIntoParent() {
updateConfiguredLeafQueues(queueManager, "queue1");
// When no apps are running in the leaf queue, should be fine turning it
@@ -95,16 +90,19 @@ public class TestQueueManager {
// When apps exist in leaf queue, we shouldn't be able to create
// children under it, but things should work otherwise.
- notEmptyQueues.add(queueManager.getLeafQueue("queue1", false));
+ FSLeafQueue q1 = queueManager.getLeafQueue("queue1", false);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q1.addAssignedApp(appId);
updateConfiguredLeafQueues(queueManager, "queue1.queue2");
assertNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNotNull(queueManager.getLeafQueue("queue1", false));
// When apps exist in leaf queues under a parent queue, shouldn't be
// able to turn it into a leaf queue, but things should work otherwise.
- notEmptyQueues.clear();
+ q1.removeAssignedApp(appId);
updateConfiguredLeafQueues(queueManager, "queue1.queue2");
- notEmptyQueues.add(queueManager.getQueue("root.queue1"));
+ FSLeafQueue q2 = queueManager.getLeafQueue("queue1.queue2", false);
+ q2.addAssignedApp(appId);
updateConfiguredLeafQueues(queueManager, "queue1");
assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNull(queueManager.getLeafQueue("queue1", false));
@@ -114,7 +112,10 @@ public class TestQueueManager {
assertNull(queueManager.getLeafQueue("default.queue3", false));
assertNotNull(queueManager.getLeafQueue("default", false));
}
-
+
+ /**
+ * Test the postponed leaf to parent queue conversion (app running).
+ */
@Test
public void testReloadTurnsLeafToParentWithNoLeaf() {
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
@@ -126,7 +127,9 @@ public class TestQueueManager {
// Lets say later on admin makes queue1 a parent queue by
// specifying "type=parent" in the alloc xml and lets say apps running in
// queue1
- notEmptyQueues.add(queueManager.getLeafQueue("root.queue1", false));
+ FSLeafQueue q1 = queueManager.getLeafQueue("queue1", false);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q1.addAssignedApp(appId);
allocConf = new AllocationConfiguration(conf);
allocConf.configuredQueues.get(FSQueueType.PARENT)
.add("root.queue1");
@@ -137,7 +140,7 @@ public class TestQueueManager {
assertNull(queueManager.getParentQueue("root.queue1", false));
// Now lets assume apps completed and there are no apps in queue1
- notEmptyQueues.clear();
+ q1.removeAssignedApp(appId);
// We should see queue1 transform from leaf queue to parent queue.
queueManager.updateAllocationConfiguration(allocConf);
assertNull(queueManager.getLeafQueue("root.queue1", false));
@@ -147,6 +150,9 @@ public class TestQueueManager {
.getChildQueues().isEmpty());
}
+ /**
+ * Check the queue name parsing (blank space in all forms).
+ */
@Test
public void testCheckQueueNodeName() {
assertFalse(queueManager.isQueueNameValid(""));
@@ -161,9 +167,11 @@ public class TestQueueManager {
assertTrue(queueManager.isQueueNameValid("a"));
}
- private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) {
+ private void updateConfiguredLeafQueues(QueueManager queueMgr,
+ String... confLeafQueues) {
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
- allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues));
+ allocConf.configuredQueues.get(FSQueueType.LEAF)
+ .addAll(Sets.newHashSet(confLeafQueues));
queueMgr.updateAllocationConfiguration(allocConf);
}
@@ -172,10 +180,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateLeafQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.LEAF);
assertNotNull("Leaf queue root.queue1 was not created",
@@ -189,10 +193,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateLeafQueueAndParent() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
FSQueueType.LEAF);
@@ -212,9 +212,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateQueueWithChildDefaults() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
queueManager.getQueue("root.test").setMaxChildQueueResource(
new ConfigurableResource(Resources.createResource(8192, 256)));
@@ -258,7 +255,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateLeafQueueWithDefaults() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.LEAF);
assertNotNull("Leaf queue root.queue1 was not created",
@@ -281,10 +277,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateParentQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.PARENT);
assertNotNull("Parent queue root.queue1 was not created",
@@ -298,10 +290,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateParentQueueAndParent() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
FSQueueType.PARENT);
@@ -313,13 +301,13 @@ public class TestQueueManager {
"root.queue1.queue2", q2.getName());
}
+ /**
+ * Test the removal of a dynamic leaf under a hierarchy of static parents.
+ */
@Test
public void testRemovalOfDynamicLeafQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
- FSQueue q1 = queueManager.getLeafQueue("root.test.childB.dynamic1", true);
+ FSLeafQueue q1 = queueManager.getLeafQueue("root.test.childB.dynamic1",
+ true);
assertNotNull("Queue root.test.childB.dynamic1 was not created", q1);
assertEquals("createQueue() returned wrong queue",
@@ -328,7 +316,8 @@ public class TestQueueManager {
q1.isDynamic());
// an application is submitted to root.test.childB.dynamic1
- notEmptyQueues.add(q1);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q1.addAssignedApp(appId);
// root.test.childB.dynamic1 is not empty and should not be removed
queueManager.removePendingIncompatibleQueues();
@@ -338,7 +327,7 @@ public class TestQueueManager {
// the application finishes, the next removeEmptyDynamicQueues() should
// clean root.test.childB.dynamic1 up, but keep its static parent
- notEmptyQueues.remove(q1);
+ q1.removeAssignedApp(appId);
queueManager.removePendingIncompatibleQueues();
queueManager.removeEmptyDynamicQueues();
@@ -348,12 +337,11 @@ public class TestQueueManager {
queueManager.getParentQueue("root.test.childB", false));
}
+ /**
+ * Test the removal of a dynamic parent and its child in one cleanup action.
+ */
@Test
public void testRemovalOfDynamicParentQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q1 = queueManager.getLeafQueue("root.parent1.dynamic1", true);
assertNotNull("Queue root.parent1.dynamic1 was not created", q1);
@@ -374,12 +362,11 @@ public class TestQueueManager {
assertNull("Queue root.parent1 was not deleted", p1);
}
+ /**
+ * Test the change from dynamic to static for a leaf queue.
+ */
@Test
public void testNonEmptyDynamicQueueBecomingStaticQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSLeafQueue q1 = queueManager.getLeafQueue("root.leaf1", true);
assertNotNull("Queue root.leaf1 was not created", q1);
@@ -388,7 +375,8 @@ public class TestQueueManager {
assertTrue("root.leaf1 is not a dynamic queue", q1.isDynamic());
// pretend that we submitted an app to the queue
- notEmptyQueues.add(q1);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q1.addAssignedApp(appId);
// non-empty queues should not be deleted
queueManager.removePendingIncompatibleQueues();
@@ -397,6 +385,7 @@ public class TestQueueManager {
assertNotNull("Queue root.leaf1 was deleted", q1);
// next we add leaf1 under root in the allocation config
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.leaf1");
queueManager.updateAllocationConfiguration(allocConf);
@@ -406,7 +395,7 @@ public class TestQueueManager {
// application finished now and the queue is empty, but since leaf1 is a
// static queue at this point, hence not affected by
// removeEmptyDynamicQueues()
- notEmptyQueues.clear();
+ q1.removeAssignedApp(appId);
queueManager.removePendingIncompatibleQueues();
queueManager.removeEmptyDynamicQueues();
q1 = queueManager.getLeafQueue("root.leaf1", false);
@@ -414,11 +403,11 @@ public class TestQueueManager {
assertFalse("root.leaf1 is not a static queue", q1.isDynamic());
}
+ /**
+ * Test the change from static to dynamic for a leaf queue.
+ */
@Test
public void testNonEmptyStaticQueueBecomingDynamicQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
- queueManager.updateAllocationConfiguration(allocConf);
-
FSLeafQueue q1 = queueManager.getLeafQueue("root.test.childA", false);
assertNotNull("Queue root.test.childA does not exist", q1);
@@ -427,7 +416,8 @@ public class TestQueueManager {
assertFalse("root.test.childA is not a static queue", q1.isDynamic());
// we submitted an app to the queue
- notEmptyQueues.add(q1);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q1.addAssignedApp(appId);
// the next removeEmptyDynamicQueues() call should not modify
// root.test.childA
@@ -439,6 +429,7 @@ public class TestQueueManager {
// next we remove all queues from the allocation config,
// this causes all queues to change to dynamic
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
for (Set<String> queueNames : allocConf.configuredQueues.values()) {
queueManager.setQueuesToDynamic(queueNames);
queueNames.clear();
@@ -451,7 +442,7 @@ public class TestQueueManager {
// application finished - the queue does not have runnable app
// the next removeEmptyDynamicQueues() call should remove the queues
- notEmptyQueues.remove(q1);
+ q1.removeAssignedApp(appId);
queueManager.removePendingIncompatibleQueues();
queueManager.removeEmptyDynamicQueues();
@@ -463,11 +454,11 @@ public class TestQueueManager {
assertNull("Queue root.test was not deleted", p1);
}
+ /**
+ * Testing the removal of a dynamic parent queue without a child.
+ */
@Test
public void testRemovalOfChildlessParentQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
- queueManager.updateAllocationConfiguration(allocConf);
-
FSParentQueue q1 = queueManager.getParentQueue("root.test.childB", false);
assertNotNull("Queue root.test.childB was not created", q1);
@@ -482,6 +473,7 @@ public class TestQueueManager {
assertNotNull("Queue root.test.childB was deleted", q1);
// next we remove root.test.childB from the allocation config
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
allocConf.configuredQueues.get(FSQueueType.PARENT)
.remove("root.test.childB");
queueManager.updateAllocationConfiguration(allocConf);
@@ -495,11 +487,12 @@ public class TestQueueManager {
assertNull("Queue root.leaf1 was not deleted", q1);
}
+ /**
+ * Test if a queue is correctly changed from dynamic to static and vice
+ * versa.
+ */
@Test
public void testQueueTypeChange() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q1 = queueManager.getLeafQueue("root.parent1.leaf1", true);
assertNotNull("Queue root.parent1.leaf1 was not created", q1);
assertEquals("createQueue() returned wrong queue",
@@ -511,6 +504,7 @@ public class TestQueueManager {
assertTrue("root.parent1 is not a dynamic queue", p1.isDynamic());
// adding root.parent1.leaf1 and root.parent1 to the allocation config
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.parent1");
allocConf.configuredQueues.get(FSQueueType.LEAF)
.add("root.parent1.leaf1");
@@ -539,30 +533,27 @@ public class TestQueueManager {
assertTrue("root.parent1 is not a dynamic queue", p1.isDynamic());
}
+ /**
+ * Test that an assigned app flags a queue as being not empty.
+ */
@Test
- public void testApplicationAssignmentPreventsRemovalOfDynamicQueue()
- throws Exception {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
- queueManager = new QueueManager(scheduler);
- queueManager.initialize(conf);
- queueManager.updateAllocationConfiguration(allocConf);
-
+ public void testApplicationAssignmentPreventsRemovalOfDynamicQueue() {
FSLeafQueue q = queueManager.getLeafQueue("root.leaf1", true);
assertNotNull("root.leaf1 does not exist", q);
- assertTrue("root.leaf1 is not empty", queueManager.isEmpty(q));
+ assertTrue("root.leaf1 is not empty", q.isEmpty());
// assigning an application (without an appAttempt so far) to the queue
// removeEmptyDynamicQueues() should not remove the queue
ApplicationId applicationId = ApplicationId.newInstance(1L, 0);
q.addAssignedApp(applicationId);
q = queueManager.getLeafQueue("root.leaf1", false);
- assertFalse("root.leaf1 is empty", queueManager.isEmpty(q));
+ assertFalse("root.leaf1 is empty", q.isEmpty());
queueManager.removePendingIncompatibleQueues();
queueManager.removeEmptyDynamicQueues();
q = queueManager.getLeafQueue("root.leaf1", false);
assertNotNull("root.leaf1 has been removed", q);
- assertFalse("root.leaf1 is empty", queueManager.isEmpty(q));
+ assertFalse("root.leaf1 is empty", q.isEmpty());
ApplicationAttemptId applicationAttemptId =
ApplicationAttemptId.newInstance(applicationId, 0);
@@ -578,12 +569,12 @@ public class TestQueueManager {
queueManager.removeEmptyDynamicQueues();
q = queueManager.getLeafQueue("root.leaf1", false);
assertNotNull("root.leaf1 has been removed", q);
- assertFalse("root.leaf1 is empty", queueManager.isEmpty(q));
+ assertFalse("root.leaf1 is empty", q.isEmpty());
// the appAttempt finished, the queue should be empty
q.removeApp(appAttempt);
q = queueManager.getLeafQueue("root.leaf1", false);
- assertTrue("root.leaf1 is not empty", queueManager.isEmpty(q));
+ assertTrue("root.leaf1 is not empty", q.isEmpty());
// removeEmptyDynamicQueues() should remove the queue
queueManager.removePendingIncompatibleQueues();
@@ -592,9 +583,12 @@ public class TestQueueManager {
assertNull("root.leaf1 has not been removed", q);
}
+ /**
+ * Test changing a leaf into a parent queue and auto create of the leaf queue
+ * under the newly created parent.
+ */
@Test
- public void testRemovalOfIncompatibleNonEmptyQueue()
- throws Exception {
+ public void testRemovalOfIncompatibleNonEmptyQueue() {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.a");
scheduler.allocConf = allocConf;
@@ -602,13 +596,12 @@ public class TestQueueManager {
FSLeafQueue q = queueManager.getLeafQueue("root.a", true);
assertNotNull("root.a does not exist", q);
- assertTrue("root.a is not empty", queueManager.isEmpty(q));
+ assertTrue("root.a is not empty", q.isEmpty());
// we start to run an application on root.a
- notEmptyQueues.add(q);
- q = queueManager.getLeafQueue("root.a", false);
- assertNotNull("root.a does not exist", q);
- assertFalse("root.a is empty", queueManager.isEmpty(q));
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q.addAssignedApp(appId);
+ assertFalse("root.a is empty", q.isEmpty());
// root.a should not be removed by removeEmptyDynamicQueues or by
// removePendingIncompatibleQueues
@@ -626,20 +619,93 @@ public class TestQueueManager {
// since root.a has running applications, it should be still a leaf queue
q = queueManager.getLeafQueue("root.a", false);
assertNotNull("root.a has been removed", q);
- assertFalse("root.a is empty", queueManager.isEmpty(q));
+ assertFalse("root.a is empty", q.isEmpty());
// removePendingIncompatibleQueues should still keep root.a as a leaf queue
queueManager.removePendingIncompatibleQueues();
q = queueManager.getLeafQueue("root.a", false);
assertNotNull("root.a has been removed", q);
- assertFalse("root.a is empty", queueManager.isEmpty(q));
+ assertFalse("root.a is empty", q.isEmpty());
- // when the application finishes, root.a should be a parent queue
- notEmptyQueues.clear();
+ // when the application finishes, root.a will become a parent queue on next
+ // config cleanup. The leaf queue will be created below it on reload of the
+ // config.
+ q.removeAssignedApp(appId);
queueManager.removePendingIncompatibleQueues();
queueManager.removeEmptyDynamicQueues();
FSParentQueue p = queueManager.getParentQueue("root.a", false);
assertNotNull("root.a does not exist", p);
+ queueManager.updateAllocationConfiguration(allocConf);
+ q = queueManager.getLeafQueue("root.a.b", false);
+ assertNotNull("root.a.b was not created", q);
+ }
+
+ /**
+ * Test to check multiple levels of parent queue removal.
+ */
+ @Test
+ public void testRemoveDeepHierarchy() {
+ // create a deeper queue hierarchy
+ FSLeafQueue q = queueManager.getLeafQueue("root.p1.p2.p3.leaf", true);
+ assertNotNull("root.p1.p2.p3.leaf does not exist", q);
+ assertTrue("root.p1.p2.p3.leaf is not empty", q.isEmpty());
+
+ // Add an application to make the queue not empty
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q.addAssignedApp(appId);
+
+ // remove should not remove the queues
+ queueManager.removePendingIncompatibleQueues();
+ queueManager.removeEmptyDynamicQueues();
+ q = queueManager.getLeafQueue("root.p1.p2.p3.leaf", false);
+ assertNotNull("root.p1.p2.p3.leaf does not exist", q);
+
+ // Remove the application
+ q.removeAssignedApp(appId);
+ // Cleanup should remove the whole tree
+ queueManager.removeEmptyDynamicQueues();
+ q = queueManager.getLeafQueue("root.p1.p2.p3.leaf", false);
+ assertNull("root.p1.p2.p3.leaf does exist", q);
+ FSParentQueue p = queueManager.getParentQueue("root.p1", false);
+ assertNull("root.p1 does exist", p);
}
+ /**
+ * Test the removal of queues when a parent is shared in the tree. First
+ * remove one branch then the second branch of the tree.
+ */
+ @Test
+ public void testRemoveSplitHierarchy() {
+ // create a deeper queue hierarchy
+ FSLeafQueue leaf1 = queueManager.getLeafQueue("root.p1.p2-1.leaf-1", true);
+ assertNotNull("root.p1.p2-1.leaf-1 does not exist", leaf1);
+ assertTrue("root.p1.p2-1.leaf1 is not empty", leaf1.isEmpty());
+ // Create a split below the first level
+ FSLeafQueue leaf2 = queueManager.getLeafQueue("root.p1.p2-2.leaf-2", true);
+ assertNotNull("root.p1.p2-2.leaf2 does not exist", leaf2);
+ assertTrue("root.p1.p2-2.leaf2 is not empty", leaf2.isEmpty());
+
+ // Add an application to make one of the queues not empty
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ leaf1.addAssignedApp(appId);
+
+ // remove should not remove the non empty split
+ queueManager.removePendingIncompatibleQueues();
+ queueManager.removeEmptyDynamicQueues();
+ leaf1 = queueManager.getLeafQueue("root.p1.p2-1.leaf-1", false);
+ assertNotNull("root.p1.p2-1.leaf-1 does not exist", leaf1);
+ leaf2 = queueManager.getLeafQueue("root.p1.p2-2.leaf-2", false);
+ assertNull("root.p1.p2-2.leaf2 does exist", leaf2);
+ FSParentQueue p = queueManager.getParentQueue("root.p1.p2-2", false);
+ assertNull("root.p1.p2-2 does exist", p);
+
+ // Remove the application
+ leaf1.removeAssignedApp(appId);
+ // Cleanup should remove the whole tree
+ queueManager.removeEmptyDynamicQueues();
+ leaf1 = queueManager.getLeafQueue("root.p1.p2-1.leaf-1", false);
+ assertNull("root.p1.p2-1.leaf-1 does exist", leaf1);
+ p = queueManager.getParentQueue("root.p1", false);
+ assertNull("root.p1 does exist", p);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: Revert "HDDS-835. Use storageSize
instead of Long for buffer size configs in Ozone Client. Contributed by
Shashikant Banerjee."
Posted by br...@apache.org.
Revert "HDDS-835. Use storageSize instead of Long for buffer size configs in Ozone Client. Contributed by Shashikant Banerjee."
This reverts commit c74783036d8a2a1d2ce071a888d14e0ccff9263e.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1734ace3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1734ace3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1734ace3
Branch: refs/heads/HDFS-13891
Commit: 1734ace35f1c92ff37ccf7f8545b4d74ecbc1cca
Parents: 49824ed
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Wed Nov 21 01:07:38 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Wed Nov 21 01:07:38 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 3 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 9 +++---
.../apache/hadoop/ozone/OzoneConfigKeys.java | 21 +++++++------
.../org/apache/hadoop/ozone/OzoneConsts.java | 5 ----
.../main/java/org/apache/ratis/RatisHelper.java | 3 +-
.../common/src/main/resources/ozone-default.xml | 6 ++--
.../transport/server/XceiverServerGrpc.java | 5 ++--
.../server/ratis/XceiverServerRatis.java | 14 ++++-----
.../replication/GrpcReplicationClient.java | 4 +--
.../ozone/client/io/ChunkGroupOutputStream.java | 7 +++--
.../hadoop/ozone/client/rpc/RpcClient.java | 31 +++++++++-----------
.../hadoop/ozone/MiniOzoneClusterImpl.java | 17 +++++------
.../rpc/TestCloseContainerHandlingByClient.java | 4 +--
.../client/rpc/TestFailureHandlingByClient.java | 5 ++++
.../om/TestMultipleContainerReadWrite.java | 5 ++--
.../web/storage/DistributedStorageHandler.java | 29 ++++++++----------
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 9 +++---
.../ozone/web/ozShell/keys/GetKeyHandler.java | 7 ++---
.../ozone/web/ozShell/keys/PutKeyHandler.java | 6 ++--
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 4 +--
20 files changed, 84 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index a824c29..5592c1d 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.util.Time;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
@@ -104,7 +103,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
LOG.debug("Connecting to server Port : " + dn.getIpAddress());
ManagedChannel channel =
NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
.build();
XceiverClientProtocolServiceStub asyncStub =
XceiverClientProtocolServiceGrpc.newStub(channel);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index f04d12f..896caed 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -73,12 +73,12 @@ public final class ScmConfigKeys {
= 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
- public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
- "16KB";
+ public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
+ 16 * 1024;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"dfs.container.ratis.segment.preallocated.size";
- public static final String
- DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "128MB";
+ public static final int
+ DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"dfs.container.ratis.statemachinedata.sync.timeout";
@@ -137,6 +137,7 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
// 16 MB by default
public static final int OZONE_SCM_CHUNK_SIZE_DEFAULT = 16 * 1024 * 1024;
+ public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
public static final String OZONE_SCM_CLIENT_PORT_KEY =
"ozone.scm.client.port";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 8d5c180..8a5762f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -95,9 +95,9 @@ public final class OzoneConfigKeys {
"ozone.container.cache.size";
public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024;
- public static final String OZONE_SCM_BLOCK_SIZE =
- "ozone.scm.block.size";
- public static final String OZONE_SCM_BLOCK_SIZE_DEFAULT = "256MB";
+ public static final String OZONE_SCM_BLOCK_SIZE_IN_MB =
+ "ozone.scm.block.size.in.mb";
+ public static final long OZONE_SCM_BLOCK_SIZE_DEFAULT = 256;
/**
* Ozone administrator users delimited by comma.
@@ -115,14 +115,12 @@ public final class OzoneConfigKeys {
public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE =
"ozone.client.stream.buffer.flush.size";
- public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT =
- "64MB";
+ public static final long OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT = 64;
public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE =
"ozone.client.stream.buffer.max.size";
- public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT =
- "128MB";
+ public static final long OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT = 128;
public static final String OZONE_CLIENT_WATCH_REQUEST_TIMEOUT =
"ozone.client.watch.request.timeout";
@@ -233,13 +231,12 @@ public final class OzoneConfigKeys {
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
- public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
+ public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
- public static final String
- DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT =
- ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
+ public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
+ = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
// config settings to enable stateMachineData write timeout
public static final String
@@ -249,6 +246,8 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
+ public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
+ = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 7dd9fe3..f8f31c8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -146,11 +146,6 @@ public final class OzoneConsts {
public static final String OM_S3_PREFIX ="S3:";
public static final String OM_S3_VOLUME_PREFIX = "s3";
- /**
- * Max chunk size limit.
- */
- public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
-
/**
* Max OM Quota size of 1024 PB.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
index 7e454d6..1ff7695 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
@@ -156,7 +155,7 @@ public interface RatisHelper {
RaftConfigKeys.Rpc.setType(properties, rpcType);
GrpcConfigKeys.setMessageSizeMax(properties,
- SizeInBytes.valueOf(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE));
+ SizeInBytes.valueOf(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE));
return RaftClient.newBuilder()
.setRaftGroup(group)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 94e134d..865335c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -623,11 +623,11 @@
</description>
</property>
<property>
- <name>ozone.scm.block.size</name>
- <value>256MB</value>
+ <name>ozone.scm.block.size.in.mb</name>
+ <value>256</value>
<tag>OZONE, SCM</tag>
<description>
- The default size of a scm block. This is maps to the default
+ The default size of a scm block in bytes. This is maps to the default
Ozone block size.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index be00c8a..992f3cb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.common.helpers.
StorageContainerException;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.ratis.thirdparty.io.grpc.BindableService;
@@ -91,12 +90,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
server = ((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher))
.build();
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher));
for (BindableService service : additionalServices) {
nettyServerBuilder.addService(service);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 3e54700..434d330 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
@@ -34,7 +33,6 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server
@@ -156,25 +154,23 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftConfigKeys.Rpc.setType(properties, rpc);
// set raft segment size
- final int raftSegmentSize = (int)conf.getStorageSize(
+ final int raftSegmentSize = conf.getInt(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT,
- StorageUnit.BYTES);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT);
RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
SizeInBytes.valueOf(raftSegmentSize));
// set raft segment pre-allocated size
- final int raftSegmentPreallocatedSize = (int) conf.getStorageSize(
+ final int raftSegmentPreallocatedSize = conf.getInt(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT,
- StorageUnit.BYTES);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT);
RaftServerConfigKeys.Log.Appender.setBufferCapacity(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
RaftServerConfigKeys.Log.setPreallocatedSize(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
// Set max write buffer size, which is the scm chunk size
- final int maxChunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
+ final int maxChunkSize = OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE;
RaftServerConfigKeys.Log.setWriteBufferSize(properties,
SizeInBytes.valueOf(maxChunkSize));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 768d266..8149e2c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -33,9 +33,9 @@ import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc;
import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc.IntraDatanodeProtocolServiceStub;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
@@ -61,7 +61,7 @@ public class GrpcReplicationClient {
channel = NettyChannelBuilder.forAddress(host, port)
.usePlaintext()
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
.build();
client = IntraDatanodeProtocolServiceGrpc.newStub(channel);
this.workingDirectory = workingDir;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
index d2a4443..5dbe9f6 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -158,9 +159,9 @@ public class ChunkGroupOutputStream extends OutputStream {
this.xceiverClientManager = xceiverClientManager;
this.chunkSize = chunkSize;
this.requestID = requestId;
- this.streamBufferFlushSize = bufferFlushSize;
- this.streamBufferMaxSize = bufferMaxSize;
- this.blockSize = size;
+ this.streamBufferFlushSize = bufferFlushSize * OzoneConsts.MB;
+ this.streamBufferMaxSize = bufferMaxSize * OzoneConsts.MB;
+ this.blockSize = size * OzoneConsts.MB;
this.watchTimeout = watchTimeout;
Preconditions.checkState(chunkSize > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 3a0f475..98481e1 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.client.rpc;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
@@ -140,28 +139,26 @@ public class RpcClient implements ClientProtocol {
this.xceiverClientManager = new XceiverClientManager(conf);
- int configuredChunkSize = (int) conf
- .getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
- if(configuredChunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
+ int configuredChunkSize = conf.getInt(
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
+ if(configuredChunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- configuredChunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
+ configuredChunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
} else {
chunkSize = configuredChunkSize;
}
- streamBufferFlushSize = (long) conf
- .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
- StorageUnit.BYTES);
- streamBufferMaxSize = (long) conf
- .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
- StorageUnit.BYTES);
- blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+ streamBufferFlushSize =
+ conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
+ streamBufferMaxSize =
+ conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
+ blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index ceea3ee..324e17b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -24,7 +24,6 @@ import java.util.Optional;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -393,7 +392,6 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
Files.createDirectories(metaDir);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
if (!chunkSize.isPresent()) {
- //set it to 1MB by default in tests
chunkSize = Optional.of(1);
}
if (!streamBufferFlushSize.isPresent()) {
@@ -405,14 +403,13 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
if (!blockSize.isPresent()) {
blockSize = Optional.of(2 * streamBufferMaxSize.get());
}
- conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- chunkSize.get(), StorageUnit.MB);
- conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- streamBufferFlushSize.get(), StorageUnit.MB);
- conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- streamBufferMaxSize.get(), StorageUnit.MB);
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, blockSize.get(),
- StorageUnit.MB);
+ conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ (int) (chunkSize.get() * OzoneConsts.MB));
+ conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ streamBufferFlushSize.get());
+ conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ streamBufferMaxSize.get());
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, blockSize.get());
configureTrace();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index d268aeb..abd60a1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.client.rpc;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -95,8 +94,7 @@ public class TestCloseContainerHandlingByClient {
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
- StorageUnit.MB);
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(7).build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index 923300f..dc6747f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -82,11 +83,15 @@ public class TestFailureHandlingByClient {
maxRetries = 100;
chunkSize = (int) OzoneConsts.MB;
blockSize = 4 * chunkSize;
+ conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
+ conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE, 1);
+ conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE, 2);
conf.setTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT, 5,
TimeUnit.SECONDS);
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(6).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
index d0f4ce2..3848cda 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -70,8 +69,8 @@ public class TestMultipleContainerReadWrite {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 1,
- StorageUnit.MB);
+ // set to as small as 100 bytes per block.
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 1);
conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 5);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
index fd10055..6a433b5 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.ozone.web.storage;
import com.google.common.base.Strings;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.io.IOUtils;
@@ -101,29 +100,27 @@ public final class DistributedStorageHandler implements StorageHandler {
this.storageContainerLocationClient = storageContainerLocation;
this.xceiverClientManager = new XceiverClientManager(conf);
- chunkSize = (int)conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
+ chunkSize = conf.getInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS,
OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT);
- if(chunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
+ if(chunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- chunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
+ chunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
}
- streamBufferFlushSize = (long) conf
- .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
- StorageUnit.BYTES);
- streamBufferMaxSize = (long) conf
- .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
- StorageUnit.BYTES);
- blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+ streamBufferFlushSize =
+ conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
+ streamBufferMaxSize =
+ conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
+ blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index a41dae6..98a27bf 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.om;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -26,6 +25,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -59,7 +59,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVI
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
/**
* Implementation of keyManager.
@@ -87,9 +87,8 @@ public class KeyManagerImpl implements KeyManager {
String omId) {
this.scmBlockClient = scmBlockClient;
this.metadataManager = metadataManager;
- this.scmBlockSize = (long) conf
- .getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT,
- StorageUnit.BYTES);
+ this.scmBlockSize = conf.getLong(OZONE_SCM_BLOCK_SIZE_IN_MB,
+ OZONE_SCM_BLOCK_SIZE_DEFAULT) * OzoneConsts.MB;
this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY,
DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
this.preallocateMax = conf.getLong(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
index 4e86699..be9342d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
@@ -24,7 +24,6 @@ import java.io.FileOutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -95,10 +94,8 @@ public class GetKeyHandler extends Handler {
OzoneInputStream keyInputStream = bucket.readKey(keyName);
if (dataFilePath != null) {
FileOutputStream outputStream = new FileOutputStream(dataFile);
- IOUtils.copyBytes(keyInputStream, outputStream,
- (int) new OzoneConfiguration()
- .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
- OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES));
+ IOUtils.copyBytes(keyInputStream, outputStream, new OzoneConfiguration()
+ .getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
outputStream.close();
} else {
throw new OzoneClientException(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
index f86af3f..b2ab68f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
@@ -22,7 +22,6 @@ import java.io.File;
import java.io.FileInputStream;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -107,9 +106,8 @@ public class PutKeyHandler extends Handler {
.createKey(keyName, dataFile.length(), replicationType,
replicationFactor);
FileInputStream fileInputStream = new FileInputStream(dataFile);
- IOUtils.copyBytes(fileInputStream, outputStream, (int) conf
- .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT,
- StorageUnit.BYTES));
+ IOUtils.copyBytes(fileInputStream, outputStream,
+ conf.getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
outputStream.close();
fileInputStream.close();
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
index 913bb3e..d3bc857 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.fs.ozone;
import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -65,8 +64,7 @@ public class TestOzoneFSInputStream {
@BeforeClass
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 10,
- StorageUnit.MB);
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(10)
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDDS-732. Add read method which
takes offset and length in SignedChunkInputStream. Contributed by chencan.
Posted by br...@apache.org.
HDDS-732. Add read method which takes offset and length in SignedChunkInputStream. Contributed by chencan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34b6fa7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34b6fa7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34b6fa7d
Branch: refs/heads/HDFS-13891
Commit: 34b6fa7d6b22439f5c703bf7e82de5c40c77994a
Parents: 22867de
Author: Márton Elek <el...@apache.org>
Authored: Wed Nov 21 17:59:20 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Wed Nov 21 17:59:36 2018 +0100
----------------------------------------------------------------------
.../ozone/s3/SignedChunksInputStream.java | 40 ++++++++++++++++++++
.../ozone/s3/TestSignedChunksInputStream.java | 30 +++++++++++++++
2 files changed, 70 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b6fa7d/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
index a35133f..b0dfce6 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
@@ -65,6 +65,46 @@ public class SignedChunksInputStream extends InputStream {
}
}
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ if (b == null) {
+ throw new NullPointerException();
+ } else if (off < 0 || len < 0 || len > b.length - off) {
+ throw new IndexOutOfBoundsException();
+ } else if (len == 0) {
+ return 0;
+ }
+ int currentOff = off;
+ int currentLen = len;
+ int totalReadBytes = 0;
+ int realReadLen = 0;
+ int maxReadLen = 0;
+ do {
+ if (remainingData > 0) {
+ maxReadLen = Math.min(remainingData, currentLen);
+ realReadLen = originalStream.read(b, currentOff, maxReadLen);
+ if (realReadLen == -1) {
+ break;
+ }
+ currentOff += realReadLen;
+ currentLen -= realReadLen;
+ totalReadBytes += realReadLen;
+ remainingData -= realReadLen;
+ if (remainingData == 0) {
+ //read the "\r\n" at the end of the data section
+ originalStream.read();
+ originalStream.read();
+ }
+ } else {
+ remainingData = readHeader();
+ if (remainingData == -1) {
+ break;
+ }
+ }
+ } while (currentLen > 0);
+ return totalReadBytes > 0 ? totalReadBytes : -1;
+ }
+
private int readHeader() throws IOException {
int prev = -1;
int curr = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b6fa7d/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
index f1af7cb..3599c05 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
@@ -49,32 +49,62 @@ public class TestSignedChunksInputStream {
@Test
public void singlechunk() throws IOException {
+ //test simple read()
InputStream is = fileContent("0A;chunk-signature"
+
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ "\n1234567890\r\n");
String result = IOUtils.toString(is, Charset.forName("UTF-8"));
Assert.assertEquals("1234567890", result);
+
+ //test read(byte[],int,int)
+ is = fileContent("0A;chunk-signature"
+ +
+ "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ + "\n1234567890\r\n");
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ Assert.assertEquals("1234567890", new String(bytes));
}
@Test
public void singlechunkwithoutend() throws IOException {
+ //test simple read()
InputStream is = fileContent("0A;chunk-signature"
+
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ "\n1234567890");
String result = IOUtils.toString(is, Charset.forName("UTF-8"));
Assert.assertEquals("1234567890", result);
+
+ //test read(byte[],int,int)
+ is = fileContent("0A;chunk-signature"
+ +
+ "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ + "\n1234567890");
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ Assert.assertEquals("1234567890", new String(bytes));
}
@Test
public void multichunks() throws IOException {
+ //test simple read()
InputStream is = fileContent("0a;chunk-signature=signature\r\n"
+ "1234567890\r\n"
+ "05;chunk-signature=signature\r\n"
+ "abcde\r\n");
String result = IOUtils.toString(is, Charset.forName("UTF-8"));
Assert.assertEquals("1234567890abcde", result);
+
+ //test read(byte[],int,int)
+ is = fileContent("0a;chunk-signature=signature\r\n"
+ + "1234567890\r\n"
+ + "05;chunk-signature=signature\r\n"
+ + "abcde\r\n");
+ byte[] bytes = new byte[15];
+ IOUtils.read(is, bytes, 0, 15);
+ Assert.assertEquals("1234567890abcde", new String(bytes));
}
private InputStream fileContent(String content) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDDS-855. Move OMMetadataManager
from hadoop-ozone/ozone-manager to hadoop-ozone/common. Contributed by Ajay
Kumar.
Posted by br...@apache.org.
HDDS-855. Move OMMetadataManager from hadoop-ozone/ozone-manager to hadoop-ozone/common. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f994b526
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f994b526
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f994b526
Branch: refs/heads/HDFS-13891
Commit: f994b526a03738fea95583a9fccbac709e8ce47f
Parents: a41b648
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Nov 20 20:31:07 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Nov 20 20:31:07 2018 -0800
----------------------------------------------------------------------
.../hadoop/ozone/om/OMMetadataManager.java | 245 +++++++++++++++++++
.../hadoop/ozone/om/OzoneManagerLock.java | 222 +++++++++++++++++
.../hadoop/ozone/om/OMMetadataManager.java | 245 -------------------
.../hadoop/ozone/om/OzoneManagerLock.java | 222 -----------------
4 files changed, 467 insertions(+), 467 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f994b526/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
new file mode 100644
index 0000000..5f490ec
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.ozone.common.BlockGroup;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.utils.db.DBStore;
+import org.apache.hadoop.utils.db.Table;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * OM metadata manager interface.
+ */
+public interface OMMetadataManager {
+ /**
+ * Start metadata manager.
+ */
+ void start();
+
+ /**
+ * Stop metadata manager.
+ */
+ void stop() throws Exception;
+
+ /**
+ * Get metadata store.
+ *
+ * @return metadata store.
+ */
+ @VisibleForTesting
+ DBStore getStore();
+
+ /**
+ * Returns the OzoneManagerLock used on Metadata DB.
+ *
+ * @return OzoneManagerLock
+ */
+ OzoneManagerLock getLock();
+
+ /**
+ * Given a volume return the corresponding DB key.
+ *
+ * @param volume - Volume name
+ */
+ byte[] getVolumeKey(String volume);
+
+ /**
+ * Given a user return the corresponding DB key.
+ *
+ * @param user - User name
+ */
+ byte[] getUserKey(String user);
+
+ /**
+ * Given a volume and bucket, return the corresponding DB key.
+ *
+ * @param volume - User name
+ * @param bucket - Bucket name
+ */
+ byte[] getBucketKey(String volume, String bucket);
+
+ /**
+ * Given a volume, bucket and a key, return the corresponding DB key.
+ *
+ * @param volume - volume name
+ * @param bucket - bucket name
+ * @param key - key name
+ * @return bytes of DB key.
+ */
+ byte[] getOzoneKeyBytes(String volume, String bucket, String key);
+
+ /**
+ * Returns the DB key name of a open key in OM metadata store. Should be
+ * #open# prefix followed by actual key name.
+ *
+ * @param volume - volume name
+ * @param bucket - bucket name
+ * @param key - key name
+ * @param id - the id for this open
+ * @return bytes of DB key.
+ */
+ byte[] getOpenKeyBytes(String volume, String bucket, String key, long id);
+
+ /**
+ * Given a volume, check if it is empty, i.e there are no buckets inside it.
+ *
+ * @param volume - Volume name
+ */
+ boolean isVolumeEmpty(String volume) throws IOException;
+
+ /**
+ * Given a volume/bucket, check if it is empty, i.e there are no keys inside
+ * it.
+ *
+ * @param volume - Volume name
+ * @param bucket - Bucket name
+ * @return true if the bucket is empty
+ */
+ boolean isBucketEmpty(String volume, String bucket) throws IOException;
+
+ /**
+ * Returns a list of buckets represented by {@link OmBucketInfo} in the given
+ * volume.
+ *
+ * @param volumeName the name of the volume. This argument is required, this
+ * method returns buckets in this given volume.
+ * @param startBucket the start bucket name. Only the buckets whose name is
+ * after this value will be included in the result. This key is excluded from
+ * the result.
+ * @param bucketPrefix bucket name prefix. Only the buckets whose name has
+ * this prefix will be included in the result.
+ * @param maxNumOfBuckets the maximum number of buckets to return. It ensures
+ * the size of the result will not exceed this limit.
+ * @return a list of buckets.
+ * @throws IOException
+ */
+ List<OmBucketInfo> listBuckets(String volumeName, String startBucket,
+ String bucketPrefix, int maxNumOfBuckets)
+ throws IOException;
+
+ /**
+ * Returns a list of keys represented by {@link OmKeyInfo} in the given
+ * bucket.
+ *
+ * @param volumeName the name of the volume.
+ * @param bucketName the name of the bucket.
+ * @param startKey the start key name, only the keys whose name is after this
+ * value will be included in the result. This key is excluded from the
+ * result.
+ * @param keyPrefix key name prefix, only the keys whose name has this prefix
+ * will be included in the result.
+ * @param maxKeys the maximum number of keys to return. It ensures the size of
+ * the result will not exceed this limit.
+ * @return a list of keys.
+ * @throws IOException
+ */
+ List<OmKeyInfo> listKeys(String volumeName,
+ String bucketName, String startKey, String keyPrefix, int maxKeys)
+ throws IOException;
+
+ /**
+ * Returns a list of volumes owned by a given user; if user is null, returns
+ * all volumes.
+ *
+ * @param userName volume owner
+ * @param prefix the volume prefix used to filter the listing result.
+ * @param startKey the start volume name determines where to start listing
+ * from, this key is excluded from the result.
+ * @param maxKeys the maximum number of volumes to return.
+ * @return a list of {@link OmVolumeArgs}
+ * @throws IOException
+ */
+ List<OmVolumeArgs> listVolumes(String userName, String prefix,
+ String startKey, int maxKeys) throws IOException;
+
+ /**
+ * Returns a list of pending deletion key info that ups to the given count.
+ * Each entry is a {@link BlockGroup}, which contains the info about the key
+ * name and all its associated block IDs. A pending deletion key is stored
+ * with #deleting# prefix in OM DB.
+ *
+ * @param count max number of keys to return.
+ * @return a list of {@link BlockGroup} represent keys and blocks.
+ * @throws IOException
+ */
+ List<BlockGroup> getPendingDeletionKeys(int count) throws IOException;
+
+ /**
+ * Returns a list of all still open key info. Which contains the info about
+ * the key name and all its associated block IDs. A pending open key has
+ * prefix #open# in OM DB.
+ *
+ * @return a list of {@link BlockGroup} representing keys and blocks.
+ * @throws IOException
+ */
+ List<BlockGroup> getExpiredOpenKeys() throws IOException;
+
+ /**
+ * Returns the user Table.
+ *
+ * @return UserTable.
+ */
+ Table getUserTable();
+
+ /**
+ * Returns the Volume Table.
+ *
+ * @return VolumeTable.
+ */
+ Table getVolumeTable();
+
+ /**
+ * Returns the BucketTable.
+ *
+ * @return BucketTable.
+ */
+ Table getBucketTable();
+
+ /**
+ * Returns the KeyTable.
+ *
+ * @return KeyTable.
+ */
+ Table getKeyTable();
+
+ /**
+ * Get Deleted Table.
+ *
+ * @return Deleted Table.
+ */
+ Table getDeletedTable();
+
+ /**
+ * Gets the OpenKeyTable.
+ *
+ * @return Table.
+ */
+ Table getOpenKeyTable();
+
+ /**
+ * Gets the S3Bucket to Ozone Volume/bucket mapping table.
+ *
+ * @return Table.
+ */
+ Table getS3Table();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f994b526/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
new file mode 100644
index 0000000..c5ce9e2
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ozone.lock.LockManager;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_USER_PREFIX;
+
+/**
+ * Provides different locks to handle concurrency in OzoneMaster.
+ * We also maintain lock hierarchy, based on the weight.
+ *
+ * <table>
+ * <caption></caption>
+ * <tr>
+ * <td><b> WEIGHT </b></td> <td><b> LOCK </b></td>
+ * </tr>
+ * <tr>
+ * <td> 0 </td> <td> User Lock </td>
+ * </tr>
+ * <tr>
+ * <td> 1 </td> <td> Volume Lock </td>
+ * </tr>
+ * <tr>
+ * <td> 2 </td> <td> Bucket Lock </td>
+ * </tr>
+ * </table>
+ *
+ * One cannot obtain a lower weight lock while holding a lock with higher
+ * weight. The other way around is possible. <br>
+ * <br>
+ * <p>
+ * For example:
+ * <br>
+ * {@literal ->} acquireVolumeLock (will work)<br>
+ * {@literal +->} acquireBucketLock (will work)<br>
+ * {@literal +-->} acquireUserLock (will throw Exception)<br>
+ * </p>
+ * <br>
+ * To acquire a user lock you should not hold any Volume/Bucket lock. Similarly
+ * to acquire a Volume lock you should not hold any Bucket lock.
+ */
+public final class OzoneManagerLock {
+
+ private static final String VOLUME_LOCK = "volumeLock";
+ private static final String BUCKET_LOCK = "bucketLock";
+ private static final String S3_BUCKET_LOCK = "s3BucketLock";
+
+ private final LockManager<String> manager;
+
+ // To maintain locks held by current thread.
+ private final ThreadLocal<Map<String, AtomicInteger>> myLocks =
+ ThreadLocal.withInitial(
+ () -> ImmutableMap.of(
+ VOLUME_LOCK, new AtomicInteger(0),
+ BUCKET_LOCK, new AtomicInteger(0),
+ S3_BUCKET_LOCK, new AtomicInteger(0)
+ )
+ );
+
+ /**
+ * Creates new OzoneManagerLock instance.
+ * @param conf Configuration object
+ */
+ public OzoneManagerLock(Configuration conf) {
+ manager = new LockManager<>(conf);
+ }
+
+ /**
+ * Acquires user lock on the given resource.
+ *
+ * <p>If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the
+ * lock has been acquired.
+ *
+ * @param user User on which the lock has to be acquired
+ */
+ public void acquireUserLock(String user) {
+ // Calling thread should not hold any volume or bucket lock.
+ if (hasAnyVolumeLock() || hasAnyBucketLock() || hasAnyS3Lock()) {
+ throw new RuntimeException(
+ "Thread '" + Thread.currentThread().getName() +
+ "' cannot acquire user lock" +
+ " while holding volume, bucket or S3 bucket lock(s).");
+ }
+ manager.lock(OM_USER_PREFIX + user);
+ }
+
+ /**
+ * Releases the user lock on given resource.
+ */
+ public void releaseUserLock(String user) {
+ manager.unlock(OM_USER_PREFIX + user);
+ }
+
+ /**
+ * Acquires volume lock on the given resource.
+ *
+ * <p>If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the
+ * lock has been acquired.
+ *
+ * @param volume Volume on which the lock has to be acquired
+ */
+ public void acquireVolumeLock(String volume) {
+ // Calling thread should not hold any bucket lock.
+ // You can take an Volume while holding S3 bucket lock, since
+ // semantically an S3 bucket maps to the ozone volume. So we check here
+ // only if ozone bucket lock is taken.
+ if (hasAnyBucketLock()) {
+ throw new RuntimeException(
+ "Thread '" + Thread.currentThread().getName() +
+ "' cannot acquire volume lock while holding bucket lock(s).");
+ }
+ manager.lock(OM_KEY_PREFIX + volume);
+ myLocks.get().get(VOLUME_LOCK).incrementAndGet();
+ }
+
+ /**
+ * Releases the volume lock on given resource.
+ */
+ public void releaseVolumeLock(String volume) {
+ manager.unlock(OM_KEY_PREFIX + volume);
+ myLocks.get().get(VOLUME_LOCK).decrementAndGet();
+ }
+
+ /**
+ * Acquires S3 Bucket lock on the given resource.
+ *
+ * <p>If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the lock has
+ * been acquired.
+ *
+ * @param s3BucketName S3Bucket Name on which the lock has to be acquired
+ */
+ public void acquireS3Lock(String s3BucketName) {
+ // Calling thread should not hold any bucket lock.
+ // You can take an Volume while holding S3 bucket lock, since
+ // semantically an S3 bucket maps to the ozone volume. So we check here
+ // only if ozone bucket lock is taken.
+ if (hasAnyBucketLock()) {
+ throw new RuntimeException(
+ "Thread '" + Thread.currentThread().getName() +
+ "' cannot acquire S3 bucket lock while holding Ozone bucket " +
+ "lock(s).");
+ }
+ manager.lock(OM_S3_PREFIX + s3BucketName);
+ myLocks.get().get(S3_BUCKET_LOCK).incrementAndGet();
+ }
+
+ /**
+ * Releases the volume lock on given resource.
+ */
+ public void releaseS3Lock(String s3BucketName) {
+ manager.unlock(OM_S3_PREFIX + s3BucketName);
+ myLocks.get().get(S3_BUCKET_LOCK).decrementAndGet();
+ }
+
+ /**
+ * Acquires bucket lock on the given resource.
+ *
+ * <p>If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the
+ * lock has been acquired.
+ *
+ * @param bucket Bucket on which the lock has to be acquired
+ */
+ public void acquireBucketLock(String volume, String bucket) {
+ manager.lock(OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket);
+ myLocks.get().get(BUCKET_LOCK).incrementAndGet();
+ }
+
+ /**
+ * Releases the bucket lock on given resource.
+ */
+ public void releaseBucketLock(String volume, String bucket) {
+ manager.unlock(OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket);
+ myLocks.get().get(BUCKET_LOCK).decrementAndGet();
+ }
+
+ /**
+ * Returns true if the current thread holds any volume lock.
+ * @return true if current thread holds volume lock, else false
+ */
+ private boolean hasAnyVolumeLock() {
+ return myLocks.get().get(VOLUME_LOCK).get() != 0;
+ }
+
+ /**
+ * Returns true if the current thread holds any bucket lock.
+ * @return true if current thread holds bucket lock, else false
+ */
+ private boolean hasAnyBucketLock() {
+ return myLocks.get().get(BUCKET_LOCK).get() != 0;
+ }
+
+ private boolean hasAnyS3Lock() {
+ return myLocks.get().get(S3_BUCKET_LOCK).get() != 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f994b526/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
deleted file mode 100644
index 5f490ec..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.om;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
-import org.apache.hadoop.utils.db.DBStore;
-import org.apache.hadoop.utils.db.Table;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * OM metadata manager interface.
- */
-public interface OMMetadataManager {
- /**
- * Start metadata manager.
- */
- void start();
-
- /**
- * Stop metadata manager.
- */
- void stop() throws Exception;
-
- /**
- * Get metadata store.
- *
- * @return metadata store.
- */
- @VisibleForTesting
- DBStore getStore();
-
- /**
- * Returns the OzoneManagerLock used on Metadata DB.
- *
- * @return OzoneManagerLock
- */
- OzoneManagerLock getLock();
-
- /**
- * Given a volume return the corresponding DB key.
- *
- * @param volume - Volume name
- */
- byte[] getVolumeKey(String volume);
-
- /**
- * Given a user return the corresponding DB key.
- *
- * @param user - User name
- */
- byte[] getUserKey(String user);
-
- /**
- * Given a volume and bucket, return the corresponding DB key.
- *
- * @param volume - User name
- * @param bucket - Bucket name
- */
- byte[] getBucketKey(String volume, String bucket);
-
- /**
- * Given a volume, bucket and a key, return the corresponding DB key.
- *
- * @param volume - volume name
- * @param bucket - bucket name
- * @param key - key name
- * @return bytes of DB key.
- */
- byte[] getOzoneKeyBytes(String volume, String bucket, String key);
-
- /**
- * Returns the DB key name of a open key in OM metadata store. Should be
- * #open# prefix followed by actual key name.
- *
- * @param volume - volume name
- * @param bucket - bucket name
- * @param key - key name
- * @param id - the id for this open
- * @return bytes of DB key.
- */
- byte[] getOpenKeyBytes(String volume, String bucket, String key, long id);
-
- /**
- * Given a volume, check if it is empty, i.e there are no buckets inside it.
- *
- * @param volume - Volume name
- */
- boolean isVolumeEmpty(String volume) throws IOException;
-
- /**
- * Given a volume/bucket, check if it is empty, i.e there are no keys inside
- * it.
- *
- * @param volume - Volume name
- * @param bucket - Bucket name
- * @return true if the bucket is empty
- */
- boolean isBucketEmpty(String volume, String bucket) throws IOException;
-
- /**
- * Returns a list of buckets represented by {@link OmBucketInfo} in the given
- * volume.
- *
- * @param volumeName the name of the volume. This argument is required, this
- * method returns buckets in this given volume.
- * @param startBucket the start bucket name. Only the buckets whose name is
- * after this value will be included in the result. This key is excluded from
- * the result.
- * @param bucketPrefix bucket name prefix. Only the buckets whose name has
- * this prefix will be included in the result.
- * @param maxNumOfBuckets the maximum number of buckets to return. It ensures
- * the size of the result will not exceed this limit.
- * @return a list of buckets.
- * @throws IOException
- */
- List<OmBucketInfo> listBuckets(String volumeName, String startBucket,
- String bucketPrefix, int maxNumOfBuckets)
- throws IOException;
-
- /**
- * Returns a list of keys represented by {@link OmKeyInfo} in the given
- * bucket.
- *
- * @param volumeName the name of the volume.
- * @param bucketName the name of the bucket.
- * @param startKey the start key name, only the keys whose name is after this
- * value will be included in the result. This key is excluded from the
- * result.
- * @param keyPrefix key name prefix, only the keys whose name has this prefix
- * will be included in the result.
- * @param maxKeys the maximum number of keys to return. It ensures the size of
- * the result will not exceed this limit.
- * @return a list of keys.
- * @throws IOException
- */
- List<OmKeyInfo> listKeys(String volumeName,
- String bucketName, String startKey, String keyPrefix, int maxKeys)
- throws IOException;
-
- /**
- * Returns a list of volumes owned by a given user; if user is null, returns
- * all volumes.
- *
- * @param userName volume owner
- * @param prefix the volume prefix used to filter the listing result.
- * @param startKey the start volume name determines where to start listing
- * from, this key is excluded from the result.
- * @param maxKeys the maximum number of volumes to return.
- * @return a list of {@link OmVolumeArgs}
- * @throws IOException
- */
- List<OmVolumeArgs> listVolumes(String userName, String prefix,
- String startKey, int maxKeys) throws IOException;
-
- /**
- * Returns a list of pending deletion key info that ups to the given count.
- * Each entry is a {@link BlockGroup}, which contains the info about the key
- * name and all its associated block IDs. A pending deletion key is stored
- * with #deleting# prefix in OM DB.
- *
- * @param count max number of keys to return.
- * @return a list of {@link BlockGroup} represent keys and blocks.
- * @throws IOException
- */
- List<BlockGroup> getPendingDeletionKeys(int count) throws IOException;
-
- /**
- * Returns a list of all still open key info. Which contains the info about
- * the key name and all its associated block IDs. A pending open key has
- * prefix #open# in OM DB.
- *
- * @return a list of {@link BlockGroup} representing keys and blocks.
- * @throws IOException
- */
- List<BlockGroup> getExpiredOpenKeys() throws IOException;
-
- /**
- * Returns the user Table.
- *
- * @return UserTable.
- */
- Table getUserTable();
-
- /**
- * Returns the Volume Table.
- *
- * @return VolumeTable.
- */
- Table getVolumeTable();
-
- /**
- * Returns the BucketTable.
- *
- * @return BucketTable.
- */
- Table getBucketTable();
-
- /**
- * Returns the KeyTable.
- *
- * @return KeyTable.
- */
- Table getKeyTable();
-
- /**
- * Get Deleted Table.
- *
- * @return Deleted Table.
- */
- Table getDeletedTable();
-
- /**
- * Gets the OpenKeyTable.
- *
- * @return Table.
- */
- Table getOpenKeyTable();
-
- /**
- * Gets the S3Bucket to Ozone Volume/bucket mapping table.
- *
- * @return Table.
- */
- Table getS3Table();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f994b526/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
deleted file mode 100644
index c5ce9e2..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.om;
-
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.lock.LockManager;
-
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_USER_PREFIX;
-
-/**
- * Provides different locks to handle concurrency in OzoneMaster.
- * We also maintain lock hierarchy, based on the weight.
- *
- * <table>
- * <caption></caption>
- * <tr>
- * <td><b> WEIGHT </b></td> <td><b> LOCK </b></td>
- * </tr>
- * <tr>
- * <td> 0 </td> <td> User Lock </td>
- * </tr>
- * <tr>
- * <td> 1 </td> <td> Volume Lock </td>
- * </tr>
- * <tr>
- * <td> 2 </td> <td> Bucket Lock </td>
- * </tr>
- * </table>
- *
- * One cannot obtain a lower weight lock while holding a lock with higher
- * weight. The other way around is possible. <br>
- * <br>
- * <p>
- * For example:
- * <br>
- * {@literal ->} acquireVolumeLock (will work)<br>
- * {@literal +->} acquireBucketLock (will work)<br>
- * {@literal +-->} acquireUserLock (will throw Exception)<br>
- * </p>
- * <br>
- * To acquire a user lock you should not hold any Volume/Bucket lock. Similarly
- * to acquire a Volume lock you should not hold any Bucket lock.
- */
-public final class OzoneManagerLock {
-
- private static final String VOLUME_LOCK = "volumeLock";
- private static final String BUCKET_LOCK = "bucketLock";
- private static final String S3_BUCKET_LOCK = "s3BucketLock";
-
- private final LockManager<String> manager;
-
- // To maintain locks held by current thread.
- private final ThreadLocal<Map<String, AtomicInteger>> myLocks =
- ThreadLocal.withInitial(
- () -> ImmutableMap.of(
- VOLUME_LOCK, new AtomicInteger(0),
- BUCKET_LOCK, new AtomicInteger(0),
- S3_BUCKET_LOCK, new AtomicInteger(0)
- )
- );
-
- /**
- * Creates new OzoneManagerLock instance.
- * @param conf Configuration object
- */
- public OzoneManagerLock(Configuration conf) {
- manager = new LockManager<>(conf);
- }
-
- /**
- * Acquires user lock on the given resource.
- *
- * <p>If the lock is not available then the current thread becomes
- * disabled for thread scheduling purposes and lies dormant until the
- * lock has been acquired.
- *
- * @param user User on which the lock has to be acquired
- */
- public void acquireUserLock(String user) {
- // Calling thread should not hold any volume or bucket lock.
- if (hasAnyVolumeLock() || hasAnyBucketLock() || hasAnyS3Lock()) {
- throw new RuntimeException(
- "Thread '" + Thread.currentThread().getName() +
- "' cannot acquire user lock" +
- " while holding volume, bucket or S3 bucket lock(s).");
- }
- manager.lock(OM_USER_PREFIX + user);
- }
-
- /**
- * Releases the user lock on given resource.
- */
- public void releaseUserLock(String user) {
- manager.unlock(OM_USER_PREFIX + user);
- }
-
- /**
- * Acquires volume lock on the given resource.
- *
- * <p>If the lock is not available then the current thread becomes
- * disabled for thread scheduling purposes and lies dormant until the
- * lock has been acquired.
- *
- * @param volume Volume on which the lock has to be acquired
- */
- public void acquireVolumeLock(String volume) {
- // Calling thread should not hold any bucket lock.
- // You can take an Volume while holding S3 bucket lock, since
- // semantically an S3 bucket maps to the ozone volume. So we check here
- // only if ozone bucket lock is taken.
- if (hasAnyBucketLock()) {
- throw new RuntimeException(
- "Thread '" + Thread.currentThread().getName() +
- "' cannot acquire volume lock while holding bucket lock(s).");
- }
- manager.lock(OM_KEY_PREFIX + volume);
- myLocks.get().get(VOLUME_LOCK).incrementAndGet();
- }
-
- /**
- * Releases the volume lock on given resource.
- */
- public void releaseVolumeLock(String volume) {
- manager.unlock(OM_KEY_PREFIX + volume);
- myLocks.get().get(VOLUME_LOCK).decrementAndGet();
- }
-
- /**
- * Acquires S3 Bucket lock on the given resource.
- *
- * <p>If the lock is not available then the current thread becomes
- * disabled for thread scheduling purposes and lies dormant until the lock has
- * been acquired.
- *
- * @param s3BucketName S3Bucket Name on which the lock has to be acquired
- */
- public void acquireS3Lock(String s3BucketName) {
- // Calling thread should not hold any bucket lock.
- // You can take an Volume while holding S3 bucket lock, since
- // semantically an S3 bucket maps to the ozone volume. So we check here
- // only if ozone bucket lock is taken.
- if (hasAnyBucketLock()) {
- throw new RuntimeException(
- "Thread '" + Thread.currentThread().getName() +
- "' cannot acquire S3 bucket lock while holding Ozone bucket " +
- "lock(s).");
- }
- manager.lock(OM_S3_PREFIX + s3BucketName);
- myLocks.get().get(S3_BUCKET_LOCK).incrementAndGet();
- }
-
- /**
- * Releases the volume lock on given resource.
- */
- public void releaseS3Lock(String s3BucketName) {
- manager.unlock(OM_S3_PREFIX + s3BucketName);
- myLocks.get().get(S3_BUCKET_LOCK).decrementAndGet();
- }
-
- /**
- * Acquires bucket lock on the given resource.
- *
- * <p>If the lock is not available then the current thread becomes
- * disabled for thread scheduling purposes and lies dormant until the
- * lock has been acquired.
- *
- * @param bucket Bucket on which the lock has to be acquired
- */
- public void acquireBucketLock(String volume, String bucket) {
- manager.lock(OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket);
- myLocks.get().get(BUCKET_LOCK).incrementAndGet();
- }
-
- /**
- * Releases the bucket lock on given resource.
- */
- public void releaseBucketLock(String volume, String bucket) {
- manager.unlock(OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket);
- myLocks.get().get(BUCKET_LOCK).decrementAndGet();
- }
-
- /**
- * Returns true if the current thread holds any volume lock.
- * @return true if current thread holds volume lock, else false
- */
- private boolean hasAnyVolumeLock() {
- return myLocks.get().get(VOLUME_LOCK).get() != 0;
- }
-
- /**
- * Returns true if the current thread holds any bucket lock.
- * @return true if current thread holds bucket lock, else false
- */
- private boolean hasAnyBucketLock() {
- return myLocks.get().get(BUCKET_LOCK).get() != 0;
- }
-
- private boolean hasAnyS3Lock() {
- return myLocks.get().get(S3_BUCKET_LOCK).get() != 0;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDFS-14064. WEBHDFS: Support
Enable/Disable EC Policy. Contributed by Ayush Saxena.
Posted by br...@apache.org.
HDFS-14064. WEBHDFS: Support Enable/Disable EC Policy. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/892b33e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/892b33e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/892b33e0
Branch: refs/heads/HDFS-13891
Commit: 892b33e054efdcfb6127f20b7468fb08b0e2b8aa
Parents: c4d0ef6
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Thu Nov 22 00:06:22 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:13:56 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 10 +++++
.../hdfs/web/resources/ECPolicyParam.java | 42 ++++++++++++++++++
.../hadoop/hdfs/web/resources/PutOpParam.java | 3 ++
.../federation/router/RouterWebHdfsMethods.java | 8 +++-
.../web/resources/NamenodeWebHdfsMethods.java | 25 ++++++++---
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 45 +++++++++++++++++++
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 46 ++++++++++++++++++++
.../hadoop/hdfs/web/resources/TestParam.java | 8 ++++
8 files changed, 180 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index c139cb0..51d4442 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -1310,6 +1310,16 @@ public class WebHdfsFileSystem extends FileSystem
new FsPathRunner(op, p).run();
}
+ public void enableECPolicy(String policyName) throws IOException {
+ final HttpOpParam.Op op = PutOpParam.Op.ENABLEECPOLICY;
+ new FsPathRunner(op, null, new ECPolicyParam(policyName)).run();
+ }
+
+ public void disableECPolicy(String policyName) throws IOException {
+ final HttpOpParam.Op op = PutOpParam.Op.DISABLEECPOLICY;
+ new FsPathRunner(op, null, new ECPolicyParam(policyName)).run();
+ }
+
@Override
public Path createSnapshot(final Path path, final String snapshotName)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/ECPolicyParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/ECPolicyParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/ECPolicyParam.java
new file mode 100644
index 0000000..35502e3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/ECPolicyParam.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** policy parameter. */
+public class ECPolicyParam extends StringParam {
+ /** Parameter name. */
+ public static final String NAME = "ecpolicy";
+ /** Default parameter value. */
+ public static final String DEFAULT = "";
+
+ private static final Domain DOMAIN = new Domain(NAME, null);
+
+ /**
+ * Constructor.
+ *
+ * @param str a string representation of the parameter value.
+ */
+ public ECPolicyParam(final String str) {
+ super(DOMAIN, str == null || str.equals(DEFAULT) ? null : str);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
index 558bb53..7bbd361 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
@@ -46,6 +46,9 @@ public class PutOpParam extends HttpOpParam<PutOpParam.Op> {
SETXATTR(false, HttpURLConnection.HTTP_OK),
REMOVEXATTR(false, HttpURLConnection.HTTP_OK),
+ ENABLEECPOLICY(false, HttpURLConnection.HTTP_OK),
+ DISABLEECPOLICY(false, HttpURLConnection.HTTP_OK),
+
ALLOWSNAPSHOT(false, HttpURLConnection.HTTP_OK),
DISALLOWSNAPSHOT(false, HttpURLConnection.HTTP_OK),
CREATESNAPSHOT(false, HttpURLConnection.HTTP_OK),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
index 5e9d4d0..34c3788 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
import org.apache.hadoop.hdfs.web.resources.DelegationParam;
import org.apache.hadoop.hdfs.web.resources.DestinationParam;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
+import org.apache.hadoop.hdfs.web.resources.ECPolicyParam;
import org.apache.hadoop.hdfs.web.resources.ExcludeDatanodesParam;
import org.apache.hadoop.hdfs.web.resources.FsActionParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
@@ -215,7 +216,8 @@ public class RouterWebHdfsMethods extends NamenodeWebHdfsMethods {
final ExcludeDatanodesParam exclDatanodes,
final CreateFlagParam createFlagParam,
final NoRedirectParam noredirectParam,
- final StoragePolicyParam policyName
+ final StoragePolicyParam policyName,
+ final ECPolicyParam ecpolicy
) throws IOException, URISyntaxException {
switch(op.getValue()) {
@@ -252,6 +254,8 @@ public class RouterWebHdfsMethods extends NamenodeWebHdfsMethods {
case RENAMESNAPSHOT:
case DISALLOWSNAPSHOT:
case SETSTORAGEPOLICY:
+ case ENABLEECPOLICY:
+ case DISABLEECPOLICY:
{
// Whitelist operations that can handled by NamenodeWebHdfsMethods
return super.put(ugi, delegation, username, doAsUser, fullpath, op,
@@ -260,7 +264,7 @@ public class RouterWebHdfsMethods extends NamenodeWebHdfsMethods {
accessTime, renameOptions, createParent, delegationTokenArgument,
aclPermission, xattrName, xattrValue, xattrSetFlag, snapshotName,
oldSnapshotName, exclDatanodes, createFlagParam, noredirectParam,
- policyName);
+ policyName, ecpolicy);
}
default:
throw new UnsupportedOperationException(op + " is not supported");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index c4d3239..6940f2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -492,14 +492,16 @@ public class NamenodeWebHdfsMethods {
@QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT)
final NoRedirectParam noredirect,
@QueryParam(StoragePolicyParam.NAME) @DefaultValue(StoragePolicyParam
- .DEFAULT) final StoragePolicyParam policyName
+ .DEFAULT) final StoragePolicyParam policyName,
+ @QueryParam(ECPolicyParam.NAME) @DefaultValue(ECPolicyParam
+ .DEFAULT) final ECPolicyParam ecpolicy
) throws IOException, InterruptedException {
return put(ugi, delegation, username, doAsUser, ROOT, op, destination,
owner, group, permission, unmaskedPermission, overwrite, bufferSize,
replication, blockSize, modificationTime, accessTime, renameOptions,
createParent, delegationTokenArgument, aclPermission, xattrName,
xattrValue, xattrSetFlag, snapshotName, oldSnapshotName,
- excludeDatanodes, createFlagParam, noredirect, policyName);
+ excludeDatanodes, createFlagParam, noredirect, policyName, ecpolicy);
}
/** Validate all required params. */
@@ -579,7 +581,9 @@ public class NamenodeWebHdfsMethods {
@QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT)
final NoRedirectParam noredirect,
@QueryParam(StoragePolicyParam.NAME) @DefaultValue(StoragePolicyParam
- .DEFAULT) final StoragePolicyParam policyName
+ .DEFAULT) final StoragePolicyParam policyName,
+ @QueryParam(ECPolicyParam.NAME) @DefaultValue(ECPolicyParam.DEFAULT)
+ final ECPolicyParam ecpolicy
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, destination, owner,
@@ -599,7 +603,7 @@ public class NamenodeWebHdfsMethods {
renameOptions, createParent, delegationTokenArgument,
aclPermission, xattrName, xattrValue, xattrSetFlag,
snapshotName, oldSnapshotName, excludeDatanodes,
- createFlagParam, noredirect, policyName);
+ createFlagParam, noredirect, policyName, ecpolicy);
}
});
}
@@ -634,7 +638,8 @@ public class NamenodeWebHdfsMethods {
final ExcludeDatanodesParam exclDatanodes,
final CreateFlagParam createFlagParam,
final NoRedirectParam noredirectParam,
- final StoragePolicyParam policyName
+ final StoragePolicyParam policyName,
+ final ECPolicyParam ecpolicy
) throws IOException, URISyntaxException {
final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF);
final ClientProtocol cp = getRpcClientProtocol();
@@ -795,6 +800,16 @@ public class NamenodeWebHdfsMethods {
cp.setStoragePolicy(fullpath, policyName.getValue());
return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
}
+ case ENABLEECPOLICY:
+ validateOpParams(op, ecpolicy);
+ cp.enableErasureCodingPolicy(ecpolicy.getValue());
+ return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
+
+ case DISABLEECPOLICY:
+ validateOpParams(op, ecpolicy);
+ cp.disableErasureCodingPolicy(ecpolicy.getValue());
+ return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
+
default:
throw new UnsupportedOperationException(op + " is not supported");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index e29b932..383eda0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -69,6 +69,8 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
* [`SETXATTR`](#Set_XAttr) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setXAttr)
* [`REMOVEXATTR`](#Remove_XAttr) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).removeXAttr)
* [`SETSTORAGEPOLICY`](#Set_Storage_Policy) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setStoragePolicy)
+ * [`ENABLEECPOLICY`](#Enable_EC_Policy) (see [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).enablePolicy)
+ * [`DISABLEECPOLICY`](#Disable_EC_Policy) (see [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).disablePolicy)
* HTTP POST
* [`APPEND`](#Append_to_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).append)
* [`CONCAT`](#Concat_Files) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).concat)
@@ -1266,6 +1268,37 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getXAttrs
See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).listXAttrs
+Erasure Coding Operations
+-------------------------
+
+### Enable EC Policy
+
+* Submit a HTTP PUT request.
+
+ curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/?op=ENABLEECPOLICY
+ &ecpolicy=<policy>"
+
+ The client receives a response with zero content length:
+
+ HTTP/1.1 200 OK
+ Content-Length: 0
+
+See also: [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).enablePolicy)
+
+### Disable EC Policy
+
+* Submit a HTTP PUT request.
+
+ curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/?op=DISABLEECPOLICY
+ &ecpolicy=<policy>"
+
+ The client receives a response with zero content length:
+
+ HTTP/1.1 200 OK
+ Content-Length: 0
+
+See also: [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).disablePolicy)
+
Snapshot Operations
-------------------
@@ -2822,6 +2855,18 @@ See also: [Create and Write to a File](#Create_and_Write_to_a_File)
See also: [`SETSTORAGEPOLICY`](#Set_Storage_Policy)
+### Erasure Coding Policy
+
+| Name | `ecpolicy` |
+|:---- |:---- |
+| Description | The name of the erasure coding policy. |
+| Type | String |
+| Default Value | \<empty\> |
+| Valid Values | Any valid erasure coding policy name; |
+| Syntax | Any string. |
+
+See also: [`ENABLEECPOLICY`](#Enable_EC_Policy) or [`DISABLEECPOLICY`](#Disable_EC_Policy)
+
### Start After
| Name | `startAfter` |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 71948d6..8173bca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -47,6 +47,8 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
import java.util.Map;
import java.util.Random;
@@ -83,6 +85,7 @@ import org.apache.hadoop.hdfs.TestDFSClientRetries;
import org.apache.hadoop.hdfs.TestFileCreation;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
@@ -1599,6 +1602,49 @@ public class TestWebHDFS {
}
}
+ private void checkECPolicyState(Collection<ErasureCodingPolicyInfo> policies,
+ String ecpolicy, String state) {
+ Iterator<ErasureCodingPolicyInfo> itr = policies.iterator();
+ boolean found = false;
+ while (policies.iterator().hasNext()) {
+ ErasureCodingPolicyInfo policy = itr.next();
+ if (policy.getPolicy().getName().equals(ecpolicy)) {
+ found = true;
+ if (state.equals("disable")) {
+ Assert.assertTrue(policy.isDisabled());
+ } else if (state.equals("enable")) {
+ Assert.assertTrue(policy.isEnabled());
+ }
+ break;
+ }
+ }
+ Assert.assertTrue(found);
+ }
+
+ // Test For Enable/Disable EC Policy in DFS.
+ @Test
+ public void testEnableDisableECPolicy() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ try (MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(0).build()) {
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final WebHdfsFileSystem webHdfs = WebHdfsTestUtil
+ .getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
+ String policy = "RS-10-4-1024k";
+
+ // Check for Enable EC policy via WEBHDFS.
+ dfs.disableErasureCodingPolicy(policy);
+ checkECPolicyState(dfs.getAllErasureCodingPolicies(), policy, "disable");
+ webHdfs.enableECPolicy("RS-10-4-1024k");
+ checkECPolicyState(dfs.getAllErasureCodingPolicies(), policy, "enable");
+
+ // Check for Disable EC policy via WEBHDFS.
+ webHdfs.disableECPolicy(policy);
+ checkECPolicyState(dfs.getAllErasureCodingPolicies(), policy, "disable");
+ }
+ }
+
@Test
public void testWebHdfsAppend() throws Exception {
MiniDFSCluster cluster = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
index 9851ede..c9247df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
@@ -506,6 +506,14 @@ public class TestParam {
}
@Test
+ public void testECPolicyParam() {
+ ECPolicyParam p = new ECPolicyParam(ECPolicyParam.DEFAULT);
+ Assert.assertEquals(null, p.getValue());
+ p = new ECPolicyParam("RS-6-3-1024k");
+ Assert.assertEquals("RS-6-3-1024k", p.getValue());
+ }
+
+ @Test
public void testHttpOpParams() {
try {
new PostOpParam("TEST");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-8881. [YARN-8851] Add basic
pluggable device plugin framework. (Zhankun Tang via wangda)
Posted by br...@apache.org.
YARN-8881. [YARN-8851] Add basic pluggable device plugin framework. (Zhankun Tang via wangda)
Change-Id: If9a2f68cd4713b4ec932cdeda68106f17437c3d3
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63578036
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63578036
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63578036
Branch: refs/heads/HDFS-13891
Commit: 63578036450f660d49ae204327efcd629d9dd137
Parents: 9366608
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Nov 19 08:54:31 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Nov 19 08:54:31 2018 -0800
----------------------------------------------------------------------
.../nodemanager/api/deviceplugin/Device.java | 233 +++++++++++++++++++
.../api/deviceplugin/DevicePlugin.java | 65 ++++++
.../api/deviceplugin/DeviceRegisterRequest.java | 73 ++++++
.../api/deviceplugin/DeviceRuntimeSpec.java | 137 +++++++++++
.../api/deviceplugin/MountDeviceSpec.java | 131 +++++++++++
.../api/deviceplugin/MountVolumeSpec.java | 118 ++++++++++
.../api/deviceplugin/VolumeSpec.java | 114 +++++++++
.../api/deviceplugin/YarnRuntimeType.java | 42 ++++
.../api/deviceplugin/package-info.java | 19 ++
.../resourceplugin/ResourcePluginManager.java | 116 ++++++++-
.../deviceframework/DevicePluginAdapter.java | 88 +++++++
.../DeviceResourceUpdaterImpl.java | 66 ++++++
.../deviceframework/package-info.java | 19 ++
.../resourceplugin/package-info.java | 19 ++
.../TestResourcePluginManager.java | 169 +++++++++++++-
.../deviceframework/FakeTestDevicePlugin1.java | 61 +++++
.../deviceframework/FakeTestDevicePlugin2.java | 27 +++
.../deviceframework/FakeTestDevicePlugin3.java | 65 ++++++
.../deviceframework/FakeTestDevicePlugin4.java | 54 +++++
.../TestDevicePluginAdapter.java | 145 ++++++++++++
.../resource-types-pluggable-devices.xml | 23 ++
21 files changed, 1773 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java
new file mode 100644
index 0000000..551c2d8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Represent one "device" resource.
+ * */
+public final class Device implements Serializable, Comparable {
+
+ private static final long serialVersionUID = -7270474563684671656L;
+
+ /**
+ * An plugin specified index number.
+ * Must set. Recommend starting from 0
+ * */
+ private final int id;
+
+ /**
+ * The device node like "/dev/devname".
+ * Optional
+ * */
+ private final String devPath;
+
+ /**
+ * The major device number.
+ * Optional
+ * */
+ private final int majorNumber;
+
+ /**
+ * The minor device number.
+ * Optional
+ * */
+ private final int minorNumber;
+
+ /**
+ * PCI Bus ID in format [[[[<domain>]:]<bus>]:][<slot>][.[<func>]].
+ * Optional. Can get from "lspci -D" in Linux
+ * */
+ private final String busID;
+
+ /**
+ * Is healthy or not.
+ * false by default
+ * */
+ private boolean isHealthy;
+
+ /**
+ * Plugin customized status info.
+ * Optional
+ * */
+ private String status;
+
+ /**
+ * Private constructor.
+ * @param builder
+ */
+ private Device(Builder builder) {
+ if (builder.id == -1) {
+ throw new IllegalArgumentException("Please set the id for Device");
+ }
+ this.id = builder.id;
+ this.devPath = builder.devPath;
+ this.majorNumber = builder.majorNumber;
+ this.minorNumber = builder.minorNumber;
+ this.busID = builder.busID;
+ this.isHealthy = builder.isHealthy;
+ this.status = builder.status;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public String getDevPath() {
+ return devPath;
+ }
+
+ public int getMajorNumber() {
+ return majorNumber;
+ }
+
+ public int getMinorNumber() {
+ return minorNumber;
+ }
+
+ public String getBusID() {
+ return busID;
+ }
+
+ public boolean isHealthy() {
+ return isHealthy;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Device device = (Device) o;
+ return id == device.getId()
+ && Objects.equals(devPath, device.getDevPath())
+ && majorNumber == device.getMajorNumber()
+ && minorNumber == device.getMinorNumber()
+ && Objects.equals(busID, device.getBusID());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, devPath, majorNumber, minorNumber, busID);
+ }
+
+ @Override
+ public int compareTo(Object o) {
+ if (o == null || (!(o instanceof Device))) {
+ return -1;
+ }
+
+ Device other = (Device) o;
+
+ int result = Integer.compare(id, other.getId());
+ if (0 != result) {
+ return result;
+ }
+
+ result = Integer.compare(majorNumber, other.getMajorNumber());
+ if (0 != result) {
+ return result;
+ }
+
+ result = Integer.compare(minorNumber, other.getMinorNumber());
+ if (0 != result) {
+ return result;
+ }
+
+ result = devPath.compareTo(other.getDevPath());
+ if (0 != result) {
+ return result;
+ }
+
+ return busID.compareTo(other.getBusID());
+ }
+
+ @Override
+ public String toString() {
+ return "(" + getId() + ", " + getDevPath() + ", "
+ + getMajorNumber() + ":" + getMinorNumber() + ")";
+ }
+
+ /**
+ * Builder for Device.
+ * */
+ public final static class Builder {
+ // default -1 representing the value is not set
+ private int id = -1;
+ private String devPath = "";
+ private int majorNumber;
+ private int minorNumber;
+ private String busID = "";
+ private boolean isHealthy;
+ private String status = "";
+
+ public static Builder newInstance() {
+ return new Builder();
+ }
+
+ public Device build() {
+ return new Device(this);
+ }
+
+ public Builder setId(int i) {
+ this.id = i;
+ return this;
+ }
+
+ public Builder setDevPath(String dp) {
+ this.devPath = dp;
+ return this;
+ }
+
+ public Builder setMajorNumber(int maN) {
+ this.majorNumber = maN;
+ return this;
+ }
+
+ public Builder setMinorNumber(int miN) {
+ this.minorNumber = miN;
+ return this;
+ }
+
+ public Builder setBusID(String bI) {
+ this.busID = bI;
+ return this;
+ }
+
+ public Builder setHealthy(boolean healthy) {
+ isHealthy = healthy;
+ return this;
+ }
+
+ public Builder setStatus(String s) {
+ this.status = s;
+ return this;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePlugin.java
new file mode 100644
index 0000000..5b7a569
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePlugin.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
+
+import java.util.Set;
+
+/**
+ * A must interface for vendor plugin to implement.
+ * */
+public interface DevicePlugin {
+ /**
+ * Called first when device plugin framework wants to register.
+ * @return DeviceRegisterRequest {@link DeviceRegisterRequest}
+ * */
+ DeviceRegisterRequest getRegisterRequestInfo()
+ throws Exception;
+
+ /**
+ * Called when update node resource.
+ * @return a set of {@link Device}, {@link java.util.TreeSet} recommended
+ * */
+ Set<Device> getDevices() throws Exception;
+
+ /**
+ * Asking how these devices should be prepared/used
+ * before/when container launch. A plugin can do some tasks in its own or
+ * define it in DeviceRuntimeSpec to let the framework do it.
+ * For instance, define {@code VolumeSpec} to let the
+ * framework to create volume before running container.
+ *
+ * @param allocatedDevices A set of allocated {@link Device}.
+ * @param yarnRuntime Indicate which runtime YARN will use
+ * Could be {@code RUNTIME_DEFAULT} or {@code RUNTIME_DOCKER}
+ * in {@link DeviceRuntimeSpec} constants. The default means YARN's
+ * non-docker container runtime is used. The docker means YARN's
+ * docker container runtime is used.
+ * @return a {@link DeviceRuntimeSpec} description about environment,
+ * {@link VolumeSpec}, {@link MountVolumeSpec}. etc
+ * */
+ DeviceRuntimeSpec onDevicesAllocated(Set<Device> allocatedDevices,
+ YarnRuntimeType yarnRuntime) throws Exception;
+
+ /**
+ * Called after device released.
+ * @param releasedDevices A set of released devices
+ * */
+ void onDevicesReleased(Set<Device> releasedDevices)
+ throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRegisterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRegisterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRegisterRequest.java
new file mode 100644
index 0000000..8fbff88
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRegisterRequest.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
+
+import java.util.Objects;
+
+/**
+ * Contains plugin register request info.
+ * */
+public final class DeviceRegisterRequest {
+
+ // plugin's own version
+ private final String pluginVersion;
+ private final String resourceName;
+
+ private DeviceRegisterRequest(Builder builder) {
+ this.resourceName = Objects.requireNonNull(builder.resourceName);
+ this.pluginVersion = builder.pluginVersion;
+ }
+
+ public String getResourceName() {
+ return resourceName;
+ }
+
+ public String getPluginVersion() {
+ return pluginVersion;
+ }
+
+ /**
+ * Builder class for construct {@link DeviceRegisterRequest}.
+ * */
+ public final static class Builder {
+ private String pluginVersion;
+ private String resourceName;
+
+ private Builder() {}
+
+ public static Builder newInstance() {
+ return new Builder();
+ }
+
+ public DeviceRegisterRequest build() {
+ return new DeviceRegisterRequest(this);
+ }
+
+ public Builder setResourceName(String resName) {
+ this.resourceName = resName;
+ return this;
+ }
+
+ public Builder setPluginVersion(String plVersion) {
+ this.pluginVersion = plVersion;
+ return this;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRuntimeSpec.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRuntimeSpec.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRuntimeSpec.java
new file mode 100644
index 0000000..32bd593
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRuntimeSpec.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * This is a spec used to prepare & run container.
+ * It's return value of onDeviceAllocated invoked by the framework.
+ * For preparation, if volumeSpecs is populated then the framework will
+ * create the volume before using the device
+ * When running container, the envs indicates environment variable needed.
+ * The containerRuntime indicates what Docker runtime to use.
+ * The volume & device mounts describes key isolation requirements
+ * */
+public final class DeviceRuntimeSpec implements Serializable {
+
+ private static final long serialVersionUID = 554704120015467660L;
+
+ /**
+ * The containerRuntime gives device framework a hint (not forced to).
+ * On which containerRuntime be used
+ * (if empty then default "runc" is used).
+ * For instance, it could be "nvidia" in Nvidia GPU Docker v2.
+ * The "nvidia" will be passed as a parameter to docker run
+ * with --runtime "nvidia"
+ */
+ private final String containerRuntime;
+ private final Map<String, String> envs;
+ private final Set<MountVolumeSpec> volumeMounts;
+ private final Set<MountDeviceSpec> deviceMounts;
+ private final Set<VolumeSpec> volumeSpecs;
+
+ private DeviceRuntimeSpec(Builder builder) {
+ this.containerRuntime = builder.containerRuntime;
+ this.deviceMounts = builder.deviceMounts;
+ this.envs = builder.envs;
+ this.volumeSpecs = builder.volumeSpecs;
+ this.volumeMounts = builder.volumeMounts;
+ }
+
+ public String getContainerRuntime() {
+ return containerRuntime;
+ }
+
+ public Map<String, String> getEnvs() {
+ return envs;
+ }
+
+ public Set<MountVolumeSpec> getVolumeMounts() {
+ return volumeMounts;
+ }
+
+ public Set<MountDeviceSpec> getDeviceMounts() {
+ return deviceMounts;
+ }
+
+ public Set<VolumeSpec> getVolumeSpecs() {
+ return volumeSpecs;
+ }
+ /**
+ * Builder for DeviceRuntimeSpec.
+ * */
+ public final static class Builder {
+
+ private String containerRuntime;
+ private Map<String, String> envs;
+ private Set<MountVolumeSpec> volumeMounts;
+ private Set<MountDeviceSpec> deviceMounts;
+ private Set<VolumeSpec> volumeSpecs;
+
+ private Builder() {
+ containerRuntime = "";
+ envs = new HashMap<>();
+ volumeSpecs = new TreeSet<>();
+ deviceMounts = new TreeSet<>();
+ volumeMounts = new TreeSet<>();
+ }
+
+ public static Builder newInstance() {
+ return new Builder();
+ }
+
+ public DeviceRuntimeSpec build() {
+ return new DeviceRuntimeSpec(this);
+ }
+
+ public Builder setContainerRuntime(String cRuntime) {
+ this.containerRuntime = cRuntime;
+ return this;
+ }
+
+ public Builder addVolumeSpec(VolumeSpec spec) {
+ this.volumeSpecs.add(spec);
+ return this;
+ }
+
+ public Builder addMountVolumeSpec(MountVolumeSpec spec) {
+ this.volumeMounts.add(spec);
+ return this;
+ }
+
+ public Builder addMountDeviceSpec(MountDeviceSpec spec) {
+ this.deviceMounts.add(spec);
+ return this;
+ }
+
+ public Builder addEnv(String key, String value) {
+ this.envs.put(Objects.requireNonNull(key),
+ Objects.requireNonNull(value));
+ return this;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/MountDeviceSpec.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/MountDeviceSpec.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/MountDeviceSpec.java
new file mode 100644
index 0000000..e4a88de
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/MountDeviceSpec.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Describe one device mount.
+ * */
+public final class MountDeviceSpec implements Serializable, Comparable {
+
+ private static final long serialVersionUID = -160806358136943052L;
+
+ private final String devicePathInHost;
+ private final String devicePathInContainer;
+
+ // r for only read, rw can do read and write
+ private final String devicePermission;
+
+ public final static String RO = "r";
+ public final static String RW = "rw";
+
+ private MountDeviceSpec(Builder builder) {
+ this.devicePathInContainer = builder.devicePathInContainer;
+ this.devicePathInHost = builder.devicePathInHost;
+ this.devicePermission = builder.devicePermission;
+ }
+
+ public String getDevicePathInHost() {
+ return devicePathInHost;
+ }
+
+ public String getDevicePathInContainer() {
+ return devicePathInContainer;
+ }
+
+ public String getDevicePermission() {
+ return devicePermission;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()){
+ return false;
+ }
+ MountDeviceSpec other = (MountDeviceSpec) o;
+ return Objects.equals(devicePathInHost, other.getDevicePathInHost())
+ && Objects.equals(devicePathInContainer,
+ other.getDevicePathInContainer())
+ && Objects.equals(devicePermission, other.getDevicePermission());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(devicePathInContainer,
+ devicePathInHost, devicePermission);
+ }
+
+ @Override
+ public int compareTo(Object o) {
+ if (o == null || (!(o instanceof MountDeviceSpec))) {
+ return -1;
+ }
+ MountDeviceSpec other = (MountDeviceSpec) o;
+ int result = devicePathInContainer.compareTo(
+ other.getDevicePathInContainer());
+ if (0 != result) {
+ return result;
+ }
+ result = devicePathInHost.compareTo(other.getDevicePathInHost());
+ if (0 != result) {
+ return result;
+ }
+ return devicePermission.compareTo(other.getDevicePermission());
+ }
+
+ /**
+ * Builder for MountDeviceSpec.
+ * */
+ public final static class Builder {
+ private String devicePathInHost;
+ private String devicePathInContainer;
+ private String devicePermission;
+
+ private Builder() {}
+
+ public static Builder newInstance() {
+ return new Builder();
+ }
+
+ public MountDeviceSpec build() {
+ return new MountDeviceSpec(this);
+ }
+
+ public Builder setDevicePermission(String permission) {
+ this.devicePermission = permission;
+ return this;
+ }
+
+ public Builder setDevicePathInContainer(String pathInContainer) {
+ this.devicePathInContainer = pathInContainer;
+ return this;
+ }
+
+ public Builder setDevicePathInHost(String pathInHost) {
+ this.devicePathInHost = pathInHost;
+ return this;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/MountVolumeSpec.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/MountVolumeSpec.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/MountVolumeSpec.java
new file mode 100644
index 0000000..6f6b183
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/MountVolumeSpec.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Describe one volume mount.
+ * */
+public final class MountVolumeSpec implements Serializable, Comparable {
+
+ private static final long serialVersionUID = 2479676805545997492L;
+
+ // host path or volume name
+ private final String hostPath;
+
+ // path in the container
+ private final String mountPath;
+
+ // if true, data in mountPath can only be read
+ // "-v hostPath:mountPath:ro"
+ private final Boolean isReadOnly;
+
+ public final static String READONLYOPTION = "ro";
+
+ private MountVolumeSpec(Builder builder) {
+ this.hostPath = builder.hostPath;
+ this.mountPath = builder.mountPath;
+ this.isReadOnly = builder.isReadOnly;
+ }
+
+ public String getHostPath() {
+ return hostPath;
+ }
+
+ public String getMountPath() {
+ return mountPath;
+ }
+
+ public Boolean getReadOnly() {
+ return isReadOnly;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()){
+ return false;
+ }
+ MountVolumeSpec other = (MountVolumeSpec) o;
+ return Objects.equals(hostPath, other.getHostPath())
+ && Objects.equals(mountPath, other.getMountPath())
+ && Objects.equals(isReadOnly, other.getReadOnly());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(hostPath, mountPath, isReadOnly);
+ }
+
+ @Override
+ public int compareTo(Object o) {
+ return 0;
+ }
+
+ /**
+ * Builder for MountVolumeSpec.
+ * */
+ public final static class Builder {
+ private String hostPath;
+ private String mountPath;
+ private Boolean isReadOnly;
+
+ private Builder() {}
+
+ public static Builder newInstance() {
+ return new Builder();
+ }
+
+ public MountVolumeSpec build() {
+ return new MountVolumeSpec(this);
+ }
+
+ public Builder setHostPath(String hPath) {
+ this.hostPath = hPath;
+ return this;
+ }
+
+ public Builder setMountPath(String mPath) {
+ this.mountPath = mPath;
+ return this;
+ }
+
+ public Builder setReadOnly(Boolean readOnly) {
+ isReadOnly = readOnly;
+ return this;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/VolumeSpec.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/VolumeSpec.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/VolumeSpec.java
new file mode 100644
index 0000000..0f06be7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/VolumeSpec.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Describe one volume creation or deletion.
+ * */
+public final class VolumeSpec implements Serializable, Comparable {
+
+ private static final long serialVersionUID = 3483619025106416736L;
+
+ private final String volumeDriver;
+ private final String volumeName;
+ private final String volumeOperation;
+
+ public final static String CREATE = "create";
+ public final static String DELETE = "delete";
+
+ private VolumeSpec(Builder builder) {
+ this.volumeDriver = builder.volumeDriver;
+ this.volumeName = builder.volumeName;
+ this.volumeOperation = builder.volumeOperation;
+ }
+
+ public String getVolumeDriver() {
+ return volumeDriver;
+ }
+
+ public String getVolumeName() {
+ return volumeName;
+ }
+
+ public String getVolumeOperation() {
+ return volumeOperation;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()){
+ return false;
+ }
+ VolumeSpec other = (VolumeSpec) o;
+ return Objects.equals(volumeDriver, other.getVolumeDriver())
+ && Objects.equals(volumeName, other.getVolumeName())
+ && Objects.equals(volumeOperation, other.getVolumeOperation());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(volumeDriver, volumeName, volumeOperation);
+ }
+
+ @Override
+ public int compareTo(Object o) {
+ return 0;
+ }
+
+ /**
+ * Builder for VolumeSpec.
+ * */
+ public final static class Builder {
+ private String volumeDriver;
+ private String volumeName;
+ private String volumeOperation;
+
+ private Builder(){}
+
+ public static Builder newInstance() {
+ return new Builder();
+ }
+
+ public VolumeSpec build() {
+ return new VolumeSpec(this);
+ }
+
+ public Builder setVolumeDriver(String volDriver) {
+ this.volumeDriver = volDriver;
+ return this;
+ }
+
+ public Builder setVolumeName(String volName) {
+ this.volumeName = volName;
+ return this;
+ }
+
+ public Builder setVolumeOperation(String volOperation) {
+ this.volumeOperation = volOperation;
+ return this;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/YarnRuntimeType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/YarnRuntimeType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/YarnRuntimeType.java
new file mode 100644
index 0000000..e0eb9dd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/YarnRuntimeType.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
+
+
+/**
+ * YarnRuntime parameter enum for {@link DevicePlugin}.
+ * It's passed into {@code onDevicesAllocated}.
+ * Device plugin could populate {@link DeviceRuntimeSpec}
+ * based on which YARN container runtime will use.
+ * */
+public enum YarnRuntimeType {
+
+ RUNTIME_DEFAULT("default"),
+ RUNTIME_DOCKER("docker");
+
+ private final String name;
+
+ YarnRuntimeType(String n) {
+ this.name = n;
+ }
+
+ public String getName() {
+ return name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/package-info.java
new file mode 100644
index 0000000..fd7ef24
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/package-info.java
@@ -0,0 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
index e930d96..9741b12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
@@ -18,18 +18,26 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.DevicePluginAdapter;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.lang.reflect.Method;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -50,8 +58,9 @@ public class ResourcePluginManager {
private Map<String, ResourcePlugin> configuredPlugins =
Collections.emptyMap();
+
public synchronized void initialize(Context context)
- throws YarnException {
+ throws YarnException, ClassNotFoundException {
Configuration conf = context.getConf();
Map<String, ResourcePlugin> pluginMap = new HashMap<>();
@@ -111,7 +120,7 @@ public class ResourcePluginManager {
public void initializePluggableDevicePlugins(Context context,
Configuration configuration,
Map<String, ResourcePlugin> pluginMap)
- throws YarnRuntimeException {
+ throws YarnRuntimeException, ClassNotFoundException {
LOG.info("The pluggable device framework enabled," +
"trying to load the vendor plugins");
@@ -121,6 +130,109 @@ public class ResourcePluginManager {
throw new YarnRuntimeException("Null value found in configuration: "
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
}
+
+ for (String pluginClassName : pluginClassNames) {
+ Class<?> pluginClazz = Class.forName(pluginClassName);
+ if (!DevicePlugin.class.isAssignableFrom(pluginClazz)) {
+ throw new YarnRuntimeException("Class: " + pluginClassName
+ + " not instance of " + DevicePlugin.class.getCanonicalName());
+ }
+ // sanity-check before initialization
+ checkInterfaceCompatibility(DevicePlugin.class, pluginClazz);
+
+ DevicePlugin dpInstance =
+ (DevicePlugin) ReflectionUtils.newInstance(
+ pluginClazz, configuration);
+
+ // Try to register plugin
+ // TODO: handle the plugin method timeout issue
+ DeviceRegisterRequest request = null;
+ try {
+ request = dpInstance.getRegisterRequestInfo();
+ } catch (Exception e) {
+ throw new YarnRuntimeException("Exception thrown from plugin's"
+ + " getRegisterRequestInfo:"
+ + e.getMessage());
+ }
+ String resourceName = request.getResourceName();
+ // check if someone has already registered this resource type name
+ if (pluginMap.containsKey(resourceName)) {
+ throw new YarnRuntimeException(resourceName
+ + " already registered! Please change resource type name"
+ + " or configure correct resource type name"
+ + " in resource-types.xml for "
+ + pluginClassName);
+ }
+ // check resource name is valid and configured in resource-types.xml
+ if (!isConfiguredResourceName(resourceName)) {
+ throw new YarnRuntimeException(resourceName
+ + " is not configured inside "
+ + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE
+ + " , please configure it first");
+ }
+ LOG.info("New resource type: {} registered successfully by {}",
+ resourceName,
+ pluginClassName);
+ DevicePluginAdapter pluginAdapter = new DevicePluginAdapter(
+ resourceName, dpInstance);
+ LOG.info("Adapter of {} created. Initializing..", pluginClassName);
+ try {
+ pluginAdapter.initialize(context);
+ } catch (YarnException e) {
+ throw new YarnRuntimeException("Adapter of "
+ + pluginClassName + " init failed!");
+ }
+ LOG.info("Adapter of {} init success!", pluginClassName);
+ // Store plugin as adapter instance
+ pluginMap.put(request.getResourceName(), pluginAdapter);
+ } // end for
+ }
+
+ @VisibleForTesting
+ // Check if the implemented interfaces' signature is compatible
+ public void checkInterfaceCompatibility(Class<?> expectedClass,
+ Class<?> actualClass) throws YarnRuntimeException{
+ LOG.debug("Checking implemented interface's compatibility: {}",
+ expectedClass.getSimpleName());
+ Method[] expectedDevicePluginMethods = expectedClass.getMethods();
+
+ // Check method compatibility
+ boolean found;
+ for (Method method: expectedDevicePluginMethods) {
+ found = false;
+ LOG.debug("Try to find method: {}",
+ method.getName());
+ for (Method m : actualClass.getDeclaredMethods()) {
+ if (m.getName().equals(method.getName())) {
+ LOG.debug("Method {} found in class {}",
+ actualClass.getSimpleName(),
+ m.getName());
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ LOG.error("Method {} is not found in plugin",
+ method.getName());
+ throw new YarnRuntimeException(
+ "Method " + method.getName()
+ + " is expected but not implemented in "
+ + actualClass.getCanonicalName());
+ }
+ }// end for
+ LOG.info("{} compatibility is ok.",
+ expectedClass.getSimpleName());
+ }
+
+ @VisibleForTesting
+ public boolean isConfiguredResourceName(String resourceName) {
+ // check configured
+ Map<String, ResourceInformation> configuredResourceTypes =
+ ResourceUtils.getResourceTypes();
+ if (!configuredResourceTypes.containsKey(resourceName)) {
+ return false;
+ }
+ return true;
}
public synchronized void cleanup() throws YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java
new file mode 100644
index 0000000..18a6992
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
+
+
+/**
+ * The {@link DevicePluginAdapter} will adapt existing hooks
+ * into vendor plugin's logic.
+ * It decouples the vendor plugin from YARN's device framework
+ *
+ * */
+public class DevicePluginAdapter implements ResourcePlugin {
+ private final static Log LOG = LogFactory.getLog(DevicePluginAdapter.class);
+
+ private final String resourceName;
+ private final DevicePlugin devicePlugin;
+ private DeviceResourceUpdaterImpl deviceResourceUpdater;
+
+ public DevicePluginAdapter(String name, DevicePlugin dp) {
+ resourceName = name;
+ devicePlugin = dp;
+ }
+
+ @Override
+ public void initialize(Context context) throws YarnException {
+ deviceResourceUpdater = new DeviceResourceUpdaterImpl(
+ resourceName, devicePlugin);
+ LOG.info(resourceName + " plugin adapter initialized");
+ return;
+ }
+
+ @Override
+ public ResourceHandler createResourceHandler(Context nmContext,
+ CGroupsHandler cGroupsHandler,
+ PrivilegedOperationExecutor privilegedOperationExecutor) {
+ return null;
+ }
+
+ @Override
+ public NodeResourceUpdaterPlugin getNodeResourceHandlerInstance() {
+ return deviceResourceUpdater;
+ }
+
+ @Override
+ public void cleanup() {
+
+ }
+
+ @Override
+ public DockerCommandPlugin getDockerCommandPluginInstance() {
+ return null;
+ }
+
+ @Override
+ public NMResourceInfo getNMResourceInfo() throws YarnException {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceUpdaterImpl.java
new file mode 100644
index 0000000..e5ef578
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceUpdaterImpl.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
+
+import java.util.Set;
+
+/**
+ * Hooks into NodeStatusUpdater to update resource.
+ * */
+public class DeviceResourceUpdaterImpl extends NodeResourceUpdaterPlugin {
+
+ final static Log LOG = LogFactory.getLog(DeviceResourceUpdaterImpl.class);
+
+ private String resourceName;
+ private DevicePlugin devicePlugin;
+
+ public DeviceResourceUpdaterImpl(String resourceName,
+ DevicePlugin devicePlugin) {
+ this.devicePlugin = devicePlugin;
+ this.resourceName = resourceName;
+ }
+
+ @Override
+ public void updateConfiguredResource(Resource res)
+ throws YarnException {
+ LOG.info(resourceName + " plugin update resource ");
+ Set<Device> devices = null;
+ try {
+ devices = devicePlugin.getDevices();
+ } catch (Exception e) {
+ throw new YarnException("Exception thrown from plugin's getDevices"
+ + e.getMessage());
+ }
+ if (null == devices) {
+ LOG.warn(resourceName
+ + " plugin failed to discover resource ( null value got).");
+ return;
+ }
+ res.setResourceValue(resourceName, devices.size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/package-info.java
new file mode 100644
index 0000000..c64fa95
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/package-info.java
@@ -0,0 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/package-info.java
new file mode 100644
index 0000000..b4be249
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/package-info.java
@@ -0,0 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
index eb8bf54..f623375 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.NodeManagerTestBase;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
@@ -42,7 +43,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.*;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -51,6 +55,7 @@ import org.junit.Test;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.io.File;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
@@ -64,9 +69,16 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
private YarnConfiguration conf;
+ private String tempResourceTypesFile;
+
@Before
public void setup() throws Exception {
this.conf = createNMConfig();
+ // setup resource-types.xml
+ ResourceUtils.resetResourceTypes();
+ String resourceTypesFile = "resource-types-pluggable-devices.xml";
+ this.tempResourceTypesFile =
+ TestResourceUtils.setupResourceTypes(this.conf, resourceTypesFile);
}
ResourcePluginManager stubResourcePluginmanager() {
@@ -101,6 +113,11 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
// ignore
}
}
+ // cleanup resource-types.xml
+ File dest = new File(this.tempResourceTypesFile);
+ if (dest.exists()) {
+ dest.delete();
+ }
}
private class CustomizedResourceHandler implements ResourceHandler {
@@ -153,8 +170,8 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
((NodeManager.NMContext)context).setResourcePluginManager(rpm);
- return new BaseNodeStatusUpdaterForTest(context, dispatcher, healthChecker,
- metrics, new BaseResourceTrackerForTest());
+ return new BaseNodeStatusUpdaterForTest(context, dispatcher,
+ healthChecker, metrics, new BaseResourceTrackerForTest());
}
@Override
@@ -174,7 +191,8 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
}
public class MyLCE extends LinuxContainerExecutor {
- private PrivilegedOperationExecutor poe = mock(PrivilegedOperationExecutor.class);
+ private PrivilegedOperationExecutor poe =
+ mock(PrivilegedOperationExecutor.class);
@Override
protected PrivilegedOperationExecutor getPrivilegedOperationExecutor() {
@@ -199,7 +217,8 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
* Make sure ResourcePluginManager is invoked during NM update.
*/
@Test(timeout = 30000)
- public void testNodeStatusUpdaterWithResourcePluginsEnabled() throws Exception {
+ public void testNodeStatusUpdaterWithResourcePluginsEnabled()
+ throws Exception {
final ResourcePluginManager rpm = stubResourcePluginmanager();
nm = new MyMockNM(rpm);
@@ -211,15 +230,16 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
rpm.getNameToPlugins().get("resource1")
.getNodeResourceHandlerInstance();
- verify(nodeResourceUpdaterPlugin, times(1)).updateConfiguredResource(
- any(Resource.class));
+ verify(nodeResourceUpdaterPlugin, times(1))
+ .updateConfiguredResource(any(Resource.class));
}
/*
* Make sure ResourcePluginManager is used to initialize ResourceHandlerChain
*/
@Test(timeout = 30000)
- public void testLinuxContainerExecutorWithResourcePluginsEnabled() throws Exception {
+ public void testLinuxContainerExecutorWithResourcePluginsEnabled()
+ throws Exception {
final ResourcePluginManager rpm = stubResourcePluginmanager();
final LinuxContainerExecutor lce = new MyLCE();
@@ -261,6 +281,9 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
boolean newHandlerAdded = false;
for (ResourceHandler h : ((ResourceHandlerChain) handler)
.getResourceHandlerList()) {
+ if (h instanceof DevicePluginAdapter) {
+ Assert.assertTrue(false);
+ }
if (h instanceof CustomizedResourceHandler) {
newHandlerAdded = true;
break;
@@ -320,7 +343,7 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
true);
conf.setStrings(
YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES,
- "com.cmp1.hdw1plugin");
+ FakeTestDevicePlugin1.class.getCanonicalName());
nm.init(conf);
nm.start();
verify(rpmSpy, times(1)).initialize(
@@ -332,7 +355,8 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
// Enable pluggable framework, but leave device classes un-configured
// initializePluggableDevicePlugins invoked but it should throw an exception
@Test(timeout = 30000)
- public void testInitializationWithPluggableDeviceFrameworkEnabled2() {
+ public void testInitializationWithPluggableDeviceFrameworkEnabled2()
+ throws ClassNotFoundException {
ResourcePluginManager rpm = new ResourcePluginManager();
ResourcePluginManager rpmSpy = spy(rpm);
@@ -353,4 +377,131 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
any(Context.class), any(Configuration.class), any(Map.class));
Assert.assertTrue(fail);
}
+
+ @Test(timeout = 30000)
+ public void testNormalInitializationOfPluggableDeviceClasses()
+ throws Exception {
+
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ true);
+ conf.setStrings(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES,
+ FakeTestDevicePlugin1.class.getCanonicalName());
+ nm.init(conf);
+ nm.start();
+ Map<String, ResourcePlugin> pluginMap = rpmSpy.getNameToPlugins();
+ Assert.assertEquals(1, pluginMap.size());
+ ResourcePlugin rp = pluginMap.get("cmpA.com/hdwA");
+ if (!(rp instanceof DevicePluginAdapter)) {
+ Assert.assertTrue(false);
+ }
+ verify(rpmSpy, times(1)).checkInterfaceCompatibility(
+ DevicePlugin.class, FakeTestDevicePlugin1.class);
+ }
+
+ // Fail to load a class which doesn't implement interface DevicePlugin
+ @Test(timeout = 30000)
+ public void testLoadInvalidPluggableDeviceClasses()
+ throws Exception{
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ true);
+ conf.setStrings(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES,
+ FakeTestDevicePlugin2.class.getCanonicalName());
+
+ String expectedMessage = "Class: "
+ + FakeTestDevicePlugin2.class.getCanonicalName()
+ + " not instance of " + DevicePlugin.class.getCanonicalName();
+ String actualMessage = "";
+ try {
+ nm.init(conf);
+ nm.start();
+ } catch (YarnRuntimeException e) {
+ actualMessage = e.getMessage();
+ }
+ Assert.assertEquals(expectedMessage, actualMessage);
+ }
+
+ // Fail to register duplicated resource name.
+ @Test(timeout = 30000)
+ public void testLoadDuplicateResourceNameDevicePlugin()
+ throws Exception{
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ true);
+ conf.setStrings(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES,
+ FakeTestDevicePlugin1.class.getCanonicalName() + "," +
+ FakeTestDevicePlugin3.class.getCanonicalName());
+
+ String expectedMessage = "cmpA.com/hdwA" +
+ " already registered! Please change resource type name"
+ + " or configure correct resource type name"
+ + " in resource-types.xml for "
+ + FakeTestDevicePlugin3.class.getCanonicalName();
+ String actualMessage = "";
+ try {
+ nm.init(conf);
+ nm.start();
+ } catch (YarnRuntimeException e) {
+ actualMessage = e.getMessage();
+ }
+ Assert.assertEquals(expectedMessage, actualMessage);
+ }
+
+ /**
+ * Fail a plugin due to incompatible interface implemented.
+ * It doesn't implement the "getRegisterRequestInfo"
+ */
+ @Test(timeout = 30000)
+ public void testIncompatibleDevicePlugin()
+ throws Exception {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ true);
+ conf.setStrings(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES,
+ FakeTestDevicePlugin4.class.getCanonicalName());
+
+ String expectedMessage = "Method getRegisterRequestInfo" +
+ " is expected but not implemented in "
+ + FakeTestDevicePlugin4.class.getCanonicalName();
+ String actualMessage = "";
+ try {
+ nm.init(conf);
+ nm.start();
+ } catch (YarnRuntimeException e) {
+ actualMessage = e.getMessage();
+ }
+ Assert.assertEquals(expectedMessage, actualMessage);
+ }
+
+ @Test(timeout = 30000)
+ public void testRequestedResourceNameIsConfigured()
+ throws Exception{
+ ResourcePluginManager rpm = new ResourcePluginManager();
+ String resourceName = "a.com/a";
+ Assert.assertFalse(rpm.isConfiguredResourceName(resourceName));
+ resourceName = "cmp.com/cmp";
+ Assert.assertTrue(rpm.isConfiguredResourceName(resourceName));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java
new file mode 100644
index 0000000..d708c87
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.*;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Used only for testing.
+ * A fake normal vendor plugin
+ * */
+public class FakeTestDevicePlugin1 implements DevicePlugin {
+ @Override
+ public DeviceRegisterRequest getRegisterRequestInfo() {
+ return DeviceRegisterRequest.Builder.newInstance()
+ .setResourceName("cmpA.com/hdwA").build();
+ }
+
+ @Override
+ public Set<Device> getDevices() {
+ TreeSet<Device> r = new TreeSet<>();
+ r.add(Device.Builder.newInstance()
+ .setId(0)
+ .setDevPath("/dev/hdwA0")
+ .setMajorNumber(243)
+ .setMinorNumber(0)
+ .setBusID("0000:65:00.0")
+ .setHealthy(true)
+ .build());
+ return r;
+ }
+
+ @Override
+ public DeviceRuntimeSpec onDevicesAllocated(Set<Device> allocatedDevices,
+ YarnRuntimeType yarnRuntime) throws Exception {
+ return null;
+ }
+
+ @Override
+ public void onDevicesReleased(Set<Device> allocatedDevices) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin2.java
new file mode 100644
index 0000000..8c1a61d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin2.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+/**
+ * Only used for testing.
+ * This isn't a implementation of DevicePlugin
+ * */
+public class FakeTestDevicePlugin2 {
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin3.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin3.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin3.java
new file mode 100644
index 0000000..67bf8db
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin3.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRuntimeSpec;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Only used for testing.
+ * This plugin register a same name with FakeTestDevicePlugin1
+ * */
+public class FakeTestDevicePlugin3 implements DevicePlugin {
+ @Override
+ public DeviceRegisterRequest getRegisterRequestInfo() {
+ return DeviceRegisterRequest.Builder.newInstance()
+ .setResourceName("cmpA.com/hdwA").build();
+ }
+
+ @Override
+ public Set<Device> getDevices() {
+ TreeSet<Device> r = new TreeSet<>();
+ r.add(Device.Builder.newInstance()
+ .setId(0)
+ .setDevPath("/dev/hdwA0")
+ .setMajorNumber(243)
+ .setMinorNumber(0)
+ .setBusID("0000:65:00.0")
+ .setHealthy(true)
+ .build());
+ return r;
+ }
+
+ @Override
+ public DeviceRuntimeSpec onDevicesAllocated(Set<Device> allocatedDevices,
+ YarnRuntimeType yarnRuntime) throws Exception {
+ return null;
+ }
+
+ @Override
+ public void onDevicesReleased(Set<Device> allocatedDevices) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin4.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin4.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin4.java
new file mode 100644
index 0000000..445994d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin4.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRuntimeSpec;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType;
+
+import java.util.Set;
+
+/**
+ * Implement the interface but missed getRegisterRequestInfo method.
+ * This is equivalent to implements a old version of DevicePlugin
+ */
+public abstract class FakeTestDevicePlugin4 implements DevicePlugin {
+
+ public DeviceRegisterRequest getOldRegisterRequestInfo() {
+ return null;
+ }
+
+ @Override
+ public Set<Device> getDevices() {
+ return null;
+ }
+
+ @Override
+ public DeviceRuntimeSpec onDevicesAllocated(Set<Device> allocatedDevices,
+ YarnRuntimeType yarnRuntime) throws Exception {
+ return null;
+ }
+
+ @Override
+ public void onDevicesReleased(Set<Device> releasedDevices) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java
new file mode 100644
index 0000000..c938b83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.*;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.*;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.times;
+
+/**
+ * Unit tests for DevicePluginAdapter.
+ * About interaction with vendor plugin
+ * */
+public class TestDevicePluginAdapter {
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestDevicePluginAdapter.class);
+
+ private YarnConfiguration conf;
+ private String tempResourceTypesFile;
+
+ @Before
+ public void setup() throws Exception {
+ this.conf = new YarnConfiguration();
+ // setup resource-types.xml
+ ResourceUtils.resetResourceTypes();
+ String resourceTypesFile = "resource-types-pluggable-devices.xml";
+ this.tempResourceTypesFile =
+ TestResourceUtils.setupResourceTypes(this.conf, resourceTypesFile);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ // cleanup resource-types.xml
+ File dest = new File(this.tempResourceTypesFile);
+ if (dest.exists()) {
+ dest.delete();
+ }
+ }
+
+ @Test
+ public void testDeviceResourceUpdaterImpl() throws YarnException {
+ Resource nodeResource = mock(Resource.class);
+ // Init an plugin
+ MyPlugin plugin = new MyPlugin();
+ MyPlugin spyPlugin = spy(plugin);
+ String resourceName = MyPlugin.RESOURCE_NAME;
+ // Init an adapter for the plugin
+ DevicePluginAdapter adapter = new DevicePluginAdapter(
+ resourceName,
+ spyPlugin);
+ adapter.initialize(mock(Context.class));
+ adapter.getNodeResourceHandlerInstance()
+ .updateConfiguredResource(nodeResource);
+ verify(spyPlugin, times(1)).getDevices();
+ verify(nodeResource, times(1)).setResourceValue(
+ resourceName, 3);
+ }
+
+ private class MyPlugin implements DevicePlugin {
+ private final static String RESOURCE_NAME = "cmpA.com/hdwA";
+ @Override
+ public DeviceRegisterRequest getRegisterRequestInfo() {
+ return DeviceRegisterRequest.Builder.newInstance()
+ .setResourceName(RESOURCE_NAME)
+ .setPluginVersion("v1.0").build();
+ }
+
+ @Override
+ public Set<Device> getDevices() {
+ TreeSet<Device> r = new TreeSet<>();
+ r.add(Device.Builder.newInstance()
+ .setId(0)
+ .setDevPath("/dev/hdwA0")
+ .setMajorNumber(256)
+ .setMinorNumber(0)
+ .setBusID("0000:80:00.0")
+ .setHealthy(true)
+ .build());
+ r.add(Device.Builder.newInstance()
+ .setId(1)
+ .setDevPath("/dev/hdwA1")
+ .setMajorNumber(256)
+ .setMinorNumber(0)
+ .setBusID("0000:80:01.0")
+ .setHealthy(true)
+ .build());
+ r.add(Device.Builder.newInstance()
+ .setId(2)
+ .setDevPath("/dev/hdwA2")
+ .setMajorNumber(256)
+ .setMinorNumber(0)
+ .setBusID("0000:80:02.0")
+ .setHealthy(true)
+ .build());
+ return r;
+ }
+
+ @Override
+ public DeviceRuntimeSpec onDevicesAllocated(Set<Device> allocatedDevices,
+ YarnRuntimeType yarnRuntime) throws Exception {
+ return null;
+ }
+
+ @Override
+ public void onDevicesReleased(Set<Device> releasedDevices) {
+
+ }
+ } // MyPlugin
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/resource-types-pluggable-devices.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/resource-types-pluggable-devices.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/resource-types-pluggable-devices.xml
new file mode 100644
index 0000000..42d3278
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/resource-types-pluggable-devices.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+ <name>yarn.resource-types</name>
+ <value>cmp.com/cmp,cmpA.com/hdwA</value>
+ </property>
+</configuration>
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDFS-13776. RBF: Add Storage
policies related ClientProtocol APIs. Contributed by Dibyendu Karmakar.
Posted by br...@apache.org.
HDFS-13776. RBF: Add Storage policies related ClientProtocol APIs. Contributed by Dibyendu Karmakar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/864cad23
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/864cad23
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/864cad23
Branch: refs/heads/HDFS-13891
Commit: 864cad231f12f0ea9a2e02bf4b042dda8771ae66
Parents: 7e34074
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Thu Nov 22 00:34:08 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:34:52 2018 +0530
----------------------------------------------------------------------
.../federation/router/RouterClientProtocol.java | 24 +--
.../federation/router/RouterStoragePolicy.java | 98 ++++++++++++
.../server/federation/MiniRouterDFSCluster.java | 13 ++
.../server/federation/router/TestRouterRpc.java | 57 +++++++
.../TestRouterRpcStoragePolicySatisfier.java | 149 +++++++++++++++++++
5 files changed, 325 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/864cad23/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
index 2fc5358..2c9bc13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
@@ -120,6 +120,8 @@ public class RouterClientProtocol implements ClientProtocol {
private final String superGroup;
/** Erasure coding calls. */
private final ErasureCoding erasureCoding;
+ /** StoragePolicy calls. **/
+ private final RouterStoragePolicy storagePolicy;
RouterClientProtocol(Configuration conf, RouterRpcServer rpcServer) {
this.rpcServer = rpcServer;
@@ -137,6 +139,7 @@ public class RouterClientProtocol implements ClientProtocol {
DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
this.erasureCoding = new ErasureCoding(rpcServer);
+ this.storagePolicy = new RouterStoragePolicy(rpcServer);
}
@Override
@@ -270,22 +273,12 @@ public class RouterClientProtocol implements ClientProtocol {
@Override
public void setStoragePolicy(String src, String policyName)
throws IOException {
- rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
-
- List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("setStoragePolicy",
- new Class<?>[] {String.class, String.class},
- new RemoteParam(), policyName);
- rpcClient.invokeSequential(locations, method, null, null);
+ storagePolicy.setStoragePolicy(src, policyName);
}
@Override
public BlockStoragePolicy[] getStoragePolicies() throws IOException {
- rpcServer.checkOperation(NameNode.OperationCategory.READ);
-
- RemoteMethod method = new RemoteMethod("getStoragePolicies");
- String ns = subclusterResolver.getDefaultNamespace();
- return (BlockStoragePolicy[]) rpcClient.invokeSingle(ns, method);
+ return storagePolicy.getStoragePolicies();
}
@Override
@@ -1455,13 +1448,12 @@ public class RouterClientProtocol implements ClientProtocol {
@Override
public void unsetStoragePolicy(String src) throws IOException {
- rpcServer.checkOperation(NameNode.OperationCategory.WRITE, false);
+ storagePolicy.unsetStoragePolicy(src);
}
@Override
public BlockStoragePolicy getStoragePolicy(String path) throws IOException {
- rpcServer.checkOperation(NameNode.OperationCategory.READ, false);
- return null;
+ return storagePolicy.getStoragePolicy(path);
}
@Override
@@ -1544,7 +1536,7 @@ public class RouterClientProtocol implements ClientProtocol {
@Override
public void satisfyStoragePolicy(String path) throws IOException {
- rpcServer.checkOperation(NameNode.OperationCategory.WRITE, false);
+ storagePolicy.satisfyStoragePolicy(path);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/864cad23/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java
new file mode 100644
index 0000000..7145940
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Module that implements all the RPC calls in
+ * {@link org.apache.hadoop.hdfs.protocol.ClientProtocol} related to
+ * Storage Policy in the {@link RouterRpcServer}.
+ */
+public class RouterStoragePolicy {
+
+ /** RPC server to receive client calls. */
+ private final RouterRpcServer rpcServer;
+ /** RPC clients to connect to the Namenodes. */
+ private final RouterRpcClient rpcClient;
+ /** Interface to map global name space to HDFS subcluster name spaces. */
+ private final FileSubclusterResolver subclusterResolver;
+
+ public RouterStoragePolicy(RouterRpcServer server) {
+ this.rpcServer = server;
+ this.rpcClient = this.rpcServer.getRPCClient();
+ this.subclusterResolver = this.rpcServer.getSubclusterResolver();
+ }
+
+ public void setStoragePolicy(String src, String policyName)
+ throws IOException {
+ rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
+
+ List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setStoragePolicy",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(),
+ policyName);
+ rpcClient.invokeSequential(locations, method, null, null);
+ }
+
+ public BlockStoragePolicy[] getStoragePolicies() throws IOException {
+ rpcServer.checkOperation(NameNode.OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getStoragePolicies");
+ String ns = subclusterResolver.getDefaultNamespace();
+ return (BlockStoragePolicy[]) rpcClient.invokeSingle(ns, method);
+ }
+
+ public void unsetStoragePolicy(String src) throws IOException {
+ rpcServer.checkOperation(NameNode.OperationCategory.WRITE, true);
+
+ List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("unsetStoragePolicy",
+ new Class<?>[] {String.class},
+ new RemoteParam());
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ public BlockStoragePolicy getStoragePolicy(String path)
+ throws IOException {
+ rpcServer.checkOperation(NameNode.OperationCategory.READ, true);
+
+ List<RemoteLocation> locations = rpcServer.getLocationsForPath(path, false);
+ RemoteMethod method = new RemoteMethod("getStoragePolicy",
+ new Class<?>[] {String.class},
+ new RemoteParam());
+ return (BlockStoragePolicy) rpcClient.invokeSequential(locations, method);
+ }
+
+ public void satisfyStoragePolicy(String path) throws IOException {
+ rpcServer.checkOperation(NameNode.OperationCategory.READ, true);
+
+ List<RemoteLocation> locations = rpcServer.getLocationsForPath(path, true);
+ RemoteMethod method = new RemoteMethod("satisfyStoragePolicy",
+ new Class<?>[] {String.class},
+ new RemoteParam());
+ rpcClient.invokeSequential(locations, method);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/864cad23/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
index a5693a6..2df883c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.DFSClient;
@@ -118,6 +119,8 @@ public class MiniRouterDFSCluster {
private boolean highAvailability;
/** Number of datanodes per nameservice. */
private int numDatanodesPerNameservice = 2;
+ /** Custom storage type for each datanode. */
+ private StorageType[][] storageTypes = null;
/** Mini cluster. */
private MiniDFSCluster cluster;
@@ -615,6 +618,15 @@ public class MiniRouterDFSCluster {
}
/**
+ * Set custom storage type configuration for each datanode.
+ * If storageTypes is uninitialized or passed null then
+ * StorageType.DEFAULT is used.
+ */
+ public void setStorageTypes(StorageType[][] storageTypes) {
+ this.storageTypes = storageTypes;
+ }
+
+ /**
* Set the DNs to belong to only one subcluster.
*/
public void setIndependentDNs() {
@@ -767,6 +779,7 @@ public class MiniRouterDFSCluster {
.numDataNodes(numDNs)
.nnTopology(topology)
.dataNodeConfOverlays(dnConfs)
+ .storageTypes(storageTypes)
.build();
cluster.waitActive();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/864cad23/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index 62303c1..4689fc7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -770,6 +771,62 @@ public class TestRouterRpc {
}
@Test
+ public void testProxyGetAndUnsetStoragePolicy() throws Exception {
+ String file = "/testGetStoragePolicy";
+ String nnFilePath = cluster.getNamenodeTestDirectoryForNS(ns) + file;
+ String routerFilePath = cluster.getFederatedTestDirectoryForNS(ns) + file;
+
+ createFile(routerFS, routerFilePath, 32);
+
+ // Get storage policy via router
+ BlockStoragePolicy policy = routerProtocol.getStoragePolicy(routerFilePath);
+ // Verify default policy is HOT
+ assertEquals(HdfsConstants.HOT_STORAGE_POLICY_NAME, policy.getName());
+ assertEquals(HdfsConstants.HOT_STORAGE_POLICY_ID, policy.getId());
+
+ // Get storage policies via router
+ BlockStoragePolicy[] policies = routerProtocol.getStoragePolicies();
+ BlockStoragePolicy[] nnPolicies = namenode.getClient().getStoragePolicies();
+ // Verify policie returned by router is same as policies returned by NN
+ assertArrayEquals(nnPolicies, policies);
+
+ BlockStoragePolicy newPolicy = policies[0];
+ while (newPolicy.isCopyOnCreateFile()) {
+ // Pick a non copy on create policy. Beacuse if copyOnCreateFile is set
+ // then the policy cannot be changed after file creation.
+ Random rand = new Random();
+ int randIndex = rand.nextInt(policies.length);
+ newPolicy = policies[randIndex];
+ }
+ routerProtocol.setStoragePolicy(routerFilePath, newPolicy.getName());
+
+ // Get storage policy via router
+ policy = routerProtocol.getStoragePolicy(routerFilePath);
+ // Verify default policy
+ assertEquals(newPolicy.getName(), policy.getName());
+ assertEquals(newPolicy.getId(), policy.getId());
+
+ // Verify policy via NN
+ BlockStoragePolicy nnPolicy =
+ namenode.getClient().getStoragePolicy(nnFilePath);
+ assertEquals(nnPolicy.getName(), policy.getName());
+ assertEquals(nnPolicy.getId(), policy.getId());
+
+ // Unset storage policy via router
+ routerProtocol.unsetStoragePolicy(routerFilePath);
+
+ // Get storage policy
+ policy = routerProtocol.getStoragePolicy(routerFilePath);
+ assertEquals(HdfsConstants.HOT_STORAGE_POLICY_NAME, policy.getName());
+ assertEquals(HdfsConstants.HOT_STORAGE_POLICY_ID, policy.getId());
+
+ // Verify policy via NN
+ nnPolicy = namenode.getClient().getStoragePolicy(nnFilePath);
+ assertEquals(nnPolicy.getName(), policy.getName());
+ assertEquals(nnPolicy.getId(), policy.getId());
+ }
+
+ @Test
public void testProxyGetPreferedBlockSize() throws Exception {
// Query via NN and Router and verify
http://git-wip-us.apache.org/repos/asf/hadoop/blob/864cad23/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcStoragePolicySatisfier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcStoragePolicySatisfier.java
new file mode 100644
index 0000000..fa1079a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcStoragePolicySatisfier.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics;
+import org.apache.hadoop.hdfs.server.namenode.sps.Context;
+import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
+import org.apache.hadoop.hdfs.server.sps.ExternalSPSContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test StoragePolicySatisfy through router rpc calls.
+ */
+public class TestRouterRpcStoragePolicySatisfier {
+
+ /** Federated HDFS cluster. */
+ private static MiniRouterDFSCluster cluster;
+
+ /** Client interface to the Router. */
+ private static ClientProtocol routerProtocol;
+
+ /** Filesystem interface to the Router. */
+ private static FileSystem routerFS;
+ /** Filesystem interface to the Namenode. */
+ private static FileSystem nnFS;
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+ cluster = new MiniRouterDFSCluster(false, 1);
+ // Set storage types for the cluster
+ StorageType[][] newtypes = new StorageType[][] {
+ {StorageType.ARCHIVE, StorageType.DISK}};
+ cluster.setStorageTypes(newtypes);
+
+ Configuration conf = cluster.getNamenodes().get(0).getConf();
+ conf.set(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
+ HdfsConstants.StoragePolicySatisfierMode.EXTERNAL.toString());
+ // Reduced refresh cycle to update latest datanodes.
+ conf.setLong(DFSConfigKeys.DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS,
+ 1000);
+ cluster.addNamenodeOverrides(conf);
+
+ cluster.setNumDatanodesPerNameservice(1);
+
+ // Start NNs and DNs and wait until ready
+ cluster.startCluster();
+
+ // Start routers with only an RPC service
+ Configuration routerConf = new RouterConfigBuilder()
+ .metrics()
+ .rpc()
+ .build();
+ // We decrease the DN cache times to make the test faster
+ routerConf.setTimeDuration(
+ RBFConfigKeys.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS);
+ cluster.addRouterOverrides(routerConf);
+ cluster.startRouters();
+
+ // Register and verify all NNs with all routers
+ cluster.registerNamenodes();
+ cluster.waitNamenodeRegistration();
+
+ // Create mock locations
+ cluster.installMockLocations();
+
+ // Random router for this test
+ MiniRouterDFSCluster.RouterContext rndRouter = cluster.getRandomRouter();
+
+ routerProtocol = rndRouter.getClient().getNamenode();
+ routerFS = rndRouter.getFileSystem();
+ nnFS = cluster.getNamenodes().get(0).getFileSystem();
+
+ NameNodeConnector nnc = DFSTestUtil.getNameNodeConnector(conf,
+ HdfsServerConstants.MOVER_ID_PATH, 1, false);
+
+ StoragePolicySatisfier externalSps = new StoragePolicySatisfier(conf);
+ Context externalCtxt = new ExternalSPSContext(externalSps, nnc);
+
+ externalSps.init(externalCtxt);
+ externalSps.start(HdfsConstants.StoragePolicySatisfierMode.EXTERNAL);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ cluster.shutdown();
+ }
+
+ @Test
+ public void testStoragePolicySatisfier() throws Exception {
+ final String file = "/testStoragePolicySatisfierCommand";
+ short repl = 1;
+ int size = 32;
+ DFSTestUtil.createFile(routerFS, new Path(file), size, repl, 0);
+ // Varify storage type is DISK
+ DFSTestUtil.waitExpectedStorageType(file, StorageType.DISK, 1, 20000,
+ (DistributedFileSystem) routerFS);
+ // Set storage policy as COLD
+ routerProtocol
+ .setStoragePolicy(file, HdfsConstants.COLD_STORAGE_POLICY_NAME);
+ // Verify storage policy is set properly
+ BlockStoragePolicy storagePolicy = routerProtocol.getStoragePolicy(file);
+ assertEquals(HdfsConstants.COLD_STORAGE_POLICY_NAME,
+ storagePolicy.getName());
+ // Invoke satisfy storage policy
+ routerProtocol.satisfyStoragePolicy(file);
+ // Verify storage type is ARCHIVE
+ DFSTestUtil.waitExpectedStorageType(file, StorageType.ARCHIVE, 1, 20000,
+ (DistributedFileSystem) routerFS);
+
+ // Verify storage type via NN
+ DFSTestUtil.waitExpectedStorageType(file, StorageType.ARCHIVE, 1, 20000,
+ (DistributedFileSystem) nnFS);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDDS-837. Persist originNodeId as
part of .container file in datanode. Contributed by Nanda kumar.
Posted by br...@apache.org.
HDDS-837. Persist originNodeId as part of .container file in datanode.
Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a7ca6ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a7ca6ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a7ca6ac
Branch: refs/heads/HDFS-13891
Commit: 5a7ca6ac3e9964c1fbd5bb654d39d5b8fb731701
Parents: 10cf577
Author: Nanda kumar <na...@apache.org>
Authored: Mon Nov 19 14:38:45 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Mon Nov 19 14:39:55 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../main/proto/DatanodeContainerProtocol.proto | 41 +++++++--------
.../container/common/impl/ContainerData.java | 44 +++++++++++++++--
.../common/impl/ContainerDataYaml.java | 7 ++-
.../container/common/impl/HddsDispatcher.java | 1 +
.../container/common/interfaces/Handler.java | 12 +++++
.../server/ratis/ContainerStateMachine.java | 7 ++-
.../container/keyvalue/KeyValueContainer.java | 3 +-
.../keyvalue/KeyValueContainerData.java | 52 +++-----------------
.../container/keyvalue/KeyValueHandler.java | 7 ++-
.../ozoneimpl/ContainerController.java | 9 ++--
.../DownloadAndImportReplicator.java | 2 +
.../StorageContainerDatanodeProtocol.proto | 1 +
.../common/TestKeyValueContainerData.java | 7 ++-
.../common/impl/TestContainerDataYaml.java | 4 +-
.../container/common/impl/TestContainerSet.java | 7 ++-
.../common/impl/TestHddsDispatcher.java | 20 ++++++--
.../common/interfaces/TestHandler.java | 11 ++++-
.../TestCloseContainerCommandHandler.java | 29 +++++------
.../keyvalue/TestBlockManagerImpl.java | 7 +--
.../keyvalue/TestChunkManagerImpl.java | 6 ++-
.../keyvalue/TestKeyValueBlockIterator.java | 3 +-
.../keyvalue/TestKeyValueContainer.java | 13 +++--
.../container/keyvalue/TestKeyValueHandler.java | 17 +++++--
.../keyvalue/TestTarContainerPacker.java | 4 +-
.../container/ozoneimpl/TestOzoneContainer.java | 14 +++++-
.../replication/TestReplicationSupervisor.java | 4 +-
.../test/resources/additionalfields.container | 4 +-
.../test/resources/incorrect.checksum.container | 2 +
.../src/test/resources/incorrect.container | 2 +
.../ozone/container/common/TestEndPoint.java | 23 +++++++--
.../common/TestBlockDeletingService.java | 3 +-
.../TestContainerDeletionChoosingPolicy.java | 8 ++-
.../common/impl/TestContainerPersistence.java | 3 +-
.../container/metrics/TestContainerMetrics.java | 15 ++++--
.../container/ozoneimpl/TestOzoneContainer.java | 10 +++-
36 files changed, 266 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index d37ce0f..f8f31c8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -188,6 +188,8 @@ public final class OzoneConsts {
public static final String CHUNKS_PATH = "chunksPath";
public static final String CONTAINER_DB_TYPE = "containerDBType";
public static final String CHECKSUM = "checksum";
+ public static final String ORIGIN_PIPELINE_ID = "originPipelineId";
+ public static final String ORIGIN_NODE_ID = "originNodeId";
// For OM Audit usage
public static final String VOLUME = "volume";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index ac2d277..9e94dd1 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -168,31 +168,32 @@ message ContainerCommandRequestProto {
required int64 containerID = 3;
required string datanodeUuid = 4;
+ optional string pipelineID = 5;
// One of the following command is available when the corresponding
// cmdType is set. At the protocol level we allow only
// one command in each packet.
// TODO : Upgrade to Protobuf 2.6 or later.
- optional CreateContainerRequestProto createContainer = 5;
- optional ReadContainerRequestProto readContainer = 6;
- optional UpdateContainerRequestProto updateContainer = 7;
- optional DeleteContainerRequestProto deleteContainer = 8;
- optional ListContainerRequestProto listContainer = 9;
- optional CloseContainerRequestProto closeContainer = 10;
-
- optional PutBlockRequestProto putBlock = 11;
- optional GetBlockRequestProto getBlock = 12;
- optional DeleteBlockRequestProto deleteBlock = 13;
- optional ListBlockRequestProto listBlock = 14;
-
- optional ReadChunkRequestProto readChunk = 15;
- optional WriteChunkRequestProto writeChunk = 16;
- optional DeleteChunkRequestProto deleteChunk = 17;
- optional ListChunkRequestProto listChunk = 18;
-
- optional PutSmallFileRequestProto putSmallFile = 19;
- optional GetSmallFileRequestProto getSmallFile = 20;
- optional GetCommittedBlockLengthRequestProto getCommittedBlockLength = 21;
+ optional CreateContainerRequestProto createContainer = 6;
+ optional ReadContainerRequestProto readContainer = 7;
+ optional UpdateContainerRequestProto updateContainer = 8;
+ optional DeleteContainerRequestProto deleteContainer = 9;
+ optional ListContainerRequestProto listContainer = 10;
+ optional CloseContainerRequestProto closeContainer = 11;
+
+ optional PutBlockRequestProto putBlock = 12;
+ optional GetBlockRequestProto getBlock = 13;
+ optional DeleteBlockRequestProto deleteBlock = 14;
+ optional ListBlockRequestProto listBlock = 15;
+
+ optional ReadChunkRequestProto readChunk = 16;
+ optional WriteChunkRequestProto writeChunk = 17;
+ optional DeleteChunkRequestProto deleteChunk = 18;
+ optional ListChunkRequestProto listChunk = 19;
+
+ optional PutSmallFileRequestProto putSmallFile = 20;
+ optional GetSmallFileRequestProto getSmallFile = 21;
+ optional GetCommittedBlockLengthRequestProto getCommittedBlockLength = 22;
}
message ContainerCommandResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
index 6f51517..d2fa2c8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
@@ -42,6 +42,8 @@ import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_TYPE;
import static org.apache.hadoop.ozone.OzoneConsts.LAYOUTVERSION;
import static org.apache.hadoop.ozone.OzoneConsts.MAX_SIZE;
import static org.apache.hadoop.ozone.OzoneConsts.METADATA;
+import static org.apache.hadoop.ozone.OzoneConsts.ORIGIN_NODE_ID;
+import static org.apache.hadoop.ozone.OzoneConsts.ORIGIN_PIPELINE_ID;
import static org.apache.hadoop.ozone.OzoneConsts.STATE;
/**
@@ -69,6 +71,11 @@ public abstract class ContainerData {
private final long maxSize;
+ //ID of the pipeline where this container is created
+ private String originPipelineId;
+ //ID of the datanode where this container is created
+ private String originNodeId;
+
/** parameters for read/write statistics on the container. **/
private final AtomicLong readBytes;
private final AtomicLong writeBytes;
@@ -93,17 +100,22 @@ public abstract class ContainerData {
STATE,
METADATA,
MAX_SIZE,
- CHECKSUM));
+ CHECKSUM,
+ ORIGIN_PIPELINE_ID,
+ ORIGIN_NODE_ID));
/**
* Creates a ContainerData Object, which holds metadata of the container.
* @param type - ContainerType
* @param containerId - ContainerId
* @param size - container maximum size in bytes
+ * @param originPipelineId - Pipeline Id where this container is/was created
+ * @param originNodeId - Node Id where this container is/was created
*/
- protected ContainerData(ContainerType type, long containerId, long size) {
- this(type, containerId,
- ChunkLayOutVersion.getLatestVersion().getVersion(), size);
+ protected ContainerData(ContainerType type, long containerId, long size,
+ String originPipelineId, String originNodeId) {
+ this(type, containerId, ChunkLayOutVersion.getLatestVersion().getVersion(),
+ size, originPipelineId, originNodeId);
}
/**
@@ -112,9 +124,12 @@ public abstract class ContainerData {
* @param containerId - ContainerId
* @param layOutVersion - Container layOutVersion
* @param size - Container maximum size in bytes
+ * @param originPipelineId - Pipeline Id where this container is/was created
+ * @param originNodeId - Node Id where this container is/was created
*/
protected ContainerData(ContainerType type, long containerId,
- int layOutVersion, long size) {
+ int layOutVersion, long size, String originPipelineId,
+ String originNodeId) {
Preconditions.checkNotNull(type);
this.containerType = type;
@@ -129,6 +144,8 @@ public abstract class ContainerData {
this.bytesUsed = new AtomicLong(0L);
this.keyCount = new AtomicLong(0L);
this.maxSize = size;
+ this.originPipelineId = originPipelineId;
+ this.originNodeId = originNodeId;
setChecksumTo0ByteArray();
}
@@ -418,6 +435,23 @@ public abstract class ContainerData {
return this.checksum;
}
+
+ /**
+ * Returns the origin pipeline Id of this container.
+ * @return origin node Id
+ */
+ public String getOriginPipelineId() {
+ return originPipelineId;
+ }
+
+ /**
+ * Returns the origin node Id of this container.
+ * @return origin node Id
+ */
+ public String getOriginNodeId() {
+ return originNodeId;
+ }
+
/**
* Compute the checksum for ContainerData using the specified Yaml (based
* on ContainerType) and set the checksum.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
index af705c4..2d09124 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
@@ -238,9 +238,14 @@ public final class ContainerDataYaml {
long size = (long) nodes.get(OzoneConsts.MAX_SIZE);
+ String originPipelineId = (String) nodes.get(
+ OzoneConsts.ORIGIN_PIPELINE_ID);
+ String originNodeId = (String) nodes.get(OzoneConsts.ORIGIN_NODE_ID);
+
//When a new field is added, it needs to be added here.
KeyValueContainerData kvData = new KeyValueContainerData(
- (long) nodes.get(OzoneConsts.CONTAINER_ID), lv, size);
+ (long) nodes.get(OzoneConsts.CONTAINER_ID), lv, size,
+ originPipelineId, originNodeId);
kvData.setContainerDBType((String)nodes.get(
OzoneConsts.CONTAINER_DB_TYPE));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index ec58aa5..b8669fb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -262,6 +262,7 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor {
.setCmdType(ContainerProtos.Type.CreateContainer)
.setContainerID(containerRequest.getContainerID())
.setCreateContainer(createRequest.build())
+ .setPipelineID(containerRequest.getPipelineID())
.setDatanodeUuid(containerRequest.getDatanodeUuid())
.setTraceID(containerRequest.getTraceID());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 19e61c4..ad55618 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -23,6 +23,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
@@ -52,6 +53,7 @@ public abstract class Handler {
protected final ContainerMetrics metrics;
private final StateContext context;
+ private final DatanodeDetails datanodeDetails;
protected Handler(Configuration config, StateContext context,
ContainerSet contSet, VolumeSet volumeSet,
@@ -61,6 +63,7 @@ public abstract class Handler {
this.containerSet = contSet;
this.volumeSet = volumeSet;
this.metrics = containerMetrics;
+ this.datanodeDetails = context.getParent().getDatanodeDetails();
}
public static Handler getHandlerForContainerType(
@@ -77,6 +80,13 @@ public abstract class Handler {
}
/**
+ * Returns the Id of this datanode.
+ * @return datanode Id
+ */
+ protected DatanodeDetails getDatanodeDetails() {
+ return datanodeDetails;
+ }
+ /**
* This should be called whenever there is state change. It will trigger
* an ICR to SCM.
*
@@ -101,6 +111,8 @@ public abstract class Handler {
public abstract Container importContainer(
long containerID,
long maxSize,
+ String originPipelineId,
+ String originNodeId,
FileInputStream rawContainerStream,
TarContainerPacker packer)
throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index a3b496a..24ecbb2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -281,7 +281,12 @@ public class ContainerStateMachine extends BaseStateMachine {
private ContainerCommandRequestProto getRequestProto(ByteString request)
throws InvalidProtocolBufferException {
- return ContainerCommandRequestProto.parseFrom(request);
+ // TODO: We can avoid creating new builder and set pipeline Id if
+ // the client is already sending the pipeline id, then we just have to
+ // validate the pipeline Id.
+ return ContainerCommandRequestProto.newBuilder(
+ ContainerCommandRequestProto.parseFrom(request))
+ .setPipelineID(gid.getUuid().toString()).build();
}
private ContainerCommandResponseProto dispatchCommand(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index f725d22..b46fd38 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -550,7 +550,8 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
.setUsed(containerData.getBytesUsed())
.setState(getHddsState())
.setDeleteTransactionId(containerData.getDeleteTransactionId())
- .setBlockCommitSequenceId(containerData.getBlockCommitSequenceId());
+ .setBlockCommitSequenceId(containerData.getBlockCommitSequenceId())
+ .setOriginNodeId(containerData.getOriginNodeId());
return ciBuilder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
index f114d34..2a9eedc 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
@@ -18,22 +18,17 @@
package org.apache.hadoop.ozone.container.keyvalue;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import java.util.Collections;
-import org.apache.hadoop.conf.StorageSize;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerDataProto;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.yaml.snakeyaml.nodes.Tag;
import java.io.File;
-import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
@@ -90,8 +85,10 @@ public class KeyValueContainerData extends ContainerData {
* @param id - ContainerId
* @param size - maximum size of the container in bytes
*/
- public KeyValueContainerData(long id, long size) {
- super(ContainerProtos.ContainerType.KeyValueContainer, id, size);
+ public KeyValueContainerData(long id, long size,
+ String originPipelineId, String originNodeId) {
+ super(ContainerProtos.ContainerType.KeyValueContainer, id, size,
+ originPipelineId, originNodeId);
this.numPendingDeletionBlocks = new AtomicInteger(0);
this.deleteTransactionId = 0;
}
@@ -102,9 +99,10 @@ public class KeyValueContainerData extends ContainerData {
* @param layOutVersion
* @param size - maximum size of the container in bytes
*/
- public KeyValueContainerData(long id, int layOutVersion, long size) {
+ public KeyValueContainerData(long id, int layOutVersion, long size,
+ String originPipelineId, String originNodeId) {
super(ContainerProtos.ContainerType.KeyValueContainer, id, layOutVersion,
- size);
+ size, originPipelineId, originNodeId);
this.numPendingDeletionBlocks = new AtomicInteger(0);
this.deleteTransactionId = 0;
}
@@ -275,40 +273,4 @@ public class KeyValueContainerData extends ContainerData {
return Collections.unmodifiableList(KV_YAML_FIELDS);
}
- /**
- * Constructs a KeyValueContainerData object from ProtoBuf classes.
- *
- * @param protoData - ProtoBuf Message
- * @throws IOException
- */
- @VisibleForTesting
- public static KeyValueContainerData getFromProtoBuf(
- ContainerDataProto protoData) throws IOException {
- // TODO: Add containerMaxSize to ContainerProtos.ContainerData
- StorageSize storageSize = StorageSize.parse(
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
- KeyValueContainerData data = new KeyValueContainerData(
- protoData.getContainerID(),
- (long)storageSize.getUnit().toBytes(storageSize.getValue()));
- for (int x = 0; x < protoData.getMetadataCount(); x++) {
- data.addMetadata(protoData.getMetadata(x).getKey(),
- protoData.getMetadata(x).getValue());
- }
-
- if (protoData.hasContainerPath()) {
- String metadataPath = protoData.getContainerPath()+ File.separator +
- OzoneConsts.CONTAINER_META_PATH;
- data.setMetadataPath(metadataPath);
- }
-
- if (protoData.hasState()) {
- data.setState(protoData.getState());
- }
-
- if (protoData.hasBytesUsed()) {
- data.setBytesUsed(protoData.getBytesUsed());
- }
-
- return data;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 4a5ef18..36c48c5 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -220,7 +220,8 @@ public class KeyValueHandler extends Handler {
long containerID = request.getContainerID();
KeyValueContainerData newContainerData = new KeyValueContainerData(
- containerID, maxContainerSize);
+ containerID, maxContainerSize, request.getPipelineID(),
+ getDatanodeDetails().getUuidString());
// TODO: Add support to add metadataList to ContainerData. Add metadata
// to container during creation.
KeyValueContainer newContainer = new KeyValueContainer(
@@ -772,13 +773,15 @@ public class KeyValueHandler extends Handler {
}
public Container importContainer(long containerID, long maxSize,
+ String originPipelineId,
+ String originNodeId,
FileInputStream rawContainerStream,
TarContainerPacker packer)
throws IOException {
KeyValueContainerData containerData =
new KeyValueContainerData(containerID,
- maxSize);
+ maxSize, originPipelineId, originNodeId);
KeyValueContainer container = new KeyValueContainer(containerData,
conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
index 41b74e7..1a918ed 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
@@ -105,11 +105,12 @@ public class ContainerController {
}
public Container importContainer(final ContainerType type,
- final long containerId, final long maxSize,
- final FileInputStream rawContainerStream, final TarContainerPacker packer)
+ final long containerId, final long maxSize, final String originPipelineId,
+ final String originNodeId, final FileInputStream rawContainerStream,
+ final TarContainerPacker packer)
throws IOException {
- return handlers.get(type).importContainer(
- containerId, maxSize, rawContainerStream, packer);
+ return handlers.get(type).importContainer(containerId, maxSize,
+ originPipelineId, originNodeId, rawContainerStream, packer);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
index e8f0d17..3100a68 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
@@ -83,6 +83,8 @@ public class DownloadAndImportReplicator implements ContainerReplicator {
originalContainerData.getContainerType(),
containerID,
originalContainerData.getMaxSize(),
+ originalContainerData.getOriginPipelineId(),
+ originalContainerData.getOriginNodeId(),
tempContainerTarStream,
packer);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index 9fdef7d..c5f709c 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -157,6 +157,7 @@ message ContainerReplicaProto {
optional string finalhash = 10;
optional int64 deleteTransactionId = 11;
optional uint64 blockCommitSequenceId = 12;
+ optional string originNodeId = 13;
}
message CommandStatusReportsProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
index 824b770..c6fa8d6 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
@@ -25,6 +25,7 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
+import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -43,9 +44,11 @@ public class TestKeyValueContainerData {
ContainerProtos.ContainerDataProto.State state =
ContainerProtos.ContainerDataProto.State.CLOSED;
AtomicLong val = new AtomicLong(0);
+ UUID pipelineId = UUID.randomUUID();
+ UUID datanodeId = UUID.randomUUID();
KeyValueContainerData kvData = new KeyValueContainerData(containerId,
- MAXSIZE);
+ MAXSIZE, pipelineId.toString(), datanodeId.toString());
assertEquals(containerType, kvData.getContainerType());
assertEquals(containerId, kvData.getContainerID());
@@ -83,6 +86,8 @@ public class TestKeyValueContainerData {
assertEquals(1, kvData.getWriteCount());
assertEquals(1, kvData.getKeyCount());
assertEquals(1, kvData.getNumPendingDeletionBlocks());
+ assertEquals(pipelineId.toString(), kvData.getOriginPipelineId());
+ assertEquals(datanodeId.toString(), kvData.getOriginNodeId());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
index 087d627..a52e18b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
@@ -29,6 +29,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.util.UUID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -55,7 +56,8 @@ public class TestContainerDataYaml {
String containerPath = containerID + ".container";
KeyValueContainerData keyValueContainerData = new KeyValueContainerData(
- containerID, MAXSIZE);
+ containerID, MAXSIZE, UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
keyValueContainerData.setContainerDBType("RocksDB");
keyValueContainerData.setMetadataPath(testRoot);
keyValueContainerData.setChunksPath(testRoot);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
index 7d8e438..d65d972 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
@@ -35,6 +35,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -55,7 +56,8 @@ public class TestContainerSet {
.ContainerDataProto.State.CLOSED;
KeyValueContainerData kvData = new KeyValueContainerData(containerId,
- (long) StorageUnit.GB.toBytes(5));
+ (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
kvData.setState(state);
KeyValueContainer keyValueContainer = new KeyValueContainer(kvData, new
OzoneConfiguration());
@@ -166,7 +168,8 @@ public class TestContainerSet {
ContainerSet containerSet = new ContainerSet();
for (int i=0; i<10; i++) {
KeyValueContainerData kvData = new KeyValueContainerData(i,
- (long) StorageUnit.GB.toBytes(5));
+ (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
if (i%2 == 0) {
kvData.setState(ContainerProtos.ContainerDataProto.State.CLOSED);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index 085e6f9..9940d4d 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
@@ -76,9 +77,14 @@ public class TestHddsDispatcher {
DatanodeDetails dd = randomDatanodeDetails();
ContainerSet containerSet = new ContainerSet();
VolumeSet volumeSet = new VolumeSet(dd.getUuidString(), conf);
+ DatanodeStateMachine stateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
StateContext context = Mockito.mock(StateContext.class);
+ Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(dd);
+ Mockito.when(context.getParent()).thenReturn(stateMachine);
KeyValueContainerData containerData = new KeyValueContainerData(1L,
- (long) StorageUnit.GB.toBytes(1));
+ (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(),
+ dd.getUuidString());
Container container = new KeyValueContainer(containerData, conf);
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(),
scmId.toString());
@@ -87,8 +93,8 @@ public class TestHddsDispatcher {
Map<ContainerType, Handler> handlers = Maps.newHashMap();
for (ContainerType containerType : ContainerType.values()) {
handlers.put(containerType,
- Handler.getHandlerForContainerType(
- containerType, conf, null, containerSet, volumeSet, metrics));
+ Handler.getHandlerForContainerType(containerType, conf, context,
+ containerSet, volumeSet, metrics));
}
HddsDispatcher hddsDispatcher = new HddsDispatcher(
conf, containerSet, volumeSet, handlers, context, metrics);
@@ -125,13 +131,17 @@ public class TestHddsDispatcher {
DatanodeDetails dd = randomDatanodeDetails();
ContainerSet containerSet = new ContainerSet();
VolumeSet volumeSet = new VolumeSet(dd.getUuidString(), conf);
+ DatanodeStateMachine stateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
StateContext context = Mockito.mock(StateContext.class);
+ Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(dd);
+ Mockito.when(context.getParent()).thenReturn(stateMachine);
ContainerMetrics metrics = ContainerMetrics.create(conf);
Map<ContainerType, Handler> handlers = Maps.newHashMap();
for (ContainerType containerType : ContainerType.values()) {
handlers.put(containerType,
- Handler.getHandlerForContainerType(
- containerType, conf, null, containerSet, volumeSet, metrics));
+ Handler.getHandlerForContainerType(containerType, conf, context,
+ containerSet, volumeSet, metrics));
}
HddsDispatcher hddsDispatcher = new HddsDispatcher(
conf, containerSet, volumeSet, handlers, context, metrics);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
index e763289..fb85194 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
@@ -20,10 +20,13 @@ package org.apache.hadoop.ozone.container.common.interfaces;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.junit.Assert;
@@ -54,13 +57,19 @@ public class TestHandler {
this.conf = new Configuration();
this.containerSet = Mockito.mock(ContainerSet.class);
this.volumeSet = Mockito.mock(VolumeSet.class);
+ DatanodeDetails datanodeDetails = Mockito.mock(DatanodeDetails.class);
+ DatanodeStateMachine stateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
+ StateContext context = Mockito.mock(StateContext.class);
+ Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails);
+ Mockito.when(context.getParent()).thenReturn(stateMachine);
ContainerMetrics metrics = ContainerMetrics.create(conf);
Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
for (ContainerProtos.ContainerType containerType :
ContainerProtos.ContainerType.values()) {
handlers.put(containerType,
Handler.getHandlerForContainerType(
- containerType, conf, null, containerSet, volumeSet, metrics));
+ containerType, conf, context, containerSet, volumeSet, metrics));
}
this.dispatcher = new HddsDispatcher(
conf, containerSet, volumeSet, handlers, null, metrics);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
index 0cf95b7..a6fad88 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
@@ -51,7 +51,7 @@ import java.util.UUID;
*/
public class TestCloseContainerCommandHandler {
- private static final StateContext CONTEXT = Mockito.mock(StateContext.class);
+ private final StateContext context = Mockito.mock(StateContext.class);
private static File testDir;
@@ -62,7 +62,12 @@ public class TestCloseContainerCommandHandler {
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath());
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testDir.getPath());
- return new OzoneContainer(datanodeDetails, conf, CONTEXT);
+ final DatanodeStateMachine datanodeStateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
+ Mockito.when(datanodeStateMachine.getDatanodeDetails())
+ .thenReturn(datanodeDetails);
+ Mockito.when(context.getParent()).thenReturn(datanodeStateMachine);
+ return new OzoneContainer(datanodeDetails, conf, context);
}
@@ -106,20 +111,14 @@ public class TestCloseContainerCommandHandler {
new CloseContainerCommandHandler();
final CloseContainerCommand command = new CloseContainerCommand(
containerId.getId(), pipelineID);
- final DatanodeStateMachine datanodeStateMachine = Mockito.mock(
- DatanodeStateMachine.class);
-
- Mockito.when(datanodeStateMachine.getDatanodeDetails())
- .thenReturn(datanodeDetails);
- Mockito.when(CONTEXT.getParent()).thenReturn(datanodeStateMachine);
- closeHandler.handle(command, container, CONTEXT, null);
+ closeHandler.handle(command, container, context, null);
Assert.assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED,
container.getContainerSet().getContainer(
containerId.getId()).getContainerState());
- Mockito.verify(datanodeStateMachine, Mockito.times(2)).triggerHeartbeat();
+ Mockito.verify(context.getParent(), Mockito.times(2)).triggerHeartbeat();
container.stop();
}
@@ -164,20 +163,14 @@ public class TestCloseContainerCommandHandler {
// Specify a pipeline which doesn't exist in the datanode.
final CloseContainerCommand command = new CloseContainerCommand(
containerId.getId(), PipelineID.randomId());
- final DatanodeStateMachine datanodeStateMachine = Mockito.mock(
- DatanodeStateMachine.class);
-
- Mockito.when(datanodeStateMachine.getDatanodeDetails())
- .thenReturn(datanodeDetails);
- Mockito.when(CONTEXT.getParent()).thenReturn(datanodeStateMachine);
- closeHandler.handle(command, container, CONTEXT, null);
+ closeHandler.handle(command, container, context, null);
Assert.assertEquals(ContainerProtos.ContainerDataProto.State.QUASI_CLOSED,
container.getContainerSet().getContainer(
containerId.getId()).getContainerState());
- Mockito.verify(datanodeStateMachine, Mockito.times(2)).triggerHeartbeat();
+ Mockito.verify(context.getParent(), Mockito.times(2)).triggerHeartbeat();
container.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
index e3e683c..1d580a0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
@@ -68,9 +68,9 @@ public class TestBlockManagerImpl {
@Before
public void setUp() throws Exception {
config = new OzoneConfiguration();
-
+ UUID datanodeId = UUID.randomUUID();
HddsVolume hddsVolume = new HddsVolume.Builder(folder.getRoot()
- .getAbsolutePath()).conf(config).datanodeUuid(UUID.randomUUID()
+ .getAbsolutePath()).conf(config).datanodeUuid(datanodeId
.toString()).build();
volumeSet = mock(VolumeSet.class);
@@ -80,7 +80,8 @@ public class TestBlockManagerImpl {
.thenReturn(hddsVolume);
keyValueContainerData = new KeyValueContainerData(1L,
- (long) StorageUnit.GB.toBytes(5));
+ (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(),
+ datanodeId.toString());
keyValueContainer = new KeyValueContainer(
keyValueContainerData, config);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index 3f181d1..abb9059 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -72,8 +72,9 @@ public class TestChunkManagerImpl {
@Before
public void setUp() throws Exception {
config = new OzoneConfiguration();
+ UUID datanodeId = UUID.randomUUID();
hddsVolume = new HddsVolume.Builder(folder.getRoot()
- .getAbsolutePath()).conf(config).datanodeUuid(UUID.randomUUID()
+ .getAbsolutePath()).conf(config).datanodeUuid(datanodeId
.toString()).build();
volumeSet = mock(VolumeSet.class);
@@ -83,7 +84,8 @@ public class TestChunkManagerImpl {
.thenReturn(hddsVolume);
keyValueContainerData = new KeyValueContainerData(1L,
- (long) StorageUnit.GB.toBytes(5));
+ (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(),
+ datanodeId.toString());
keyValueContainer = new KeyValueContainer(keyValueContainerData, config);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
index 5fa7b54..15d7b34 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
@@ -246,7 +246,8 @@ public class TestKeyValueBlockIterator {
normalBlocks, int deletedBlocks) throws
Exception {
containerData = new KeyValueContainerData(containerId,
- (long) StorageUnit.GB.toBytes(1));
+ (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
container = new KeyValueContainer(containerData, conf);
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), UUID
.randomUUID().toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index 770a9f7..002d375 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -81,12 +81,14 @@ public class TestKeyValueContainer {
private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy;
private KeyValueContainerData keyValueContainerData;
private KeyValueContainer keyValueContainer;
+ private UUID datanodeId;
@Before
public void setUp() throws Exception {
conf = new OzoneConfiguration();
+ datanodeId = UUID.randomUUID();
HddsVolume hddsVolume = new HddsVolume.Builder(folder.getRoot()
- .getAbsolutePath()).conf(conf).datanodeUuid(UUID.randomUUID()
+ .getAbsolutePath()).conf(conf).datanodeUuid(datanodeId
.toString()).build();
volumeSet = mock(VolumeSet.class);
@@ -95,7 +97,8 @@ public class TestKeyValueContainer {
.thenReturn(hddsVolume);
keyValueContainerData = new KeyValueContainerData(1L,
- (long) StorageUnit.GB.toBytes(5));
+ (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(),
+ datanodeId.toString());
keyValueContainer = new KeyValueContainer(
keyValueContainerData, conf);
@@ -105,7 +108,8 @@ public class TestKeyValueContainer {
@Test
public void testBlockIterator() throws Exception{
keyValueContainerData = new KeyValueContainerData(100L,
- (long) StorageUnit.GB.toBytes(1));
+ (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(),
+ datanodeId.toString());
keyValueContainer = new KeyValueContainer(
keyValueContainerData, conf);
keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
@@ -214,7 +218,8 @@ public class TestKeyValueContainer {
//create a new one
KeyValueContainerData containerData =
new KeyValueContainerData(containerId, 1,
- keyValueContainerData.getMaxSize());
+ keyValueContainerData.getMaxSize(), UUID.randomUUID().toString(),
+ datanodeId.toString());
KeyValueContainer container = new KeyValueContainer(containerData, conf);
HddsVolume containerVolume = volumeChoosingPolicy.chooseVolume(volumeSet
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 265fb18..698af35 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
@@ -32,6 +33,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
@@ -225,7 +228,14 @@ public class TestKeyValueHandler {
interval[0] = 2;
ContainerMetrics metrics = new ContainerMetrics(interval);
VolumeSet volumeSet = new VolumeSet(UUID.randomUUID().toString(), conf);
- KeyValueHandler keyValueHandler = new KeyValueHandler(conf, null, cset,
+ DatanodeDetails datanodeDetails = Mockito.mock(DatanodeDetails.class);
+ DatanodeStateMachine stateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
+ StateContext context = Mockito.mock(StateContext.class);
+ Mockito.when(stateMachine.getDatanodeDetails())
+ .thenReturn(datanodeDetails);
+ Mockito.when(context.getParent()).thenReturn(stateMachine);
+ KeyValueHandler keyValueHandler = new KeyValueHandler(conf, context, cset,
volumeSet, metrics);
assertEquals("org.apache.hadoop.ozone.container.common" +
".volume.RoundRobinVolumeChoosingPolicy",
@@ -236,7 +246,7 @@ public class TestKeyValueHandler {
conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY,
"org.apache.hadoop.ozone.container.common.impl.HddsDispatcher");
try {
- new KeyValueHandler(conf, null, cset, volumeSet, metrics);
+ new KeyValueHandler(conf, context, cset, volumeSet, metrics);
} catch (RuntimeException ex) {
GenericTestUtils.assertExceptionContains("class org.apache.hadoop" +
".ozone.container.common.impl.HddsDispatcher not org.apache" +
@@ -266,7 +276,8 @@ public class TestKeyValueHandler {
long containerID = 1234L;
Configuration conf = new Configuration();
KeyValueContainerData kvData = new KeyValueContainerData(containerID,
- (long) StorageUnit.GB.toBytes(1));
+ (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
KeyValueContainer container = new KeyValueContainer(kvData, conf);
kvData.setState(ContainerProtos.ContainerDataProto.State.INVALID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java
index 1a92ca4..9e6f653 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java
@@ -29,6 +29,7 @@ import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker;
@@ -91,7 +92,8 @@ public class TestTarContainerPacker {
Files.createDirectories(dbDir);
Files.createDirectories(dataDir);
- KeyValueContainerData containerData = new KeyValueContainerData(id, -1);
+ KeyValueContainerData containerData = new KeyValueContainerData(
+ id, -1, UUID.randomUUID().toString(), UUID.randomUUID().toString());
containerData.setChunksPath(dataDir.toString());
containerData.setMetadataPath(dbDir.getParent().toString());
containerData.setDbFile(dbDir.toFile());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index 2326818..388d0c1 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
@@ -34,6 +36,8 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.mockito.Mockito;
+
import java.util.Random;
import java.util.UUID;
@@ -79,16 +83,22 @@ public class TestOzoneContainer {
// Add containers to disk
for (int i=0; i<10; i++) {
keyValueContainerData = new KeyValueContainerData(i,
- (long) StorageUnit.GB.toBytes(1));
+ (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(),
+ datanodeDetails.getUuidString());
keyValueContainer = new KeyValueContainer(
keyValueContainerData, conf);
keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
}
+ DatanodeStateMachine stateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
+ StateContext context = Mockito.mock(StateContext.class);
+ Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails);
+ Mockito.when(context.getParent()).thenReturn(stateMachine);
// When OzoneContainer is started, the containers from disk should be
// loaded into the containerSet.
OzoneContainer ozoneContainer = new
- OzoneContainer(datanodeDetails, conf, null);
+ OzoneContainer(datanodeDetails, conf, context);
ContainerSet containerset = ozoneContainer.getContainerSet();
assertEquals(10, containerset.containerCount());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
index 030412c..c3d3b17 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.replication;
import java.util.ArrayList;
import java.util.List;
+import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -118,7 +119,8 @@ public class TestReplicationSupervisor {
@Override
public void replicate(ReplicationTask task) {
KeyValueContainerData kvcd =
- new KeyValueContainerData(task.getContainerId(), 100L);
+ new KeyValueContainerData(task.getContainerId(), 100L,
+ UUID.randomUUID().toString(), UUID.randomUUID().toString());
KeyValueContainer kvc =
new KeyValueContainer(kvcd, conf);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/resources/additionalfields.container
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/resources/additionalfields.container b/hadoop-hdds/container-service/src/test/resources/additionalfields.container
index fff5304..faaed06 100644
--- a/hadoop-hdds/container-service/src/test/resources/additionalfields.container
+++ b/hadoop-hdds/container-service/src/test/resources/additionalfields.container
@@ -6,7 +6,9 @@ containerType: KeyValueContainer
metadataPath: /hdds/current/aed-fg4-hji-jkl/containerDir0/1
layOutVersion: 1
maxSize: 5368709120
+originPipelineId: 1297e8a9-2850-4ced-b96c-5ae31d2c73ad
+originNodeId: 7f541a06-6c26-476d-9994-c6e1947e11cb
metadata: {OWNER: ozone, VOLUME: hdfs}
state: CLOSED
aclEnabled: true
-checksum: c5b5373b8755c4e7199478dcaded9d996f9aca089704e08950259cdb0f290680
\ No newline at end of file
+checksum: 61db56da7d50798561b5365c123c5fbf7faf99fbbbd571a746af79020b7f79ba
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container b/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container
index d06ba57..ce32947 100644
--- a/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container
+++ b/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container
@@ -6,6 +6,8 @@ containerType: KeyValueContainer
metadataPath: /hdds/current/aed-fg4-hji-jkl/containerdir0/1
layOutVersion: 1
maxSize: 5368709120
+originPipelineId: 4d41dd20-6d73-496a-b247-4c6cb483f54e
+originNodeId: 54842560-67a5-48a5-a7d4-4701d9538706
metadata: {OWNER: ozone, VOLUME: hdfs}
state: OPEN
checksum: 08bc9d390f9183aeed3cf33c789e2a07310bba60f3cf55941caccc939db8670f
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/container-service/src/test/resources/incorrect.container
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/resources/incorrect.container b/hadoop-hdds/container-service/src/test/resources/incorrect.container
index 0053ab2..d14c75f 100644
--- a/hadoop-hdds/container-service/src/test/resources/incorrect.container
+++ b/hadoop-hdds/container-service/src/test/resources/incorrect.container
@@ -6,6 +6,8 @@ containerType: KeyValueContainer
metadataPath: /hdds/current/aed-fg4-hji-jkl/containerDir0/1
layOutVersion: 1
maxSize: 5368709120
+originPipelineId: b2c96aa4-b757-4f97-b286-6fb80a1baf8e
+originNodeId: 6dcfb385-caea-4efb-9ef3-f87fadca0f51
metadata: {OWNER: ozone, VOLUME: hdfs}
state: INVALID
checksum: 08bc9d390f9183aeed3cf33c789e2a07310bba60f3cf55941caccc939db8670f
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index 16c9f22..600ee5a 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -154,8 +154,9 @@ public class TestEndPoint {
OzoneConfiguration conf = SCMTestUtils.getConf();
try (EndpointStateMachine rpcEndPoint = createEndpoint(conf,
serverAddress, 1000)) {
+ DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
OzoneContainer ozoneContainer = new OzoneContainer(
- TestUtils.randomDatanodeDetails(), conf, null);
+ datanodeDetails, conf, getContext(datanodeDetails));
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
@@ -178,8 +179,9 @@ public class TestEndPoint {
serverAddress, 1000)) {
GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
.captureLogs(VersionEndpointTask.LOG);
- OzoneContainer ozoneContainer = new OzoneContainer(TestUtils
- .randomDatanodeDetails(), conf, null);
+ DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
+ OzoneContainer ozoneContainer = new OzoneContainer(
+ datanodeDetails, conf, getContext(datanodeDetails));
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
@@ -231,8 +233,9 @@ public class TestEndPoint {
try (EndpointStateMachine rpcEndPoint = createEndpoint(conf,
nonExistentServerAddress, 1000)) {
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
+ DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
OzoneContainer ozoneContainer = new OzoneContainer(
- TestUtils.randomDatanodeDetails(), conf, null);
+ datanodeDetails, conf, getContext(datanodeDetails));
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
EndpointStateMachine.EndPointStates newState = versionTask.call();
@@ -258,8 +261,9 @@ public class TestEndPoint {
try (EndpointStateMachine rpcEndPoint = createEndpoint(conf,
serverAddress, (int) rpcTimeout)) {
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
+ DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
OzoneContainer ozoneContainer = new OzoneContainer(
- TestUtils.randomDatanodeDetails(), conf, null);
+ datanodeDetails, conf, getContext(datanodeDetails));
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
@@ -527,4 +531,13 @@ public class TestEndPoint {
lessThanOrEqualTo(rpcTimeout + tolerance));
}
+ private StateContext getContext(DatanodeDetails datanodeDetails) {
+ DatanodeStateMachine stateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
+ StateContext context = Mockito.mock(StateContext.class);
+ Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails);
+ Mockito.when(context.getParent()).thenReturn(stateMachine);
+ return context;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 7e30c5f..c2fb2ea 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -110,7 +110,8 @@ public class TestBlockDeletingService {
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testRoot.getAbsolutePath());
long containerID = ContainerTestHelper.getTestContainerID();
KeyValueContainerData data = new KeyValueContainerData(containerID,
- ContainerTestHelper.CONTAINER_MAX_SIZE);
+ ContainerTestHelper.CONTAINER_MAX_SIZE, UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
Container container = new KeyValueContainer(data, conf);
container.create(new VolumeSet(scmId, clusterID, conf),
new RoundRobinVolumeChoosingPolicy(), scmId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
index 6478fc6..745f730 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
@@ -24,6 +24,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.UUID;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomUtils;
@@ -72,7 +73,8 @@ public class TestContainerDeletionChoosingPolicy {
int numContainers = 10;
for (int i = 0; i < numContainers; i++) {
KeyValueContainerData data = new KeyValueContainerData(i,
- ContainerTestHelper.CONTAINER_MAX_SIZE);
+ ContainerTestHelper.CONTAINER_MAX_SIZE, UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
KeyValueContainer container = new KeyValueContainer(data, conf);
containerSet.addContainer(container);
Assert.assertTrue(
@@ -125,7 +127,9 @@ public class TestContainerDeletionChoosingPolicy {
long containerId = RandomUtils.nextLong();
KeyValueContainerData data =
new KeyValueContainerData(containerId,
- ContainerTestHelper.CONTAINER_MAX_SIZE);
+ ContainerTestHelper.CONTAINER_MAX_SIZE,
+ UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
if (i != numContainers) {
int deletionBlocks = random.nextInt(numContainers) + 1;
data.incrPendingDeletionBlocks(deletionBlocks);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index f6c498a..a48d637 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -157,7 +157,8 @@ public class TestContainerPersistence {
private Container addContainer(ContainerSet cSet, long cID)
throws IOException {
KeyValueContainerData data = new KeyValueContainerData(cID,
- ContainerTestHelper.CONTAINER_MAX_SIZE);
+ ContainerTestHelper.CONTAINER_MAX_SIZE, UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
KeyValueContainer container = new KeyValueContainer(data, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
index 527ab45..44c1172 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
@@ -41,6 +41,8 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.hdds.scm.TestUtils;
@@ -51,6 +53,7 @@ import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicatio
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mockito;
import java.io.File;
import java.util.Map;
@@ -90,16 +93,22 @@ public class TestContainerMetrics {
VolumeSet volumeSet = new VolumeSet(
datanodeDetails.getUuidString(), conf);
ContainerSet containerSet = new ContainerSet();
+ DatanodeStateMachine stateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
+ StateContext context = Mockito.mock(StateContext.class);
+ Mockito.when(stateMachine.getDatanodeDetails())
+ .thenReturn(datanodeDetails);
+ Mockito.when(context.getParent()).thenReturn(stateMachine);
ContainerMetrics metrics = ContainerMetrics.create(conf);
Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
for (ContainerProtos.ContainerType containerType :
ContainerProtos.ContainerType.values()) {
handlers.put(containerType,
- Handler.getHandlerForContainerType(
- containerType, conf, null, containerSet, volumeSet, metrics));
+ Handler.getHandlerForContainerType(containerType, conf, context,
+ containerSet, volumeSet, metrics));
}
HddsDispatcher dispatcher = new HddsDispatcher(conf, containerSet,
- volumeSet, handlers, null, metrics);
+ volumeSet, handlers, context, metrics);
dispatcher.setScmId(UUID.randomUUID().toString());
server = new XceiverServerGrpc(datanodeDetails, conf, dispatcher,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a7ca6ac/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index 3a15b21..cd0b060 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -29,11 +29,14 @@ import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.Timeout;
+import org.mockito.Mockito;
import java.util.*;
import java.util.concurrent.CompletableFuture;
@@ -72,8 +75,13 @@ public class TestOzoneContainer {
conf.setBoolean(
OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
+ DatanodeDetails datanodeDetails = Mockito.mock(DatanodeDetails.class);
+ StateContext context = Mockito.mock(StateContext.class);
+ DatanodeStateMachine dsm = Mockito.mock(DatanodeStateMachine.class);
+ Mockito.when(dsm.getDatanodeDetails()).thenReturn(datanodeDetails);
+ Mockito.when(context.getParent()).thenReturn(dsm);
container = new OzoneContainer(TestUtils.randomDatanodeDetails(),
- conf, null);
+ conf, context);
//Setting scmId, as we start manually ozone container.
container.getDispatcher().setScmId(UUID.randomUUID().toString());
container.start();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HDDS-120. Adding HDDS datanode
Audit Log. Contributed by Dinesh Chitlangia.
Posted by br...@apache.org.
HDDS-120. Adding HDDS datanode Audit Log. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29374999
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29374999
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29374999
Branch: refs/heads/HDFS-13891
Commit: 29374999b63f0beca17a4f3dc0a0ed1f3a64316a
Parents: abd6d48
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Nov 16 14:59:59 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Fri Nov 16 14:59:59 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/audit/DNAction.java | 54 ++++++
.../container/common/helpers/BlockData.java | 10 +
.../ContainerCommandRequestPBHelper.java | 192 +++++++++++++++++++
.../container/common/impl/HddsDispatcher.java | 133 ++++++++++++-
hadoop-ozone/common/src/main/bin/ozone | 2 +
.../dist/dev-support/bin/dist-layout-stitching | 1 +
.../src/main/conf/dn-audit-log4j2.properties | 90 +++++++++
7 files changed, 473 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29374999/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
new file mode 100644
index 0000000..ce34c46
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.audit;
+
+/**
+ * Enum to define Audit Action types for Datanode.
+ */
+public enum DNAction implements AuditAction {
+
+ CREATE_CONTAINER("CREATE_CONTAINER"),
+ READ_CONTAINER("READ_CONTAINER"),
+ UPDATE_CONTAINER("UPDATE_CONTAINER"),
+ DELETE_CONTAINER("DELETE_CONTAINER"),
+ LIST_CONTAINER("LIST_CONTAINER"),
+ PUT_BLOCK("PUT_BLOCK"),
+ GET_BLOCK("GET_BLOCK"),
+ DELETE_BLOCK("DELETE_BLOCK"),
+ LIST_BLOCK("LIST_BLOCK"),
+ READ_CHUNK("READ_CHUNK"),
+ DELETE_CHUNK("DELETE_CHUNK"),
+ WRITE_CHUNK("WRITE_CHUNK"),
+ LIST_CHUNK("LIST_CHUNK"),
+ COMPACT_CHUNK("COMPACT_CHUNK"),
+ PUT_SMALL_FILE("PUT_SMALL_FILE"),
+ GET_SMALL_FILE("GET_SMALL_FILE"),
+ CLOSE_CONTAINER("CLOSE_CONTAINER"),
+ GET_COMMITTED_BLOCK_LENGTH("GET_COMMITTED_BLOCK_LENGTH");
+
+ private String action;
+
+ DNAction(String action) {
+ this.action = action;
+ }
+
+ @Override
+ public String getAction() {
+ return this.action;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29374999/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
index 3b9e57c..e0cac8b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.ozone.container.common.helpers;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.client.BlockID;
import com.google.common.base.Preconditions;
@@ -260,4 +262,12 @@ public class BlockData {
public long getSize() {
return size;
}
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this, ToStringStyle.NO_CLASS_NAME_STYLE)
+ .append("blockId", blockID.toString())
+ .append("size", this.size)
+ .toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29374999/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerCommandRequestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerCommandRequestPBHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerCommandRequestPBHelper.java
new file mode 100644
index 0000000..0b7ae2d
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerCommandRequestPBHelper.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.helpers;
+
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
+ ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
+import org.apache.hadoop.ozone.audit.DNAction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Utilities for converting protobuf classes to Java classes.
+ */
+public final class ContainerCommandRequestPBHelper {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(ContainerCommandRequestPBHelper.class);
+
+ private ContainerCommandRequestPBHelper() {
+ }
+
+ public static Map<String, String> getAuditParams(
+ ContainerCommandRequestProto msg) {
+ Map<String, String> auditParams = new TreeMap<>();
+ Type cmdType = msg.getCmdType();
+ String containerID = String.valueOf(msg.getContainerID());
+ switch(cmdType) {
+ case CreateContainer:
+ auditParams.put("containerID", containerID);
+ auditParams.put("containerType",
+ msg.getCreateContainer().getContainerType().toString());
+ return auditParams;
+
+ case ReadContainer:
+ auditParams.put("containerID", containerID);
+ return auditParams;
+
+ case UpdateContainer:
+ auditParams.put("containerID", containerID);
+ auditParams.put("forceUpdate",
+ String.valueOf(msg.getUpdateContainer().getForceUpdate()));
+ return auditParams;
+
+ case DeleteContainer:
+ auditParams.put("containerID", containerID);
+ auditParams.put("forceDelete",
+ String.valueOf(msg.getDeleteContainer().getForceDelete()));
+ return auditParams;
+
+ case ListContainer:
+ auditParams.put("startContainerID", containerID);
+ auditParams.put("count",
+ String.valueOf(msg.getListContainer().getCount()));
+ return auditParams;
+
+ case PutBlock:
+ try{
+ auditParams.put("blockData",
+ BlockData.getFromProtoBuf(msg.getPutBlock().getBlockData())
+ .toString());
+ }catch (IOException ex){
+ LOG.trace("Encountered error parsing BlockData from protobuf:"
+ + ex.getMessage());
+ return null;
+ }
+ return auditParams;
+
+ case GetBlock:
+ auditParams.put("blockData",
+ BlockID.getFromProtobuf(msg.getGetBlock().getBlockID()).toString());
+ return auditParams;
+
+ case DeleteBlock:
+ auditParams.put("blockData",
+ BlockID.getFromProtobuf(msg.getDeleteBlock().getBlockID())
+ .toString());
+ return auditParams;
+
+ case ListBlock:
+ auditParams.put("startLocalID",
+ String.valueOf(msg.getListBlock().getStartLocalID()));
+ auditParams.put("count", String.valueOf(msg.getListBlock().getCount()));
+ return auditParams;
+
+ case ReadChunk:
+ auditParams.put("blockData",
+ BlockID.getFromProtobuf(msg.getReadChunk().getBlockID()).toString());
+ return auditParams;
+
+ case DeleteChunk:
+ auditParams.put("blockData",
+ BlockID.getFromProtobuf(msg.getDeleteChunk().getBlockID())
+ .toString());
+ return auditParams;
+
+ case WriteChunk:
+ auditParams.put("blockData",
+ BlockID.getFromProtobuf(msg.getWriteChunk().getBlockID())
+ .toString());
+ return auditParams;
+
+ case ListChunk:
+ auditParams.put("blockData",
+ BlockID.getFromProtobuf(msg.getListChunk().getBlockID()).toString());
+ auditParams.put("prevChunkName", msg.getListChunk().getPrevChunkName());
+ auditParams.put("count", String.valueOf(msg.getListChunk().getCount()));
+ return auditParams;
+
+ case CompactChunk: return null; //CompactChunk operation
+
+ case PutSmallFile:
+ try{
+ auditParams.put("blockData",
+ BlockData.getFromProtoBuf(msg.getPutSmallFile()
+ .getBlock().getBlockData()).toString());
+ }catch (IOException ex){
+ LOG.trace("Encountered error parsing BlockData from protobuf:"
+ + ex.getMessage());
+ }
+ return auditParams;
+
+ case GetSmallFile:
+ auditParams.put("blockData",
+ BlockID.getFromProtobuf(msg.getGetSmallFile().getBlock().getBlockID())
+ .toString());
+ return auditParams;
+
+ case CloseContainer:
+ auditParams.put("containerID", containerID);
+ return auditParams;
+
+ case GetCommittedBlockLength:
+ auditParams.put("blockData",
+ BlockID.getFromProtobuf(msg.getGetCommittedBlockLength().getBlockID())
+ .toString());
+ return auditParams;
+
+ default :
+ LOG.debug("Invalid command type - " + cmdType);
+ return null;
+ }
+
+ }
+
+ public static DNAction getAuditAction(Type cmdType) {
+ switch (cmdType) {
+ case CreateContainer : return DNAction.CREATE_CONTAINER;
+ case ReadContainer : return DNAction.READ_CONTAINER;
+ case UpdateContainer : return DNAction.UPDATE_CONTAINER;
+ case DeleteContainer : return DNAction.DELETE_CONTAINER;
+ case ListContainer : return DNAction.LIST_CONTAINER;
+ case PutBlock : return DNAction.PUT_BLOCK;
+ case GetBlock : return DNAction.GET_BLOCK;
+ case DeleteBlock : return DNAction.DELETE_BLOCK;
+ case ListBlock : return DNAction.LIST_BLOCK;
+ case ReadChunk : return DNAction.READ_CHUNK;
+ case DeleteChunk : return DNAction.DELETE_CHUNK;
+ case WriteChunk : return DNAction.WRITE_CHUNK;
+ case ListChunk : return DNAction.LIST_CHUNK;
+ case CompactChunk : return DNAction.COMPACT_CHUNK;
+ case PutSmallFile : return DNAction.PUT_SMALL_FILE;
+ case GetSmallFile : return DNAction.GET_SMALL_FILE;
+ case CloseContainer : return DNAction.CLOSE_CONTAINER;
+ case GetCommittedBlockLength : return DNAction.GET_COMMITTED_BLOCK_LENGTH;
+ default :
+ LOG.debug("Invalid command type - " + cmdType);
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29374999/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 29f9b20..ec58aa5 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -27,9 +27,21 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerDataProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerAction;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
-import org.apache.hadoop.hdds.scm.container.common.helpers.InvalidContainerStateException;
-import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.container.common.helpers
+ .ContainerNotOpenException;
+import org.apache.hadoop.hdds.scm.container.common.helpers
+ .InvalidContainerStateException;
+import org.apache.hadoop.hdds.scm.container.common.helpers
+ .StorageContainerException;
+import org.apache.hadoop.ozone.audit.AuditAction;
+import org.apache.hadoop.ozone.audit.AuditEventStatus;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.AuditLoggerType;
+import org.apache.hadoop.ozone.audit.AuditMarker;
+import org.apache.hadoop.ozone.audit.AuditMessage;
+import org.apache.hadoop.ozone.audit.Auditor;
+import org.apache.hadoop.ozone.container.common.helpers
+ .ContainerCommandRequestPBHelper;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
@@ -57,10 +69,11 @@ import java.util.Optional;
* Ozone Container dispatcher takes a call from the netty server and routes it
* to the right handler function.
*/
-public class HddsDispatcher implements ContainerDispatcher {
+public class HddsDispatcher implements ContainerDispatcher, Auditor {
static final Logger LOG = LoggerFactory.getLogger(HddsDispatcher.class);
-
+ private static final AuditLogger AUDIT =
+ new AuditLogger(AuditLoggerType.DNLOGGER);
private final Map<ContainerType, Handler> handlers;
private final Configuration conf;
private final ContainerSet containerSet;
@@ -123,7 +136,13 @@ public class HddsDispatcher implements ContainerDispatcher {
ContainerCommandRequestProto msg) {
LOG.trace("Command {}, trace ID: {} ", msg.getCmdType().toString(),
msg.getTraceID());
- Preconditions.checkNotNull(msg);
+ Preconditions.checkNotNull(msg.toString());
+
+ AuditAction action = ContainerCommandRequestPBHelper.getAuditAction(
+ msg.getCmdType());
+ EventType eventType = getEventType(msg);
+ Map<String, String> params =
+ ContainerCommandRequestPBHelper.getAuditParams(msg);
Container container = null;
ContainerType containerType = null;
@@ -149,11 +168,14 @@ public class HddsDispatcher implements ContainerDispatcher {
StorageContainerException sce = new StorageContainerException(
"ContainerID " + containerID + " does not exist",
ContainerProtos.Result.CONTAINER_NOT_FOUND);
+ audit(action, eventType, params, AuditEventStatus.FAILURE, sce);
return ContainerUtils.logAndReturnError(LOG, sce, msg);
}
containerType = getContainerType(container);
} else {
if (!msg.hasCreateContainer()) {
+ audit(action, eventType, params, AuditEventStatus.FAILURE,
+ new Exception("MALFORMED_REQUEST"));
return ContainerUtils.malformedRequest(msg);
}
containerType = msg.getCreateContainer().getContainerType();
@@ -168,6 +190,8 @@ public class HddsDispatcher implements ContainerDispatcher {
StorageContainerException ex = new StorageContainerException("Invalid " +
"ContainerType " + containerType,
ContainerProtos.Result.CONTAINER_INTERNAL_ERROR);
+ // log failure
+ audit(action, eventType, params, AuditEventStatus.FAILURE, ex);
return ContainerUtils.logAndReturnError(LOG, ex, msg);
}
responseProto = handler.handle(msg, container);
@@ -204,8 +228,19 @@ public class HddsDispatcher implements ContainerDispatcher {
.setState(ContainerDataProto.State.UNHEALTHY);
sendCloseContainerActionIfNeeded(container);
}
+
+ if(result == Result.SUCCESS) {
+ audit(action, eventType, params, AuditEventStatus.SUCCESS, null);
+ } else {
+ audit(action, eventType, params, AuditEventStatus.FAILURE,
+ new Exception(responseProto.getMessage()));
+ }
+
return responseProto;
} else {
+ // log failure
+ audit(action, eventType, params, AuditEventStatus.FAILURE,
+ new Exception("UNSUPPORTED_REQUEST"));
return ContainerUtils.unsupportedRequest(msg);
}
}
@@ -249,17 +284,24 @@ public class HddsDispatcher implements ContainerDispatcher {
public void validateContainerCommand(
ContainerCommandRequestProto msg) throws StorageContainerException {
ContainerType containerType = msg.getCreateContainer().getContainerType();
+ ContainerProtos.Type cmdType = msg.getCmdType();
+ AuditAction action =
+ ContainerCommandRequestPBHelper.getAuditAction(cmdType);
+ EventType eventType = getEventType(msg);
+ Map<String, String> params =
+ ContainerCommandRequestPBHelper.getAuditParams(msg);
Handler handler = getHandler(containerType);
if (handler == null) {
StorageContainerException ex = new StorageContainerException(
"Invalid " + "ContainerType " + containerType,
ContainerProtos.Result.CONTAINER_INTERNAL_ERROR);
+ audit(action, eventType, params, AuditEventStatus.FAILURE, ex);
throw ex;
}
- ContainerProtos.Type cmdType = msg.getCmdType();
long containerID = msg.getContainerID();
Container container;
container = getContainer(containerID);
+
if (container != null) {
State containerState = container.getContainerState();
if (!HddsUtils.isReadOnly(msg) && containerState != State.OPEN) {
@@ -274,12 +316,16 @@ public class HddsDispatcher implements ContainerDispatcher {
default:
// if the container is not open, no updates can happen. Just throw
// an exception
- throw new ContainerNotOpenException(
+ ContainerNotOpenException cex = new ContainerNotOpenException(
"Container " + containerID + " in " + containerState + " state");
+ audit(action, eventType, params, AuditEventStatus.FAILURE, cex);
+ throw cex;
}
} else if (HddsUtils.isReadOnly(msg) && containerState == State.INVALID) {
- throw new InvalidContainerStateException(
+ InvalidContainerStateException iex = new InvalidContainerStateException(
"Container " + containerID + " in " + containerState + " state");
+ audit(action, eventType, params, AuditEventStatus.FAILURE, iex);
+ throw iex;
}
}
}
@@ -355,4 +401,73 @@ public class HddsDispatcher implements ContainerDispatcher {
public void setMetricsForTesting(ContainerMetrics containerMetrics) {
this.metrics = containerMetrics;
}
+
+ private EventType getEventType(ContainerCommandRequestProto msg) {
+ return HddsUtils.isReadOnly(msg) ? EventType.READ : EventType.WRITE;
+ }
+
+ private void audit(AuditAction action, EventType eventType,
+ Map<String, String> params, AuditEventStatus result, Throwable exception){
+ AuditMessage amsg;
+ switch (result) {
+ case SUCCESS:
+ if(eventType == EventType.READ &&
+ AUDIT.getLogger().isInfoEnabled(AuditMarker.READ.getMarker())) {
+ amsg = buildAuditMessageForSuccess(action, params);
+ AUDIT.logReadSuccess(amsg);
+ } else if(eventType == EventType.WRITE &&
+ AUDIT.getLogger().isInfoEnabled(AuditMarker.WRITE.getMarker())) {
+ amsg = buildAuditMessageForSuccess(action, params);
+ AUDIT.logWriteSuccess(amsg);
+ }
+ break;
+
+ case FAILURE:
+ if(eventType == EventType.READ &&
+ AUDIT.getLogger().isErrorEnabled(AuditMarker.READ.getMarker())) {
+ amsg = buildAuditMessageForFailure(action, params, exception);
+ AUDIT.logReadFailure(amsg);
+ } else if(eventType == EventType.WRITE &&
+ AUDIT.getLogger().isErrorEnabled(AuditMarker.WRITE.getMarker())) {
+ amsg = buildAuditMessageForFailure(action, params, exception);
+ AUDIT.logWriteFailure(amsg);
+ }
+ break;
+
+ default: LOG.debug("Invalid audit event status - " + result);
+ }
+ }
+
+ //TODO: use GRPC to fetch user and ip details
+ @Override
+ public AuditMessage buildAuditMessageForSuccess(AuditAction op,
+ Map<String, String> auditMap) {
+ return new AuditMessage.Builder()
+ .setUser(null)
+ .atIp(null)
+ .forOperation(op.getAction())
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.SUCCESS.toString())
+ .withException(null)
+ .build();
+ }
+
+ //TODO: use GRPC to fetch user and ip details
+ @Override
+ public AuditMessage buildAuditMessageForFailure(AuditAction op,
+ Map<String, String> auditMap, Throwable throwable) {
+ return new AuditMessage.Builder()
+ .setUser(null)
+ .atIp(null)
+ .forOperation(op.getAction())
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.FAILURE.toString())
+ .withException(throwable)
+ .build();
+ }
+
+ enum EventType {
+ READ,
+ WRITE
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29374999/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 2ba9ea7..e034082 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -68,6 +68,8 @@ function ozonecmd_case
;;
datanode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HDDS_DN_OPTS="${HDDS_DN_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/dn-audit-log4j2.properties"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HDDS_DN_OPTS}"
HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-datanode"
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29374999/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
index 109bed1..e736065 100755
--- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
+++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
@@ -79,6 +79,7 @@ run mkdir -p ./libexec
run cp -r "${ROOT}/hadoop-common-project/hadoop-common/src/main/conf" "etc/hadoop"
run cp "${ROOT}/hadoop-ozone/dist/src/main/conf/om-audit-log4j2.properties" "etc/hadoop"
+run cp "${ROOT}/hadoop-ozone/dist/src/main/conf/dn-audit-log4j2.properties" "etc/hadoop"
run cp "${ROOT}/hadoop-ozone/dist/src/main/conf/ozone-site.xml" "etc/hadoop"
run cp -f "${ROOT}/hadoop-ozone/dist/src/main/conf/log4j.properties" "etc/hadoop"
run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop" "bin/"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29374999/hadoop-ozone/dist/src/main/conf/dn-audit-log4j2.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/conf/dn-audit-log4j2.properties b/hadoop-ozone/dist/src/main/conf/dn-audit-log4j2.properties
new file mode 100644
index 0000000..3c4d045
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/conf/dn-audit-log4j2.properties
@@ -0,0 +1,90 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# <p>
+# http://www.apache.org/licenses/LICENSE-2.0
+# <p>
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+name=PropertiesConfig
+
+# Checks for config change periodically and reloads
+monitorInterval=30
+
+filter=read,write
+# filter.read.onMatch=DENY avoids logging all READ events
+# filter.read.onMatch=ACCEPT permits logging all READ events
+# The above two settings ignore the log levels in configuration
+# filter.read.onMatch=NEUTRAL permits logging of only those READ events
+# which are attempted at log level equal or greater than log level specified
+# in the configuration
+filter.read.type=MarkerFilter
+filter.read.marker=READ
+filter.read.onMatch=DENY
+filter.read.onMismatch=NEUTRAL
+
+# filter.write.onMatch=DENY avoids logging all WRITE events
+# filter.write.onMatch=ACCEPT permits logging all WRITE events
+# The above two settings ignore the log levels in configuration
+# filter.write.onMatch=NEUTRAL permits logging of only those WRITE events
+# which are attempted at log level equal or greater than log level specified
+# in the configuration
+filter.write.type=MarkerFilter
+filter.write.marker=WRITE
+filter.write.onMatch=NEUTRAL
+filter.write.onMismatch=NEUTRAL
+
+# Log Levels are organized from most specific to least:
+# OFF (most specific, no logging)
+# FATAL (most specific, little data)
+# ERROR
+# WARN
+# INFO
+# DEBUG
+# TRACE (least specific, a lot of data)
+# ALL (least specific, all data)
+
+# Uncomment following section to enable logging to console appender also
+#appenders=console, rolling
+#appender.console.type=Console
+#appender.console.name=STDOUT
+#appender.console.layout.type=PatternLayout
+#appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+
+# Comment this line when using both console and rolling appenders
+appenders=rolling
+
+#Rolling File Appender with size & time thresholds.
+#Rolling is triggered when either threshold is breached.
+#The rolled over file is compressed by default
+#Time interval is specified in seconds 86400s=1 day
+appender.rolling.type=RollingFile
+appender.rolling.name=RollingFile
+appender.rolling.fileName =${sys:hadoop.log.dir}/dn-audit-${hostName}.log
+appender.rolling.filePattern=${sys:hadoop.log.dir}/dn-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+appender.rolling.layout.type=PatternLayout
+appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+appender.rolling.policies.type=Policies
+appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval=86400
+appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size=64MB
+
+loggers=audit
+logger.audit.type=AsyncLogger
+logger.audit.name=DNAudit
+logger.audit.level=INFO
+logger.audit.appenderRefs=rolling
+logger.audit.appenderRef.file.ref=RollingFile
+
+rootLogger.level=INFO
+#rootLogger.appenderRefs=stdout
+#rootLogger.appenderRef.stdout.ref=STDOUT
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDDS-840. [JDK11] Fix Javadoc
errors in hadoop-hdds-client module. Contributed by Dinesh Chitlangia.
Posted by br...@apache.org.
HDDS-840. [JDK11] Fix Javadoc errors in hadoop-hdds-client module. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3420ddd0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3420ddd0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3420ddd0
Branch: refs/heads/HDFS-13891
Commit: 3420ddd0f8278c4259435354138a0e046a9b0ac0
Parents: c4d9640
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Nov 16 10:13:41 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Nov 16 10:13:41 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/scm/XceiverClientRatis.java | 7 -------
.../hadoop/hdds/scm/client/ContainerOperationClient.java | 10 +---------
2 files changed, 1 insertion(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3420ddd0/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index dbda2e6..f6083ec 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -97,9 +97,6 @@ public final class XceiverClientRatis extends XceiverClientSpi {
this.retryPolicy = retryPolicy;
}
- /**
- * {@inheritDoc}
- */
@Override
public void createPipeline() throws IOException {
final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
@@ -108,9 +105,6 @@ public final class XceiverClientRatis extends XceiverClientSpi {
(raftClient, peer) -> raftClient.groupAdd(group, peer.getId()));
}
- /**
- * {@inheritDoc}
- */
@Override
public void destroyPipeline() throws IOException {
final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
@@ -219,7 +213,6 @@ public final class XceiverClientRatis extends XceiverClientSpi {
*
* @param request Request
* @return Response to the command
- * @throws IOException
*/
@Override
public XceiverClientAsyncReply sendCommandAsync(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3420ddd0/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
index 2bba709..8e9f4ad 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
@@ -78,9 +78,7 @@ public class ContainerOperationClient implements ScmClient {
containerSizeB = size;
}
- /**
- * @inheritDoc
- */
+
@Override
public ContainerWithPipeline createContainer(String owner)
throws IOException {
@@ -173,9 +171,6 @@ public class ContainerOperationClient implements ScmClient {
pipeline.toString());
}
- /**
- * @inheritDoc
- */
@Override
public ContainerWithPipeline createContainer(HddsProtos.ReplicationType type,
HddsProtos.ReplicationFactor factor, String owner) throws IOException {
@@ -280,9 +275,6 @@ public class ContainerOperationClient implements ScmClient {
deleteContainer(containerID, info.getPipeline(), force);
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ContainerInfo> listContainer(long startContainerID,
int count) throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HDDS-816. Create OM metrics for
bucket, volume, keys. Contributed by Bharat Viswanadham.
Posted by br...@apache.org.
HDDS-816. Create OM metrics for bucket, volume, keys. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4d0ef69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4d0ef69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4d0ef69
Branch: refs/heads/HDFS-13891
Commit: c4d0ef6946220d47eacdc7470a7bd29bc03e8db1
Parents: f42b2df
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Wed Nov 21 10:35:39 2018 -0800
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Wed Nov 21 10:35:39 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/OzoneConsts.java | 3 +
.../common/src/main/resources/ozone-default.xml | 10 +
.../apache/hadoop/ozone/om/OMConfigKeys.java | 5 +
.../hadoop/ozone/om/OMMetadataManager.java | 15 +-
.../apache/hadoop/ozone/om/TestOmMetrics.java | 140 ++++++++++--
.../org/apache/hadoop/ozone/om/KeyManager.java | 6 +-
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 35 +--
.../org/apache/hadoop/ozone/om/OMMetrics.java | 60 ++++++
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 106 ++++++----
.../apache/hadoop/ozone/om/OmMetricsInfo.java | 43 ++++
.../apache/hadoop/ozone/om/OzoneManager.java | 211 ++++++++++++++++---
.../apache/hadoop/ozone/om/S3BucketManager.java | 9 +
.../hadoop/ozone/om/S3BucketManagerImpl.java | 17 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 7 +-
.../hadoop/ozone/om/TestS3BucketManager.java | 5 +
15 files changed, 561 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 7dd9fe3..096baee 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -226,5 +226,8 @@ public final class OzoneConsts {
public static final String KEY_LOCATION_INFO = "keyLocationInfo";
+ // For OM metrics saving to a file
+ public static final String OM_METRICS_FILE = "omMetrics";
+ public static final String OM_METRICS_TEMP_FILE = OM_METRICS_FILE + ".tmp";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 767a865..869608e 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1351,5 +1351,15 @@
service principal. </description>
</property>
+ <property>
+ <name>ozone.om.save.metrics.interval</name>
+ <value>5m</value>
+ <tag>OZONE, OM</tag>
+ <description>Time interval used to store the omMetrics in to a
+ file. Background thread perodically stores the OM metrics in to a
+ file. Unit could be defined with postfix (ns,ms,s,m,h,d)
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 6e9acb4..739c75e 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.om;
import org.apache.hadoop.ozone.OzoneAcl;
+
/**
* Ozone Manager Constants.
*/
@@ -81,4 +82,8 @@ public final class OMConfigKeys {
public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
"ozone.key.deleting.limit.per.task";
public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
+
+ public static final String OZONE_OM_METRICS_SAVE_INTERVAL =
+ "ozone.om.save.metrics.interval";
+ public static final String OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT = "5m";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 5f490ec..247911a 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -33,8 +34,11 @@ import java.util.List;
public interface OMMetadataManager {
/**
* Start metadata manager.
+ *
+ * @param configuration
+ * @throws IOException
*/
- void start();
+ void start(OzoneConfiguration configuration) throws IOException;
/**
* Stop metadata manager.
@@ -242,4 +246,13 @@ public interface OMMetadataManager {
* @return Table.
*/
Table getS3Table();
+
+ /**
+ * Returns number of rows in a table. This should not be used for very
+ * large tables.
+ * @param table
+ * @return long
+ * @throws IOException
+ */
+ long countRowsInTable(Table table) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
index 665a9c7..2ff04af 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
@@ -18,13 +18,23 @@ package org.apache.hadoop.ozone.om;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.test.MetricsAsserts;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -51,6 +61,8 @@ public class TestOmMetrics {
@Before
public void setup() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setTimeDuration(OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL,
+ 1000, TimeUnit.MILLISECONDS);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
ozoneManager = cluster.getOzoneManager();
@@ -66,6 +78,8 @@ public class TestOmMetrics {
}
}
+
+
@Test
public void testVolumeOps() throws IOException {
VolumeManager volumeManager =
@@ -92,6 +106,16 @@ public class TestOmMetrics {
assertCounter("NumVolumeCheckAccesses", 1L, omMetrics);
assertCounter("NumVolumeDeletes", 1L, omMetrics);
assertCounter("NumVolumeLists", 1L, omMetrics);
+ assertCounter("NumVolumes", 0L, omMetrics);
+
+ ozoneManager.createVolume(null);
+ ozoneManager.createVolume(null);
+ ozoneManager.createVolume(null);
+ ozoneManager.deleteVolume(null);
+
+ omMetrics = getMetrics("OMMetrics");
+ assertCounter("NumVolumes", 2L, omMetrics);
+
// inject exception to test for Failure Metrics
Mockito.doThrow(exception).when(mockVm).createVolume(null);
@@ -106,12 +130,12 @@ public class TestOmMetrics {
doVolumeOps();
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumVolumeOps", 12L, omMetrics);
- assertCounter("NumVolumeCreates", 2L, omMetrics);
+ assertCounter("NumVolumeOps", 16L, omMetrics);
+ assertCounter("NumVolumeCreates", 5L, omMetrics);
assertCounter("NumVolumeUpdates", 2L, omMetrics);
assertCounter("NumVolumeInfos", 2L, omMetrics);
assertCounter("NumVolumeCheckAccesses", 2L, omMetrics);
- assertCounter("NumVolumeDeletes", 2L, omMetrics);
+ assertCounter("NumVolumeDeletes", 3L, omMetrics);
assertCounter("NumVolumeLists", 2L, omMetrics);
assertCounter("NumVolumeCreateFails", 1L, omMetrics);
@@ -120,6 +144,15 @@ public class TestOmMetrics {
assertCounter("NumVolumeCheckAccessFails", 1L, omMetrics);
assertCounter("NumVolumeDeleteFails", 1L, omMetrics);
assertCounter("NumVolumeListFails", 1L, omMetrics);
+
+ // As last call for volumesOps does not increment numVolumes as those are
+ // failed.
+ assertCounter("NumVolumes", 2L, omMetrics);
+
+ cluster.restartOzoneManager();
+ assertCounter("NumVolumes", 2L, omMetrics);
+
+
}
@Test
@@ -129,6 +162,16 @@ public class TestOmMetrics {
ozoneManager, "bucketManager");
BucketManager mockBm = Mockito.spy(bucketManager);
+ S3BucketManager s3BucketManager =
+ (S3BucketManager) HddsWhiteboxTestUtils.getInternalState(
+ ozoneManager, "s3BucketManager");
+ S3BucketManager mockS3Bm = Mockito.spy(s3BucketManager);
+
+ Mockito.doNothing().when(mockS3Bm).createS3Bucket("random", "random");
+ Mockito.doNothing().when(mockS3Bm).deleteS3Bucket("random");
+ Mockito.doReturn(true).when(mockS3Bm).createOzoneVolumeIfNeeded(null);
+
+ Mockito.doNothing().when(mockBm).createBucket(null);
Mockito.doNothing().when(mockBm).createBucket(null);
Mockito.doNothing().when(mockBm).deleteBucket(null, null);
Mockito.doReturn(null).when(mockBm).getBucketInfo(null, null);
@@ -146,6 +189,35 @@ public class TestOmMetrics {
assertCounter("NumBucketInfos", 1L, omMetrics);
assertCounter("NumBucketDeletes", 1L, omMetrics);
assertCounter("NumBucketLists", 1L, omMetrics);
+ assertCounter("NumBuckets", 0L, omMetrics);
+
+ ozoneManager.createBucket(null);
+ ozoneManager.createBucket(null);
+ ozoneManager.createBucket(null);
+ ozoneManager.deleteBucket(null, null);
+
+ //Taking already existing value, as the same metrics is used over all the
+ // test cases.
+ long numVolumesOps = MetricsAsserts.getLongCounter("NumVolumeOps",
+ omMetrics);
+ long numVolumes = MetricsAsserts.getLongCounter("NumVolumes",
+ omMetrics);
+ long numVolumeCreates = MetricsAsserts.getLongCounter("NumVolumeCreates",
+ omMetrics);
+
+ ozoneManager.createS3Bucket("random", "random");
+ ozoneManager.createS3Bucket("random1", "random1");
+ ozoneManager.createS3Bucket("random2", "random2");
+ ozoneManager.deleteS3Bucket("random");
+
+ omMetrics = getMetrics("OMMetrics");
+ assertCounter("NumBuckets", 4L, omMetrics);
+
+ assertCounter("NumVolumeOps", numVolumesOps + 3, omMetrics);
+ assertCounter("NumVolumeCreates", numVolumeCreates + 3, omMetrics);
+ assertCounter("NumVolumes", numVolumes + 3, omMetrics);
+
+
// inject exception to test for Failure Metrics
Mockito.doThrow(exception).when(mockBm).createBucket(null);
@@ -159,11 +231,11 @@ public class TestOmMetrics {
doBucketOps();
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumBucketOps", 10L, omMetrics);
- assertCounter("NumBucketCreates", 2L, omMetrics);
+ assertCounter("NumBucketOps", 18L, omMetrics);
+ assertCounter("NumBucketCreates", 8L, omMetrics);
assertCounter("NumBucketUpdates", 2L, omMetrics);
assertCounter("NumBucketInfos", 2L, omMetrics);
- assertCounter("NumBucketDeletes", 2L, omMetrics);
+ assertCounter("NumBucketDeletes", 4L, omMetrics);
assertCounter("NumBucketLists", 2L, omMetrics);
assertCounter("NumBucketCreateFails", 1L, omMetrics);
@@ -171,29 +243,49 @@ public class TestOmMetrics {
assertCounter("NumBucketInfoFails", 1L, omMetrics);
assertCounter("NumBucketDeleteFails", 1L, omMetrics);
assertCounter("NumBucketListFails", 1L, omMetrics);
+
+ assertCounter("NumBuckets", 4L, omMetrics);
+
+ cluster.restartOzoneManager();
+ assertCounter("NumBuckets", 4L, omMetrics);
}
@Test
public void testKeyOps() throws IOException {
- KeyManager bucketManager = (KeyManager) HddsWhiteboxTestUtils
+ KeyManager keyManager = (KeyManager) HddsWhiteboxTestUtils
.getInternalState(ozoneManager, "keyManager");
- KeyManager mockKm = Mockito.spy(bucketManager);
+ KeyManager mockKm = Mockito.spy(keyManager);
Mockito.doReturn(null).when(mockKm).openKey(null);
Mockito.doNothing().when(mockKm).deleteKey(null);
Mockito.doReturn(null).when(mockKm).lookupKey(null);
Mockito.doReturn(null).when(mockKm).listKeys(null, null, null, null, 0);
+ Mockito.doNothing().when(mockKm).commitKey(any(OmKeyArgs.class), anyLong());
HddsWhiteboxTestUtils.setInternalState(
ozoneManager, "keyManager", mockKm);
doKeyOps();
MetricsRecordBuilder omMetrics = getMetrics("OMMetrics");
- assertCounter("NumKeyOps", 4L, omMetrics);
+ assertCounter("NumKeyOps", 5L, omMetrics);
assertCounter("NumKeyAllocate", 1L, omMetrics);
assertCounter("NumKeyLookup", 1L, omMetrics);
assertCounter("NumKeyDeletes", 1L, omMetrics);
assertCounter("NumKeyLists", 1L, omMetrics);
+ assertCounter("NumKeys", 0L, omMetrics);
+
+
+ ozoneManager.openKey(null);
+ ozoneManager.commitKey(createKeyArgs(), 0);
+ ozoneManager.openKey(null);
+ ozoneManager.commitKey(createKeyArgs(), 0);
+ ozoneManager.openKey(null);
+ ozoneManager.commitKey(createKeyArgs(), 0);
+ ozoneManager.deleteKey(null);
+
+
+ omMetrics = getMetrics("OMMetrics");
+ assertCounter("NumKeys", 2L, omMetrics);
// inject exception to test for Failure Metrics
Mockito.doThrow(exception).when(mockKm).openKey(null);
@@ -201,22 +293,30 @@ public class TestOmMetrics {
Mockito.doThrow(exception).when(mockKm).lookupKey(null);
Mockito.doThrow(exception).when(mockKm).listKeys(
null, null, null, null, 0);
+ Mockito.doThrow(exception).when(mockKm).commitKey(any(OmKeyArgs.class),
+ anyLong());
HddsWhiteboxTestUtils.setInternalState(
ozoneManager, "keyManager", mockKm);
doKeyOps();
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumKeyOps", 8L, omMetrics);
- assertCounter("NumKeyAllocate", 2L, omMetrics);
+ assertCounter("NumKeyOps", 17L, omMetrics);
+ assertCounter("NumKeyAllocate", 5L, omMetrics);
assertCounter("NumKeyLookup", 2L, omMetrics);
- assertCounter("NumKeyDeletes", 2L, omMetrics);
+ assertCounter("NumKeyDeletes", 3L, omMetrics);
assertCounter("NumKeyLists", 2L, omMetrics);
assertCounter("NumKeyAllocateFails", 1L, omMetrics);
assertCounter("NumKeyLookupFails", 1L, omMetrics);
assertCounter("NumKeyDeleteFails", 1L, omMetrics);
assertCounter("NumKeyListFails", 1L, omMetrics);
+
+ assertCounter("NumKeys", 2L, omMetrics);
+
+ cluster.restartOzoneManager();
+ assertCounter("NumKeys", 2L, omMetrics);
+
}
/**
@@ -307,5 +407,21 @@ public class TestOmMetrics {
ozoneManager.listKeys(null, null, null, null, 0);
} catch (IOException ignored) {
}
+
+ try {
+ ozoneManager.commitKey(createKeyArgs(), 0);
+ } catch (IOException ignored) {
+ }
+ }
+
+ private OmKeyArgs createKeyArgs() {
+ OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder()
+ .setBlockID(new BlockID(new ContainerBlockID(1, 1))).build();
+ keyLocationInfo.setCreateVersion(0);
+ List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
+ omKeyLocationInfoList.add(keyLocationInfo);
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder().setLocationInfoList(
+ omKeyLocationInfoList).build();
+ return keyArgs;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
index b7b4517..38b91f5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.ozone.om;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -33,8 +34,11 @@ public interface KeyManager {
/**
* Start key manager.
+ *
+ * @param configuration
+ * @throws IOException
*/
- void start();
+ void start(OzoneConfiguration configuration);
/**
* Stop key manager.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index a41dae6..645b39f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -79,7 +79,7 @@ public class KeyManagerImpl implements KeyManager {
private final long preallocateMax;
private final String omId;
- private final BackgroundService keyDeletingService;
+ private BackgroundService keyDeletingService;
public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
OMMetadataManager metadataManager,
@@ -95,28 +95,33 @@ public class KeyManagerImpl implements KeyManager {
this.preallocateMax = conf.getLong(
OZONE_KEY_PREALLOCATION_MAXSIZE,
OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT);
- long blockDeleteInterval = conf.getTimeDuration(
- OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
- OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS);
- long serviceTimeout = conf.getTimeDuration(
- OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
- OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- keyDeletingService = new KeyDeletingService(
- scmBlockClient, this, blockDeleteInterval, serviceTimeout, conf);
-
this.omId = omId;
+ start(conf);
}
@Override
- public void start() {
- keyDeletingService.start();
+ public void start(OzoneConfiguration configuration) {
+ if (keyDeletingService == null) {
+ long blockDeleteInterval = configuration.getTimeDuration(
+ OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
+ OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ long serviceTimeout = configuration.getTimeDuration(
+ OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
+ OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ keyDeletingService = new KeyDeletingService(scmBlockClient, this,
+ blockDeleteInterval, serviceTimeout, configuration);
+ keyDeletingService.start();
+ }
}
@Override
public void stop() throws IOException {
- keyDeletingService.shutdown();
+ if (keyDeletingService != null) {
+ keyDeletingService.shutdown();
+ keyDeletingService = null;
+ }
}
private void validateBucket(String volumeName, String bucketName)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
index 1b396f9..f6925ca 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
@@ -85,6 +85,17 @@ public class OMMetrics {
private @Metric MutableCounterLong numGetServiceListFails;
private @Metric MutableCounterLong numListS3BucketsFails;
+ // Metrics for total number of volumes, buckets and keys
+
+ private @Metric MutableCounterLong numVolumes;
+ private @Metric MutableCounterLong numBuckets;
+
+ //TODO: This metric is an estimate and it may be inaccurate on restart if the
+ // OM process was not shutdown cleanly. Key creations/deletions in the last
+ // few minutes before restart may not be included in this count.
+ private @Metric MutableCounterLong numKeys;
+
+
public OMMetrics() {
}
@@ -95,6 +106,55 @@ public class OMMetrics {
new OMMetrics());
}
+ public void incNumVolumes() {
+ numVolumes.incr();
+ }
+
+ public void decNumVolumes() {
+ numVolumes.incr(-1);
+ }
+
+ public void incNumBuckets() {
+ numBuckets.incr();
+ }
+
+ public void decNumBuckets() {
+ numBuckets.incr(-1);
+ }
+
+ public void incNumKeys() {
+ numKeys.incr();
+ }
+
+ public void decNumKeys() {
+ numKeys.incr(-1);
+ }
+
+ public void setNumVolumes(long val) {
+ this.numVolumes.incr(val);
+ }
+
+ public void setNumBuckets(long val) {
+ this.numBuckets.incr(val);
+ }
+
+ public void setNumKeys(long val) {
+ this.numKeys.incr(val);
+ }
+
+ public long getNumVolumes() {
+ return numVolumes.value();
+ }
+
+ public long getNumBuckets() {
+ return numBuckets.value();
+ }
+
+ public long getNumKeys() {
+ return numKeys.value();
+ }
+
+
public void incNumVolumeCreates() {
numVolumeOps.incr();
numVolumeCreates.incr();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 0bfbc1f..ad9dcc9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -100,59 +100,25 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
private static final String OPEN_KEY_TABLE = "openKeyTable";
private static final String S3_TABLE = "s3Table";
- private final DBStore store;
+ private DBStore store;
private final OzoneManagerLock lock;
private final long openKeyExpireThresholdMS;
- private final Table userTable;
- private final Table volumeTable;
- private final Table bucketTable;
- private final Table keyTable;
- private final Table deletedTable;
- private final Table openKeyTable;
- private final Table s3Table;
+ private Table userTable;
+ private Table volumeTable;
+ private Table bucketTable;
+ private Table keyTable;
+ private Table deletedTable;
+ private Table openKeyTable;
+ private Table s3Table;
public OmMetadataManagerImpl(OzoneConfiguration conf) throws IOException {
- File metaDir = OmUtils.getOmDbDir(conf);
this.lock = new OzoneManagerLock(conf);
this.openKeyExpireThresholdMS = 1000L * conf.getInt(
OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS,
OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT);
-
- this.store = DBStoreBuilder.newBuilder(conf)
- .setName(OM_DB_NAME)
- .setPath(Paths.get(metaDir.getPath()))
- .addTable(USER_TABLE)
- .addTable(VOLUME_TABLE)
- .addTable(BUCKET_TABLE)
- .addTable(KEY_TABLE)
- .addTable(DELETED_TABLE)
- .addTable(OPEN_KEY_TABLE)
- .addTable(S3_TABLE)
- .build();
-
- userTable = this.store.getTable(USER_TABLE);
- checkTableStatus(userTable, USER_TABLE);
-
- volumeTable = this.store.getTable(VOLUME_TABLE);
- checkTableStatus(volumeTable, VOLUME_TABLE);
-
- bucketTable = this.store.getTable(BUCKET_TABLE);
- checkTableStatus(bucketTable, BUCKET_TABLE);
-
- keyTable = this.store.getTable(KEY_TABLE);
- checkTableStatus(keyTable, KEY_TABLE);
-
- deletedTable = this.store.getTable(DELETED_TABLE);
- checkTableStatus(deletedTable, DELETED_TABLE);
-
- openKeyTable = this.store.getTable(OPEN_KEY_TABLE);
- checkTableStatus(openKeyTable, OPEN_KEY_TABLE);
-
- s3Table = this.store.getTable(S3_TABLE);
- checkTableStatus(s3Table, S3_TABLE);
-
+ start(conf);
}
@Override
@@ -206,8 +172,44 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
* Start metadata manager.
*/
@Override
- public void start() {
+ public void start(OzoneConfiguration configuration) throws IOException {
+ // We need to create the DB here, as when during restart, stop closes the
+ // db, so we need to create the store object and initialize DB.
+ if (store == null) {
+ File metaDir = OmUtils.getOmDbDir(configuration);
+ this.store = DBStoreBuilder.newBuilder(configuration)
+ .setName(OM_DB_NAME)
+ .setPath(Paths.get(metaDir.getPath()))
+ .addTable(USER_TABLE)
+ .addTable(VOLUME_TABLE)
+ .addTable(BUCKET_TABLE)
+ .addTable(KEY_TABLE)
+ .addTable(DELETED_TABLE)
+ .addTable(OPEN_KEY_TABLE)
+ .addTable(S3_TABLE)
+ .build();
+
+ userTable = this.store.getTable(USER_TABLE);
+ checkTableStatus(userTable, USER_TABLE);
+
+ volumeTable = this.store.getTable(VOLUME_TABLE);
+ checkTableStatus(volumeTable, VOLUME_TABLE);
+ bucketTable = this.store.getTable(BUCKET_TABLE);
+ checkTableStatus(bucketTable, BUCKET_TABLE);
+
+ keyTable = this.store.getTable(KEY_TABLE);
+ checkTableStatus(keyTable, KEY_TABLE);
+
+ deletedTable = this.store.getTable(DELETED_TABLE);
+ checkTableStatus(deletedTable, DELETED_TABLE);
+
+ openKeyTable = this.store.getTable(OPEN_KEY_TABLE);
+ checkTableStatus(openKeyTable, OPEN_KEY_TABLE);
+
+ s3Table = this.store.getTable(S3_TABLE);
+ checkTableStatus(s3Table, S3_TABLE);
+ }
}
/**
@@ -217,6 +219,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
public void stop() throws Exception {
if (store != null) {
store.close();
+ store = null;
}
}
@@ -627,4 +630,19 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
}
return keyBlocksList;
}
+
+ @Override
+ public long countRowsInTable(Table table) throws IOException {
+ long count = 0;
+ if (table != null) {
+ try (TableIterator<Table.KeyValue> keyValueTableIterator =
+ table.iterator()) {
+ while (keyValueTableIterator.hasNext()) {
+ keyValueTableIterator.next();
+ count++;
+ }
+ }
+ }
+ return count;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetricsInfo.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetricsInfo.java
new file mode 100644
index 0000000..e9b1f43
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetricsInfo.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * OmMetricsInfo stored in a file, which will be used during OM restart to
+ * initialize the metrics. Currently this stores only numKeys.
+ */
+public class OmMetricsInfo {
+
+ @JsonProperty
+ private long numKeys;
+
+ OmMetricsInfo() {
+ this.numKeys = 0;
+ }
+
+ public long getNumKeys() {
+ return numKeys;
+ }
+
+ public void setNumKeys(long numKeys) {
+ this.numKeys = numKeys;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index bc24a50..71dde34 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -17,6 +17,9 @@
package org.apache.hadoop.ozone.om;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
@@ -41,6 +44,7 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.AuditAction;
import org.apache.hadoop.ozone.audit.AuditEventStatus;
@@ -67,19 +71,29 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Service
import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.management.ObjectName;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
@@ -87,10 +101,19 @@ import static org.apache.hadoop.hdds.HddsUtils.isHddsEnabled;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
import static org.apache.hadoop.ozone.OmUtils.getOmAddress;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
+
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_METRICS_SAVE_INTERVAL;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneManagerService.newReflectiveBlockingService;
import static org.apache.hadoop.util.ExitUtil.terminate;
@@ -111,19 +134,28 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
+ StartupOption.INIT.getName() + " ]\n " + "ozone om [ "
+ StartupOption.HELP.getName() + " ]\n";
private final OzoneConfiguration configuration;
- private final RPC.Server omRpcServer;
- private final InetSocketAddress omRpcAddress;
+ private RPC.Server omRpcServer;
+ private InetSocketAddress omRpcAddress;
private final OMMetadataManager metadataManager;
private final VolumeManager volumeManager;
private final BucketManager bucketManager;
private final KeyManager keyManager;
private final OMMetrics metrics;
- private final OzoneManagerHttpServer httpServer;
+ private OzoneManagerHttpServer httpServer;
private final OMStorage omStorage;
private final ScmBlockLocationProtocol scmBlockClient;
private final StorageContainerLocationProtocol scmContainerClient;
private ObjectName omInfoBeanName;
private final S3BucketManager s3BucketManager;
+ private Timer metricsTimer;
+ private ScheduleOMMetricsWriteTask scheduleOMMetricsWriteTask;
+ private static final ObjectWriter WRITER =
+ new ObjectMapper().writerWithDefaultPrettyPrinter();
+ private static final ObjectReader READER =
+ new ObjectMapper().readerFor(OmMetricsInfo.class);
+ private static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private final Runnable shutdownHook;
+ private final File omMetaDir;
private OzoneManager(OzoneConfiguration conf) throws IOException {
Preconditions.checkNotNull(conf);
@@ -143,33 +175,79 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
throw new OMException("SCM version info mismatch.",
ResultCodes.SCM_VERSION_MISMATCH_ERROR);
}
- final int handlerCount = conf.getInt(OZONE_OM_HANDLER_COUNT_KEY,
- OZONE_OM_HANDLER_COUNT_DEFAULT);
RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class,
ProtobufRpcEngine.class);
- BlockingService omService = newReflectiveBlockingService(
- new OzoneManagerProtocolServerSideTranslatorPB(this));
- final InetSocketAddress omNodeRpcAddr =
- getOmAddress(configuration);
- omRpcServer = startRpcServer(configuration, omNodeRpcAddr,
- OzoneManagerProtocolPB.class, omService,
- handlerCount);
- omRpcAddress = updateRPCListenAddress(configuration,
- OZONE_OM_ADDRESS_KEY, omNodeRpcAddr, omRpcServer);
metadataManager = new OmMetadataManagerImpl(configuration);
volumeManager = new VolumeManagerImpl(metadataManager, configuration);
bucketManager = new BucketManagerImpl(metadataManager);
+ metrics = OMMetrics.create();
+
s3BucketManager = new S3BucketManagerImpl(configuration, metadataManager,
volumeManager, bucketManager);
- metrics = OMMetrics.create();
keyManager =
new KeyManagerImpl(scmBlockClient, metadataManager, configuration,
omStorage.getOmId());
- httpServer = new OzoneManagerHttpServer(configuration, this);
+
+ shutdownHook = () -> {
+ saveOmMetrics();
+ };
+ ShutdownHookManager.get().addShutdownHook(shutdownHook,
+ SHUTDOWN_HOOK_PRIORITY);
+
+ omMetaDir = OmUtils.getOmDbDir(configuration);
+
+ }
+
+ /**
+ * Class which schedule saving metrics to a file.
+ */
+ private class ScheduleOMMetricsWriteTask extends TimerTask {
+ public void run() {
+ saveOmMetrics();
+ }
+ }
+
+ private void saveOmMetrics() {
+ try {
+ boolean success;
+ try (BufferedWriter writer = new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(
+ getTempMetricsStorageFile()), "UTF-8"))) {
+ OmMetricsInfo metricsInfo = new OmMetricsInfo();
+ metricsInfo.setNumKeys(metrics.getNumKeys());
+ WRITER.writeValue(writer, metricsInfo);
+ success = true;
+ }
+
+ if (success) {
+ Files.move(getTempMetricsStorageFile().toPath(),
+ getMetricsStorageFile().toPath(), StandardCopyOption
+ .ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+ }
+ } catch (IOException ex) {
+ LOG.error("Unable to write the om Metrics file", ex);
+ }
+ }
+
+ /**
+ * Returns temporary metrics storage file.
+ * @return File
+ */
+ private File getTempMetricsStorageFile() {
+ return new File(omMetaDir, OM_METRICS_TEMP_FILE);
+ }
+
+ /**
+ * Returns metrics storage file.
+ * @return File
+ */
+ private File getMetricsStorageFile() {
+ return new File(omMetaDir, OM_METRICS_FILE);
}
+
/**
* Create a scm block client, used by putKey() and getKey().
*
@@ -448,12 +526,49 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* Start service.
*/
public void start() throws IOException {
+
+ InetSocketAddress omNodeRpcAddr = getOmAddress(configuration);
+ int handlerCount = configuration.getInt(OZONE_OM_HANDLER_COUNT_KEY,
+ OZONE_OM_HANDLER_COUNT_DEFAULT);
+ BlockingService omService = newReflectiveBlockingService(
+ new OzoneManagerProtocolServerSideTranslatorPB(this));
+ omRpcServer = startRpcServer(configuration, omNodeRpcAddr,
+ OzoneManagerProtocolPB.class, omService,
+ handlerCount);
+ omRpcAddress = updateRPCListenAddress(configuration,
+ OZONE_OM_ADDRESS_KEY, omNodeRpcAddr, omRpcServer);
+ omRpcServer.start();
+
LOG.info(buildRpcServerStartMessage("OzoneManager RPC server",
omRpcAddress));
+
+
DefaultMetricsSystem.initialize("OzoneManager");
- metadataManager.start();
- keyManager.start();
- omRpcServer.start();
+
+ metadataManager.start(configuration);
+
+
+ // Set metrics and start metrics back ground thread
+ metrics.setNumVolumes(metadataManager.countRowsInTable(metadataManager
+ .getVolumeTable()));
+ metrics.setNumBuckets(metadataManager.countRowsInTable(metadataManager
+ .getBucketTable()));
+
+ if (getMetricsStorageFile().exists()) {
+ OmMetricsInfo metricsInfo = READER.readValue(getMetricsStorageFile());
+ metrics.setNumKeys(metricsInfo.getNumKeys());
+ }
+
+ // Schedule save metrics
+ long period = configuration.getTimeDuration(OZONE_OM_METRICS_SAVE_INTERVAL,
+ OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
+ scheduleOMMetricsWriteTask = new ScheduleOMMetricsWriteTask();
+ metricsTimer = new Timer();
+ metricsTimer.schedule(scheduleOMMetricsWriteTask, 0, period);
+
+ keyManager.start(configuration);
+
+ httpServer = new OzoneManagerHttpServer(configuration, this);
httpServer.start();
registerMXBean();
setStartTime();
@@ -464,10 +579,14 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/
public void stop() {
try {
- metadataManager.stop();
+ // Cancel the metrics timer and set to null.
+ metricsTimer.cancel();
+ metricsTimer = null;
+ scheduleOMMetricsWriteTask = null;
omRpcServer.stop();
keyManager.stop();
httpServer.stop();
+ metadataManager.stop();
metrics.unRegister();
unregisterMXBean();
} catch (Exception e) {
@@ -500,6 +619,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
volumeManager.createVolume(args);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_VOLUME,
(args == null) ? null : args.toAuditMap()));
+ metrics.incNumVolumes();
} catch (Exception ex) {
metrics.incNumVolumeCreateFails();
AUDIT.logWriteFailure(
@@ -633,6 +753,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
volumeManager.deleteVolume(volume);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_VOLUME,
buildAuditMap(volume)));
+ metrics.decNumVolumes();
} catch (Exception ex) {
metrics.incNumVolumeDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_VOLUME,
@@ -727,6 +848,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
bucketManager.createBucket(bucketInfo);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_BUCKET,
(bucketInfo == null) ? null : bucketInfo.toAuditMap()));
+ metrics.incNumBuckets();
} catch (Exception ex) {
metrics.incNumBucketCreateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.CREATE_BUCKET,
@@ -835,6 +957,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
keyManager.commitKey(args, clientID);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.COMMIT_KEY,
auditMap));
+ // As when we commit the key it is visible, so we should increment here.
+ // As key also can have multiple versions, we need to increment keys
+ // only if version is 0. Currently we have not complete support of
+ // versioning of keys. So, this can be revisited later.
+ if (args != null && args.getLocationInfoList() != null &&
+ args.getLocationInfoList().size() > 0 &&
+ args.getLocationInfoList().get(0) != null &&
+ args.getLocationInfoList().get(0).getCreateVersion() == 0) {
+ metrics.incNumKeys();
+ }
} catch (Exception ex) {
metrics.incNumKeyCommitFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.COMMIT_KEY,
@@ -925,6 +1057,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
keyManager.deleteKey(args);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_KEY,
(args == null) ? null : args.toAuditMap()));
+ metrics.decNumKeys();
} catch (Exception ex) {
metrics.incNumKeyDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_KEY,
@@ -998,6 +1131,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
bucketManager.deleteBucket(volume, bucket);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_BUCKET,
auditMap));
+ metrics.decNumBuckets();
} catch (Exception ex) {
metrics.incNumBucketDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_BUCKET,
@@ -1136,7 +1270,30 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/
public void createS3Bucket(String userName, String s3BucketName)
throws IOException {
- s3BucketManager.createS3Bucket(userName, s3BucketName);
+ try {
+ metrics.incNumBucketCreates();
+ try {
+ boolean newVolumeCreate = s3BucketManager.createOzoneVolumeIfNeeded(
+ userName);
+ if (newVolumeCreate) {
+ metrics.incNumVolumeCreates();
+ metrics.incNumVolumes();
+ }
+ } catch (IOException ex) {
+ // We need to increment volume creates also because this is first
+ // time we are trying to create a volume, it failed. As we increment
+ // ops and create when we try to do that operation.
+ metrics.incNumVolumeCreates();
+ metrics.incNumVolumeCreateFails();
+ throw ex;
+ }
+
+ s3BucketManager.createS3Bucket(userName, s3BucketName);
+ metrics.incNumBuckets();
+ } catch (IOException ex) {
+ metrics.incNumBucketCreateFails();
+ throw ex;
+ }
}
@Override
@@ -1145,7 +1302,13 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/
public void deleteS3Bucket(String s3BucketName)
throws IOException {
- s3BucketManager.deleteS3Bucket(s3BucketName);
+ try {
+ metrics.incNumBucketDeletes();
+ s3BucketManager.deleteS3Bucket(s3BucketName);
+ metrics.decNumBuckets();
+ } catch (IOException ex) {
+ metrics.incNumBucketDeleteFails();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
index f22af7f..9e3523f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
@@ -69,4 +69,13 @@ public interface S3BucketManager {
* @param userName
*/
String getOzoneVolumeNameForUser(String userName) throws IOException;
+
+ /**
+ * Create ozone volume if required, this will be needed during creates3Bucket.
+ * @param userName
+ * @return true - if volume is successfully created. false - if volume
+ * already exists or volume creation failure.
+ * @throws IOException - incase of volume creation failure.
+ */
+ boolean createOzoneVolumeIfNeeded(String userName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
index d6fd410..43ceb80 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
@@ -95,14 +95,7 @@ public class S3BucketManagerImpl implements S3BucketManager {
// You might wonder if all names map to this pattern, why we need to
// store the S3 bucketName in a table at all. This is to support
// anonymous access to bucket where the user name is absent.
-
- // About Locking:
- // We need to do this before we take the S3Bucket Lock since createVolume
- // takes the userLock. So an attempt to take the user lock while holding
- // S3Bucket lock will throw, so we need to create the volume if needed
- // before we execute the bucket mapping functions.
String ozoneVolumeName = formatOzoneVolumeName(userName);
- createOzoneVolumeIfNeeded(userName, ozoneVolumeName);
omMetadataManager.getLock().acquireS3Lock(bucketName);
try {
@@ -157,20 +150,24 @@ public class S3BucketManagerImpl implements S3BucketManager {
return String.format(OM_S3_VOLUME_PREFIX + "%s", userName);
}
- private void createOzoneVolumeIfNeeded(String userName, String volumeName)
+ @Override
+ public boolean createOzoneVolumeIfNeeded(String userName)
throws IOException {
// We don't have to time of check. time of use problem here because
// this call is invoked while holding the s3Bucket lock.
+ boolean newVolumeCreate = true;
+ String ozoneVolumeName = formatOzoneVolumeName(userName);
try {
OmVolumeArgs args =
OmVolumeArgs.newBuilder()
.setAdminName(S3_ADMIN_NAME)
.setOwnerName(userName)
- .setVolume(volumeName)
+ .setVolume(ozoneVolumeName)
.setQuotaInBytes(OzoneConsts.MAX_QUOTA_IN_BYTES)
.build();
volumeManager.createVolume(args);
} catch (OMException exp) {
+ newVolumeCreate = false;
if (exp.getResult().compareTo(FAILED_VOLUME_ALREADY_EXISTS) == 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("Volume already exists. {}", exp.getMessage());
@@ -179,6 +176,8 @@ public class S3BucketManagerImpl implements S3BucketManager {
throw exp;
}
}
+
+ return newVolumeCreate;
}
private void createOzoneBucket(String volumeName, String bucketName)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
index 60c6fc3..d4612af 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
@@ -95,9 +95,6 @@ public class TestKeyDeletingService {
createAndDeleteKeys(keyManager, keyCount, 1);
KeyDeletingService keyDeletingService =
(KeyDeletingService) keyManager.getDeletingService();
- keyManager.start();
- Assert.assertEquals(
- keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), keyCount);
GenericTestUtils.waitFor(
() -> keyDeletingService.getDeletedKeyCount().get() >= keyCount,
1000, 10000);
@@ -120,7 +117,7 @@ public class TestKeyDeletingService {
createAndDeleteKeys(keyManager, keyCount, 1);
KeyDeletingService keyDeletingService =
(KeyDeletingService) keyManager.getDeletingService();
- keyManager.start();
+ keyManager.start(conf);
Assert.assertEquals(
keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), keyCount);
// Make sure that we have run the background thread 5 times more
@@ -147,7 +144,7 @@ public class TestKeyDeletingService {
createAndDeleteKeys(keyManager, keyCount, 0);
KeyDeletingService keyDeletingService =
(KeyDeletingService) keyManager.getDeletingService();
- keyManager.start();
+ keyManager.start(conf);
// Since empty keys are directly deleted from db there should be no
// pending deletion keys. Also deletedKeyCount should be zero.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
index 5160045..bf631e2 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
@@ -64,6 +64,7 @@ public class TestS3BucketManager {
public void testCreateS3Bucket() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
+ s3BucketManager.createOzoneVolumeIfNeeded("bilbo");
s3BucketManager.createS3Bucket("bilbo", "bucket");
// This call should have created a ozone volume called s3bilbo and bucket
@@ -104,6 +105,7 @@ public class TestS3BucketManager {
public void testDeleteS3Bucket() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
+ s3BucketManager.createOzoneVolumeIfNeeded("ozone");
s3BucketManager.createS3Bucket("ozone", "s3bucket");
// This call should have created a ozone volume called s3ozone and bucket
@@ -122,6 +124,7 @@ public class TestS3BucketManager {
public void testGetS3BucketMapping() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
+ s3BucketManager.createOzoneVolumeIfNeeded("bilbo");
s3BucketManager.createS3Bucket("bilbo", "newBucket");
String mapping = s3BucketManager.getOzoneBucketMapping("newBucket");
Assert.assertTrue(mapping.startsWith("s3bilbo/"));
@@ -132,6 +135,7 @@ public class TestS3BucketManager {
public void testGetOzoneNames() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
+ s3BucketManager.createOzoneVolumeIfNeeded("batman");
s3BucketManager.createS3Bucket("batman", "gotham");
String volumeName = s3BucketManager.getOzoneVolumeName("gotham");
Assert.assertTrue(volumeName.equalsIgnoreCase("s3batman"));
@@ -150,6 +154,7 @@ public class TestS3BucketManager {
public void testBucketNameAreUnique() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
+ s3BucketManager.createOzoneVolumeIfNeeded("superman");
s3BucketManager.createS3Bucket("superman", "metropolis");
// recreating the same bucket even with a different user will throw.
thrown.expectMessage("Unable to create S3 bucket.");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDDS-865. GrpcXceiverService is
added twice to GRPC netty server. Contributed by Xiaoyu Yao.
Posted by br...@apache.org.
HDDS-865. GrpcXceiverService is added twice to GRPC netty server. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f42b2dfd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f42b2dfd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f42b2dfd
Branch: refs/heads/HDFS-13891
Commit: f42b2dfdf12c2f2a1952e2492f8d1b7b358013f6
Parents: 34b6fa7
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Wed Nov 21 23:43:01 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Wed Nov 21 23:43:01 2018 +0530
----------------------------------------------------------------------
.../container/common/transport/server/XceiverServerGrpc.java | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f42b2dfd/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index be00c8a..4f7799d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -90,10 +90,6 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
}
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
- server = ((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
- .addService(new GrpcXceiverService(dispatcher))
- .build();
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
.maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-8838. Check that container
user is same as websocket user for interactive shell. Contributed by Eric
Yang
Posted by br...@apache.org.
YARN-8838. Check that container user is same as websocket user for interactive shell. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49824ed2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49824ed2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49824ed2
Branch: refs/heads/HDFS-13891
Commit: 49824ed260d31350d9b836a4c31319e2b3501dd0
Parents: c747830
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Nov 19 18:14:28 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Tue Nov 20 11:12:24 2018 -0800
----------------------------------------------------------------------
.../yarn/client/api/impl/YarnClientImpl.java | 4 ++
.../webapp/ContainerShellWebSocket.java | 43 +++++++++++++++++++-
.../webapp/TestNMContainerWebSocket.java | 40 +++++++++++++++++-
3 files changed, 83 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49824ed2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 3ec371c..f920ebb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -1105,6 +1105,10 @@ public class YarnClientImpl extends YarnClient {
WebSocketClient client = new WebSocketClient();
URI uri = URI.create(protocol + host + ":" + port + "/container/" +
containerId);
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ uri = URI.create(protocol + host + ":" + port + "/container/" +
+ containerId + "?user.name=" + System.getProperty("user.name"));
+ }
try {
client.start();
// The socket that receives events
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49824ed2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
index 4d74a14..4ca4db5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -35,6 +37,8 @@ import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
import org.eclipse.jetty.websocket.api.annotations.WebSocket;
import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
+import org.apache.hadoop.security.HadoopKerberosName;
+import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,14 +96,16 @@ public class ContainerShellWebSocket {
@OnWebSocketConnect
public void onConnect(Session session) {
- LOG.info(session.getRemoteAddress().getHostString() + " connected!");
-
try {
URI containerURI = session.getUpgradeRequest().getRequestURI();
String[] containerPath = containerURI.getPath().split("/");
String cId = containerPath[2];
Container container = nmContext.getContainers().get(ContainerId
.fromString(cId));
+ if (!checkAuthorization(session, container)) {
+ session.close(1008, "Forbidden");
+ }
+ LOG.info(session.getRemoteAddress().getHostString() + " connected!");
LOG.info(
"Making interactive connection to running docker container with ID: "
+ cId);
@@ -126,4 +132,37 @@ public class ContainerShellWebSocket {
}
}
+ /**
+ * Check if user is authorized to access container.
+ * @param session websocket session
+ * @param container instance of container to access
+ * @return true if user is allowed to access container.
+ * @throws IOException
+ */
+ protected boolean checkAuthorization(Session session, Container container)
+ throws IOException {
+ boolean authorized = true;
+ String user = "";
+ if (UserGroupInformation.isSecurityEnabled()) {
+ user = new HadoopKerberosName(session.getUpgradeRequest()
+ .getUserPrincipal().getName()).getShortName();
+ } else {
+ Map<String, List<String>> parameters = session.getUpgradeRequest()
+ .getParameterMap();
+ if (parameters.containsKey("user.name")) {
+ List<String> users = parameters.get("user.name");
+ user = users.get(0);
+ }
+ }
+ boolean isAdmin = false;
+ if (nmContext.getApplicationACLsManager().areACLsEnabled()) {
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+ isAdmin = nmContext.getApplicationACLsManager().isAdmin(ugi);
+ }
+ String containerUser = container.getUser();
+ if (!user.equals(containerUser) && !isAdmin) {
+ authorized = false;
+ }
+ return authorized;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49824ed2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMContainerWebSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMContainerWebSocket.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMContainerWebSocket.java
index 50042f0..0d618fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMContainerWebSocket.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMContainerWebSocket.java
@@ -27,17 +27,25 @@ import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.eclipse.jetty.websocket.api.Session;
+import org.eclipse.jetty.websocket.api.UpgradeRequest;
import org.eclipse.jetty.websocket.client.WebSocketClient;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
+import static org.mockito.Mockito.*;
import java.io.File;
+import java.io.IOException;
import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.Future;
/**
@@ -51,6 +59,7 @@ public class TestNMContainerWebSocket {
TestNMWebServer.class.getSimpleName());
private static File testLogDir = new File("target",
TestNMWebServer.class.getSimpleName() + "LogDir");
+ private WebServer server;
@Before
public void setup() {
@@ -101,7 +110,7 @@ public class TestNMContainerWebSocket {
healthChecker.init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddr);
- WebServer server = new WebServer(nmContext, resourceView,
+ server = new WebServer(nmContext, resourceView,
new ApplicationACLsManager(conf), dirsHandler);
try {
server.init(conf);
@@ -141,9 +150,36 @@ public class TestNMContainerWebSocket {
} finally {
try {
client.stop();
+ server.close();
} catch (Exception e) {
LOG.error("Failed to close client", e);
}
}
}
+
+ @Test
+ public void testContainerShellWebSocket() {
+ Context nm = mock(Context.class);
+ Session session = mock(Session.class);
+ Container container = mock(Container.class);
+ UpgradeRequest request = mock(UpgradeRequest.class);
+ ApplicationACLsManager aclManager = mock(ApplicationACLsManager.class);
+ ContainerShellWebSocket.init(nm);
+ ContainerShellWebSocket ws = new ContainerShellWebSocket();
+ List<String> names = new ArrayList<>();
+ names.add("foobar");
+ Map<String, List<String>> mockParameters = new HashMap<>();
+ mockParameters.put("user.name", names);
+ when(session.getUpgradeRequest()).thenReturn(request);
+ when(request.getParameterMap()).thenReturn(mockParameters);
+ when(container.getUser()).thenReturn("foobar");
+ when(nm.getApplicationACLsManager()).thenReturn(aclManager);
+ when(aclManager.areACLsEnabled()).thenReturn(false);
+ try {
+ boolean authorized = ws.checkAuthorization(session, container);
+ Assert.assertTrue("Not authorized", authorized);
+ } catch (IOException e) {
+ Assert.fail("Should not throw exception.");
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDDS-835. Use storageSize instead
of Long for buffer size configs in Ozone Client. Contributed by Shashikant
Banerjee.
Posted by br...@apache.org.
HDDS-835. Use storageSize instead of Long for buffer size configs in Ozone Client. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14e1a0a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14e1a0a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14e1a0a3
Branch: refs/heads/HDFS-13891
Commit: 14e1a0a3d6cf0566ba696a73699aa7ce6ed1f94f
Parents: f994b52
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Wed Nov 21 10:51:50 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Wed Nov 21 10:51:50 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 3 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 11 ++++---
.../apache/hadoop/ozone/OzoneConfigKeys.java | 21 ++++++-------
.../org/apache/hadoop/ozone/OzoneConsts.java | 5 ++++
.../main/java/org/apache/ratis/RatisHelper.java | 3 +-
.../common/src/main/resources/ozone-default.xml | 16 +++++-----
.../transport/server/XceiverServerGrpc.java | 5 ++--
.../server/ratis/XceiverServerRatis.java | 14 +++++----
.../replication/GrpcReplicationClient.java | 4 +--
.../ozone/client/io/ChunkGroupOutputStream.java | 7 ++---
.../hadoop/ozone/client/rpc/RpcClient.java | 31 +++++++++++---------
.../hadoop/ozone/MiniOzoneClusterImpl.java | 17 ++++++-----
.../rpc/TestCloseContainerHandlingByClient.java | 4 ++-
.../client/rpc/TestFailureHandlingByClient.java | 5 ----
.../om/TestMultipleContainerReadWrite.java | 5 ++--
.../web/storage/DistributedStorageHandler.java | 29 ++++++++++--------
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 9 +++---
.../ozone/web/ozShell/keys/GetKeyHandler.java | 7 +++--
.../ozone/web/ozShell/keys/PutKeyHandler.java | 6 ++--
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 4 ++-
20 files changed, 116 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index 5592c1d..a824c29 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.util.Time;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
@@ -103,7 +104,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
LOG.debug("Connecting to server Port : " + dn.getIpAddress());
ManagedChannel channel =
NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.build();
XceiverClientProtocolServiceStub asyncStub =
XceiverClientProtocolServiceGrpc.newStub(channel);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 896caed..6733b8e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -73,12 +73,12 @@ public final class ScmConfigKeys {
= 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
- public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
- 16 * 1024;
+ public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
+ "16KB";
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"dfs.container.ratis.segment.preallocated.size";
- public static final int
- DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
+ public static final String
+ DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "128MB";
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"dfs.container.ratis.statemachinedata.sync.timeout";
@@ -136,8 +136,7 @@ public final class ScmConfigKeys {
// TODO : this is copied from OzoneConsts, may need to move to a better place
public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
// 16 MB by default
- public static final int OZONE_SCM_CHUNK_SIZE_DEFAULT = 16 * 1024 * 1024;
- public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
+ public static final String OZONE_SCM_CHUNK_SIZE_DEFAULT = "16MB";
public static final String OZONE_SCM_CLIENT_PORT_KEY =
"ozone.scm.client.port";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 8a5762f..8d5c180 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -95,9 +95,9 @@ public final class OzoneConfigKeys {
"ozone.container.cache.size";
public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024;
- public static final String OZONE_SCM_BLOCK_SIZE_IN_MB =
- "ozone.scm.block.size.in.mb";
- public static final long OZONE_SCM_BLOCK_SIZE_DEFAULT = 256;
+ public static final String OZONE_SCM_BLOCK_SIZE =
+ "ozone.scm.block.size";
+ public static final String OZONE_SCM_BLOCK_SIZE_DEFAULT = "256MB";
/**
* Ozone administrator users delimited by comma.
@@ -115,12 +115,14 @@ public final class OzoneConfigKeys {
public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE =
"ozone.client.stream.buffer.flush.size";
- public static final long OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT = 64;
+ public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT =
+ "64MB";
public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE =
"ozone.client.stream.buffer.max.size";
- public static final long OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT = 128;
+ public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT =
+ "128MB";
public static final String OZONE_CLIENT_WATCH_REQUEST_TIMEOUT =
"ozone.client.watch.request.timeout";
@@ -231,12 +233,13 @@ public final class OzoneConfigKeys {
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
- public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
+ public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
- public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
- = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
+ public static final String
+ DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
// config settings to enable stateMachineData write timeout
public static final String
@@ -246,8 +249,6 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
- public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
- = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index f8f31c8..7dd9fe3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -146,6 +146,11 @@ public final class OzoneConsts {
public static final String OM_S3_PREFIX ="S3:";
public static final String OM_S3_VOLUME_PREFIX = "s3";
+ /**
+ * Max chunk size limit.
+ */
+ public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
+
/**
* Max OM Quota size of 1024 PB.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
index 1ff7695..7e454d6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
@@ -155,7 +156,7 @@ public interface RatisHelper {
RaftConfigKeys.Rpc.setType(properties, rpcType);
GrpcConfigKeys.setMessageSizeMax(properties,
- SizeInBytes.valueOf(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE));
+ SizeInBytes.valueOf(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE));
return RaftClient.newBuilder()
.setRaftGroup(group)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 865335c..767a865 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -175,7 +175,7 @@
</property>
<property>
<name>dfs.container.ratis.segment.size</name>
- <value>16384</value>
+ <value>16KB</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
<description>The size of the raft segment used by Apache Ratis on datanodes.
(16 KB by default)
@@ -183,7 +183,7 @@
</property>
<property>
<name>dfs.container.ratis.segment.preallocated.size</name>
- <value>134217728</value>
+ <value>128MB</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
<description>The size of the buffer which is preallocated for raft segment
used by Apache Ratis on datanodes.(128 MB by default)
@@ -336,7 +336,7 @@
</property>
<property>
<name>ozone.client.stream.buffer.flush.size</name>
- <value>64</value>
+ <value>64MB</value>
<tag>OZONE, CLIENT</tag>
<description>Size in mb which determines at what buffer position , a partial
flush will be initiated during write. It should be ideally a mutiple
@@ -345,7 +345,7 @@
</property>
<property>
<name>ozone.client.stream.buffer.max.size</name>
- <value>128</value>
+ <value>128MB</value>
<tag>OZONE, CLIENT</tag>
<description>Size in mb which determines at what buffer position ,
write call be blocked till acknowledgement of the fisrt partial flush
@@ -623,17 +623,17 @@
</description>
</property>
<property>
- <name>ozone.scm.block.size.in.mb</name>
- <value>256</value>
+ <name>ozone.scm.block.size</name>
+ <value>256MB</value>
<tag>OZONE, SCM</tag>
<description>
- The default size of a scm block in bytes. This is maps to the default
+ The default size of a scm block. This is maps to the default
Ozone block size.
</description>
</property>
<property>
<name>ozone.scm.chunk.size</name>
- <value>16777216</value>
+ <value>16MB</value>
<tag>OZONE, SCM, CONTAINER, PERFORMANCE</tag>
<description>
The chunk size for reading/writing chunk operations in bytes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 992f3cb..be00c8a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.common.helpers.
StorageContainerException;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.ratis.thirdparty.io.grpc.BindableService;
@@ -90,12 +91,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
server = ((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher))
.build();
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher));
for (BindableService service : additionalServices) {
nettyServerBuilder.addService(service);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 434d330..3e54700 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
@@ -33,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server
@@ -154,23 +156,25 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftConfigKeys.Rpc.setType(properties, rpc);
// set raft segment size
- final int raftSegmentSize = conf.getInt(
+ final int raftSegmentSize = (int)conf.getStorageSize(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT,
+ StorageUnit.BYTES);
RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
SizeInBytes.valueOf(raftSegmentSize));
// set raft segment pre-allocated size
- final int raftSegmentPreallocatedSize = conf.getInt(
+ final int raftSegmentPreallocatedSize = (int) conf.getStorageSize(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT,
+ StorageUnit.BYTES);
RaftServerConfigKeys.Log.Appender.setBufferCapacity(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
RaftServerConfigKeys.Log.setPreallocatedSize(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
// Set max write buffer size, which is the scm chunk size
- final int maxChunkSize = OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE;
+ final int maxChunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
RaftServerConfigKeys.Log.setWriteBufferSize(properties,
SizeInBytes.valueOf(maxChunkSize));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 8149e2c..768d266 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -33,9 +33,9 @@ import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc;
import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc.IntraDatanodeProtocolServiceStub;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
@@ -61,7 +61,7 @@ public class GrpcReplicationClient {
channel = NettyChannelBuilder.forAddress(host, port)
.usePlaintext()
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.build();
client = IntraDatanodeProtocolServiceGrpc.newStub(channel);
this.workingDirectory = workingDir;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
index 5dbe9f6..d2a4443 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -159,9 +158,9 @@ public class ChunkGroupOutputStream extends OutputStream {
this.xceiverClientManager = xceiverClientManager;
this.chunkSize = chunkSize;
this.requestID = requestId;
- this.streamBufferFlushSize = bufferFlushSize * OzoneConsts.MB;
- this.streamBufferMaxSize = bufferMaxSize * OzoneConsts.MB;
- this.blockSize = size * OzoneConsts.MB;
+ this.streamBufferFlushSize = bufferFlushSize;
+ this.streamBufferMaxSize = bufferMaxSize;
+ this.blockSize = size;
this.watchTimeout = watchTimeout;
Preconditions.checkState(chunkSize > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 98481e1..3a0f475 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.client.rpc;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
@@ -139,26 +140,28 @@ public class RpcClient implements ClientProtocol {
this.xceiverClientManager = new XceiverClientManager(conf);
- int configuredChunkSize = conf.getInt(
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
- if(configuredChunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
+ int configuredChunkSize = (int) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
+ if(configuredChunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- configuredChunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
+ configuredChunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
} else {
chunkSize = configuredChunkSize;
}
- streamBufferFlushSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
- streamBufferMaxSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
- blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
+ streamBufferFlushSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ streamBufferMaxSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 324e17b..ceea3ee 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -24,6 +24,7 @@ import java.util.Optional;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -392,6 +393,7 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
Files.createDirectories(metaDir);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
if (!chunkSize.isPresent()) {
+ //set it to 1MB by default in tests
chunkSize = Optional.of(1);
}
if (!streamBufferFlushSize.isPresent()) {
@@ -403,13 +405,14 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
if (!blockSize.isPresent()) {
blockSize = Optional.of(2 * streamBufferMaxSize.get());
}
- conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- (int) (chunkSize.get() * OzoneConsts.MB));
- conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- streamBufferFlushSize.get());
- conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- streamBufferMaxSize.get());
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, blockSize.get());
+ conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ chunkSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ streamBufferFlushSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ streamBufferMaxSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, blockSize.get(),
+ StorageUnit.MB);
configureTrace();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index abd60a1..d268aeb 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.client.rpc;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -94,7 +95,8 @@ public class TestCloseContainerHandlingByClient {
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
+ StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(7).build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index dc6747f..923300f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -83,15 +82,11 @@ public class TestFailureHandlingByClient {
maxRetries = 100;
chunkSize = (int) OzoneConsts.MB;
blockSize = 4 * chunkSize;
- conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
- conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE, 1);
- conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE, 2);
conf.setTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT, 5,
TimeUnit.SECONDS);
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(6).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
index 3848cda..d0f4ce2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -69,8 +70,8 @@ public class TestMultipleContainerReadWrite {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- // set to as small as 100 bytes per block.
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 1);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 1,
+ StorageUnit.MB);
conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 5);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
index 6a433b5..fd10055 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.web.storage;
import com.google.common.base.Strings;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.io.IOUtils;
@@ -100,27 +101,29 @@ public final class DistributedStorageHandler implements StorageHandler {
this.storageContainerLocationClient = storageContainerLocation;
this.xceiverClientManager = new XceiverClientManager(conf);
- chunkSize = conf.getInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
+ chunkSize = (int)conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS,
OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT);
- if(chunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
+ if(chunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- chunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
+ chunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
}
- streamBufferFlushSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
- streamBufferMaxSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
- blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
+ streamBufferFlushSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ streamBufferMaxSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 98a27bf..a41dae6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -25,7 +26,6 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -59,7 +59,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVI
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
/**
* Implementation of keyManager.
@@ -87,8 +87,9 @@ public class KeyManagerImpl implements KeyManager {
String omId) {
this.scmBlockClient = scmBlockClient;
this.metadataManager = metadataManager;
- this.scmBlockSize = conf.getLong(OZONE_SCM_BLOCK_SIZE_IN_MB,
- OZONE_SCM_BLOCK_SIZE_DEFAULT) * OzoneConsts.MB;
+ this.scmBlockSize = (long) conf
+ .getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT,
+ StorageUnit.BYTES);
this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY,
DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
this.preallocateMax = conf.getLong(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
index be9342d..4e86699 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
@@ -24,6 +24,7 @@ import java.io.FileOutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -94,8 +95,10 @@ public class GetKeyHandler extends Handler {
OzoneInputStream keyInputStream = bucket.readKey(keyName);
if (dataFilePath != null) {
FileOutputStream outputStream = new FileOutputStream(dataFile);
- IOUtils.copyBytes(keyInputStream, outputStream, new OzoneConfiguration()
- .getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
+ IOUtils.copyBytes(keyInputStream, outputStream,
+ (int) new OzoneConfiguration()
+ .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
+ OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES));
outputStream.close();
} else {
throw new OzoneClientException(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
index b2ab68f..f86af3f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileInputStream;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -106,8 +107,9 @@ public class PutKeyHandler extends Handler {
.createKey(keyName, dataFile.length(), replicationType,
replicationFactor);
FileInputStream fileInputStream = new FileInputStream(dataFile);
- IOUtils.copyBytes(fileInputStream, outputStream,
- conf.getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
+ IOUtils.copyBytes(fileInputStream, outputStream, (int) conf
+ .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT,
+ StorageUnit.BYTES));
outputStream.close();
fileInputStream.close();
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
index d3bc857..913bb3e 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.ozone;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -64,7 +65,8 @@ public class TestOzoneFSInputStream {
@BeforeClass
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 10,
+ StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(10)
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDDS-845. Create a new raftClient
instance for every watch request for Ratis. Contributed by Shashikant
Banerjee.
Posted by br...@apache.org.
HDDS-845. Create a new raftClient instance for every watch request for Ratis. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10cf5773
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10cf5773
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10cf5773
Branch: refs/heads/HDFS-13891
Commit: 10cf5773ba32566dd76730e32a3ccdf2b3bd4d09
Parents: cfb915f
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Mon Nov 19 14:38:51 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Mon Nov 19 14:38:51 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 3 +-
.../hadoop/hdds/scm/XceiverClientRatis.java | 38 +++++++++++++++-----
.../hadoop/hdds/scm/XceiverClientSpi.java | 3 +-
3 files changed, 33 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10cf5773/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index c6b19ab..5592c1d 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -290,7 +290,8 @@ public class XceiverClientGrpc extends XceiverClientSpi {
@Override
public void watchForCommit(long index, long timeout)
- throws InterruptedException, ExecutionException, TimeoutException {
+ throws InterruptedException, ExecutionException, TimeoutException,
+ IOException {
// there is no notion of watch for commit index in standalone pipeline
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10cf5773/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 6b3b001..b238a09 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -122,11 +122,15 @@ public final class XceiverClientRatis extends XceiverClientSpi {
public void close() {
final RaftClient c = client.getAndSet(null);
if (c != null) {
- try {
- c.close();
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
+ closeRaftClient(c);
+ }
+ }
+
+ private void closeRaftClient(RaftClient raftClient) {
+ try {
+ raftClient.close();
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
}
}
@@ -145,19 +149,35 @@ public final class XceiverClientRatis extends XceiverClientSpi {
@Override
public void watchForCommit(long index, long timeout)
- throws InterruptedException, ExecutionException, TimeoutException {
- // TODO: Create a new Raft client instance to watch
- CompletableFuture<RaftClientReply> replyFuture = getClient()
+ throws InterruptedException, ExecutionException, TimeoutException,
+ IOException {
+ LOG.debug("commit index : {} watch timeout : {}", index, timeout);
+ // create a new RaftClient instance for watch request
+ RaftClient raftClient =
+ RatisHelper.newRaftClient(rpcType, getPipeline(), retryPolicy);
+ CompletableFuture<RaftClientReply> replyFuture = raftClient
.sendWatchAsync(index, RaftProtos.ReplicationLevel.ALL_COMMITTED);
try {
replyFuture.get(timeout, TimeUnit.MILLISECONDS);
} catch (TimeoutException toe) {
LOG.warn("3 way commit failed ", toe);
- getClient()
+
+ closeRaftClient(raftClient);
+ // generate a new raft client instance again so that next watch request
+ // does not get blocked for the previous one
+
+ // TODO : need to remove the code to create the new RaftClient instance
+ // here once the watch request bypassing sliding window in Raft Client
+ // gets fixed.
+ raftClient =
+ RatisHelper.newRaftClient(rpcType, getPipeline(), retryPolicy);
+ raftClient
.sendWatchAsync(index, RaftProtos.ReplicationLevel.MAJORITY_COMMITTED)
.get(timeout, TimeUnit.MILLISECONDS);
LOG.info("Could not commit " + index + " to all the nodes."
+ "Committed by majority.");
+ } finally {
+ closeRaftClient(raftClient);
}
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10cf5773/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
index 7000660..e9896dc 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
@@ -126,5 +126,6 @@ public abstract class XceiverClientSpi implements Closeable {
public abstract HddsProtos.ReplicationType getPipelineType();
public abstract void watchForCommit(long index, long timeout)
- throws InterruptedException, ExecutionException, TimeoutException;
+ throws InterruptedException, ExecutionException, TimeoutException,
+ IOException;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HADOOP-15939. Filter overlapping
objenesis class in hadoop-client-minicluster. Contributed by Xiaoyu Yao.
Posted by br...@apache.org.
HADOOP-15939. Filter overlapping objenesis class in hadoop-client-minicluster. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/397f523e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/397f523e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/397f523e
Branch: refs/heads/HDFS-13891
Commit: 397f523e22a4f76b5484bed26ef4e6d40200611e
Parents: b5d7b29
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Mon Nov 19 16:28:58 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Mon Nov 19 16:28:58 2018 -0800
----------------------------------------------------------------------
hadoop-client-modules/hadoop-client-minicluster/pom.xml | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/397f523e/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index ddebfa9..c356b19 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -729,6 +729,8 @@
<exclude>objenesis-license.txt</exclude>
<exclude>org/hamcrest/**/*.class</exclude>
<exclude>org/hamcrest/*.class</exclude>
+ <exclude>org/objenesis/**/*.class</exclude>
+ <exclude>org/objenesis/*.class</exclude>
</excludes>
</filter>
<!-- skip grizzly internals we don't need to run. -->
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDFS-14024. RBF:
ProvidedCapacityTotal json exception in NamenodeHeartbeatService. Contributed
by CR Hota.
Posted by br...@apache.org.
HDFS-14024. RBF: ProvidedCapacityTotal json exception in NamenodeHeartbeatService. Contributed by CR Hota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10ea6335
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10ea6335
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10ea6335
Branch: refs/heads/HDFS-13891
Commit: 10ea633552a552c32fa0259752f852a93e6eea63
Parents: 5e4e098
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Nov 1 11:49:33 2018 -0700
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:34:52 2018 +0530
----------------------------------------------------------------------
.../hdfs/server/federation/router/NamenodeHeartbeatService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10ea6335/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
index a1adf77..1349aa3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
@@ -351,7 +351,7 @@ public class NamenodeHeartbeatService extends PeriodicService {
jsonObject.getLong("PendingReplicationBlocks"),
jsonObject.getLong("UnderReplicatedBlocks"),
jsonObject.getLong("PendingDeletionBlocks"),
- jsonObject.getLong("ProvidedCapacityTotal"));
+ jsonObject.optLong("ProvidedCapacityTotal"));
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-8936. Bump up Atsv2 hbase
versions. Contributed by Vrushali C.
Posted by br...@apache.org.
YARN-8936. Bump up Atsv2 hbase versions. Contributed by Vrushali C.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f63e4e4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f63e4e4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f63e4e4f
Branch: refs/heads/HDFS-13891
Commit: f63e4e4f2eee6b75c3a6c3fb5926d44c7e846145
Parents: c8b3dfa
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Nov 21 15:02:22 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Nov 21 15:02:22 2018 +0530
----------------------------------------------------------------------
hadoop-project/pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f63e4e4f/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 6f83280..a205d2b 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -154,8 +154,8 @@
<swagger-annotations-version>1.5.4</swagger-annotations-version>
<snakeyaml.version>1.16</snakeyaml.version>
- <hbase.one.version>1.2.6</hbase.one.version>
- <hbase.two.version>2.0.0-beta-1</hbase.two.version>
+ <hbase.one.version>1.4.8</hbase.one.version>
+ <hbase.two.version>2.0.2</hbase.two.version>
<junit.version>4.12</junit.version>
<junit.jupiter.version>5.3.1</junit.jupiter.version>
<junit.vintage.version>5.3.1</junit.vintage.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDDS-849. Fix NPE in
TestKeyValueHandler because of audit log write. Contributed by Dinesh
Chitlangia.
Posted by br...@apache.org.
HDDS-849. Fix NPE in TestKeyValueHandler because of audit log write.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7438a1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7438a1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7438a1b
Branch: refs/heads/HDFS-13891
Commit: e7438a1b38ff1d2bb25aa9d849a227c6f354143b
Parents: 8b23814
Author: Nanda kumar <na...@apache.org>
Authored: Mon Nov 19 22:47:43 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Mon Nov 19 22:47:43 2018 +0530
----------------------------------------------------------------------
.../hadoop/ozone/container/keyvalue/TestKeyValueHandler.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7438a1b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 698af35..29d74c2 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -90,6 +90,10 @@ public class TestKeyValueHandler {
Mockito.when(handler.handle(any(), any())).thenCallRealMethod();
doCallRealMethod().when(dispatcher).setMetricsForTesting(any());
dispatcher.setMetricsForTesting(Mockito.mock(ContainerMetrics.class));
+ Mockito.when(dispatcher.buildAuditMessageForFailure(any(), any(), any()))
+ .thenCallRealMethod();
+ Mockito.when(dispatcher.buildAuditMessageForSuccess(any(), any()))
+ .thenCallRealMethod();
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org